diff options
| -rw-r--r-- | INFO | 10 | ||||
| -rw-r--r-- | benchmarks/perftest/dhrystone.yaml | 25 | ||||
| -rw-r--r-- | benchmarks/perftest/dpi.yaml | 26 | ||||
| -rw-r--r-- | benchmarks/perftest/fio.yaml | 29 | ||||
| -rw-r--r-- | benchmarks/perftest/iperf.yaml | 25 | ||||
| -rw-r--r-- | benchmarks/perftest/ramspeed.yaml | 24 | ||||
| -rw-r--r-- | benchmarks/perftest/ssl.yaml | 30 | ||||
| -rw-r--r-- | benchmarks/perftest/summary | 23 | ||||
| -rw-r--r-- | benchmarks/perftest/whetstone.yaml | 30 | ||||
| -rw-r--r-- | cli/commands/perftest.py | 55 | ||||
| -rw-r--r-- | cli/setup.py | 23 | ||||
| -rw-r--r-- | config/QtipKey | 27 | ||||
| -rw-r--r-- | config/QtipKey.pub | 1 | ||||
| -rw-r--r-- | config/SampleHeat.yaml | 74 | ||||
| -rw-r--r-- | docker/Dockerfile | 2 | ||||
| -rw-r--r-- | docker/README.md | 11 | ||||
| -rwxr-xr-x | docker/run_qtip.sh | 9 | ||||
| -rw-r--r-- | docs/configguide/configuration.rst | 60 | ||||
| -rw-r--r-- | docs/userguide/_01-compute.rst | 6 | ||||
| -rw-r--r-- | docs/userguide/annex.rst | 2 | ||||
| -rw-r--r-- | docs/userguide/introduction.rst | 72 | ||||
| -rw-r--r-- | func/args_handler.py | 10 | ||||
| -rw-r--r-- | func/env_setup.py | 6 | ||||
| -rw-r--r-- | func/spawn_vm.py | 207 | ||||
| -rw-r--r-- | qtip.py | 2 | ||||
| -rw-r--r-- | scripts/cleanup_creds.sh | 14 | ||||
| -rwxr-xr-x | scripts/file_permission.sh | 3 | ||||
| -rw-r--r-- | tests/spawn_vm_test.py | 11 |
28 files changed, 438 insertions, 379 deletions
@@ -13,13 +13,13 @@ IRC: #opnfv-qtip@freenode Committers: Morgan Richomme morgan.richomme@orange.com -Nauman Ahad nauman.ahad@xflowresearch.com Prabu Kuppuswamy prabu.kuppuswamy@spirent.com Prakash Ramchandran prakash.ramchandran@huawei.com Trevor Cooper trevor.cooper@intel.com Wenjing Chu chu.wenjing@gmail.com Yujun Zhang zhang.yujunz@zte.com.cn Zhifeng Jiang Jiang.ZhiFeng@zte.com.cn +Zhihui Wu wu.zhihui1@zte.com.cn Link to committer policy: @@ -32,6 +32,8 @@ Link to TSC approval of the project: Link(s) to approval of committers update: -- http://lists.opnfv.org/pipermail/opnfv-tsc/2016-July/002745.html -- http://lists.opnfv.org/pipermail/opnfv-tsc/2016-July/002755.html -- http://lists.opnfv.org/pipermail/opnfv-tsc/2016-August/002848.html +- https://lists.opnfv.org/pipermail/opnfv-tsc/2016-July/002745.html +- https://lists.opnfv.org/pipermail/opnfv-tsc/2016-July/002755.html +- https://lists.opnfv.org/pipermail/opnfv-tsc/2016-August/002848.html +- https://lists.opnfv.org/pipermail/opnfv-tsc/2016-October/003104.html +- https://lists.opnfv.org/pipermail/opnfv-tsc/2016-October/003118.html diff --git a/benchmarks/perftest/dhrystone.yaml b/benchmarks/perftest/dhrystone.yaml index 5f83bbb0..7899bbd7 100644 --- a/benchmarks/perftest/dhrystone.yaml +++ b/benchmarks/perftest/dhrystone.yaml @@ -19,29 +19,29 @@ register: home_dir - name: cleaning tempT - shell: sudo rm -rf $HOME/tempT + file: path=$HOME/tempT state=absent - name: cleaning qtip_result - shell: sudo rm -rf $HOME/qtip_result + file: path=$HOME/qtip_result state=absent - name: make directory - shell: sudo mkdir $HOME/qtip_result + file: path=$HOME/qtip_result state=directory - - include: ../common/sys_proxy_pbook.yaml + - include: ./common/sys_proxy_pbook.yaml - - include: ../common/sys_info_pbook.yaml + - include: ./common/sys_info_pbook.yaml vars: network: false - name: Installing UnixBench dependencies if CentOS - shell: sudo yum install git gcc patch perl-Time-HiRes -y + shell: yum install git gcc patch perl-Time-HiRes -y when: ansible_os_family == "RedHat" - name: Installing UnixBench dependencies if Ubuntu - shell: sudo apt-get install git gcc patch perl -y + shell: apt-get install git gcc patch perl -y when: ansible_os_family == "Debian" - - include: ../common/git_proxy_pbook.yaml + - include: ./common/git_proxy_pbook.yaml - name: Clone unixbench git: repo=https://github.com/kdlucas/byte-unixbench.git @@ -88,11 +88,10 @@ with_items: "{{copy_log_results.stdout_lines}}" - name: cleaning tempT - shell: sudo rm -rf $HOME/tempT + file: path=$HOME/tempT state=absent - name: cleaning_qtip_result - shell: sudo rm -rf $HOME/qtip_result - + file: path=$HOME/qtip_result state=absent - hosts: localhost connection: local @@ -103,10 +102,10 @@ shell: ( find {{workingdir}}/{{Dest_dir}}/dhrystone/dhrystone_temp/ -name "*.json" | xargs cp -t {{workingdir}}/{{Dest_dir}}/dhrystone/) - name: making_logs_folder - shell: mkdir -p {{workingdir}}/{{Dest_dir}}/dhrystone/logs + file: path={{workingdir}}/{{Dest_dir}}/dhrystone/logs state=directory - name: extracting_log shell: ( find {{workingdir}}/{{Dest_dir}}/dhrystone/dhrystone_temp/ -name "*.log" | xargs cp -t {{workingdir}}/{{Dest_dir}}/dhrystone/logs) - name: removing dhrystone_temp - shell: rm -rf {{workingdir}}/{{Dest_dir}}/dhrystone/dhrystone_temp + file: path={{workingdir}}/{{Dest_dir}}/dhrystone/dhrystone_temp state=directory diff --git a/benchmarks/perftest/dpi.yaml b/benchmarks/perftest/dpi.yaml index 59d01d58..5ce5d09b 100644 --- a/benchmarks/perftest/dpi.yaml +++ b/benchmarks/perftest/dpi.yaml @@ -22,17 +22,17 @@ register: home_dir - name: cleaning - shell: sudo rm -rf $HOME/tempD + file: path=$HOME/tempD state=absent - name: cleaning previous results - shell: sudo rm -rf $HOME/qtip_result + file: path=$HOME/qtip_result state=absent - name: make qtip_result - shell: sudo mkdir $HOME/qtip_result + file: path=$HOME/qtip_result state=directory - - include: ../common/sys_proxy_pbook.yaml + - include: ./common/sys_proxy_pbook.yaml - - include: ../common/sys_info_pbook.yaml + - include: ./common/sys_info_pbook.yaml vars: network: false @@ -45,9 +45,9 @@ when: ansible_os_family == "Debian" - name: making nDPI temporary directory - shell: sudo mkdir $HOME/tempD + file: path=$HOME/tempD state=directory - - include: ../common/git_proxy_pbook.yaml + - include: ./common/git_proxy_pbook.yaml - name: Clone nDPI git: repo=https://github.com/ntop/nDPI.git @@ -66,7 +66,7 @@ shell: cd $HOME/tempD/nDPI/example && wget http://build.opnfv.org/artifacts.opnfv.org/qtip/utilities/test.pcap - name: fetch Averaging script - copy: src=../etc/dpi_average.sh dest={{home_dir.stdout}}/tempD/nDPI/example mode=777 + copy: src=./etc/dpi_average.sh dest={{home_dir.stdout}}/tempD/nDPI/example mode=777 - name: Run nDPI benchmark shell: cd $HOME/tempD/nDPI/example && sudo ./dpi_average.sh @@ -102,6 +102,12 @@ fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest={{workingdir}}/{{Dest_dir}}/dpi/dpi_temp with_items: "{{copy_log_results.stdout_lines}}" + - name: cleaning tempD + file: path=$HOME/tempD state=absent + + - name: cleaning_qtip_result + file: path=$HOME/qtip_result state=absent + - hosts: localhost connection: local gather_facts: no @@ -111,10 +117,10 @@ shell: ( find {{workingdir}}/{{Dest_dir}}/dpi/dpi_temp/ -name "*.json" | xargs cp -t {{workingdir}}/{{Dest_dir}}/dpi/) - name: making_logs_folder - shell: mkdir -p {{workingdir}}/{{Dest_dir}}/dpi/logs + file: path={{workingdir}}/{{Dest_dir}}/dpi/logs state=directory - name: extracting_log shell: ( find {{workingdir}}/{{Dest_dir}}/dpi/dpi_temp/ -name "*.log" | xargs cp -t {{workingdir}}/{{Dest_dir}}/dpi/logs) - name: removing dpi_temp - shell: rm -rf {{workingdir}}/{{Dest_dir}}/dpi/dpi_temp + file: path={{workingdir}}/{{Dest_dir}}/dpi/dpi_temp state=absent diff --git a/benchmarks/perftest/fio.yaml b/benchmarks/perftest/fio.yaml index bd29c127..94a4c80d 100644 --- a/benchmarks/perftest/fio.yaml +++ b/benchmarks/perftest/fio.yaml @@ -19,21 +19,21 @@ shell: echo $HOME register: home_dir - - name: cleaning - shell: sudo rm -rf $HOME/fio + - name: cleaning fio directory + file: path=$HOME/fio state=absent - name: cleaning previous results - shell: sudo rm -rf $HOME/qtip_result + file: path=$HOME/qtip_result state=absent - name: making fio temporary directory - shell: sudo mkdir $HOME/fio + file: path=$HOME/fio state=directory - name: making results temporary directory - shell: sudo mkdir $HOME/qtip_result + file: path=$HOME/qtip_result state=directory - - include: ../common/sys_proxy_pbook.yaml + - include: ./common/sys_proxy_pbook.yaml - - include: ../common/sys_info_pbook.yaml + - include: ./common/sys_info_pbook.yaml vars: network: false @@ -47,13 +47,15 @@ - name: Fetching fio shell: cd $HOME/fio/ && wget http://freecode.com/urls/3aa21b8c106cab742bf1f20d60629e3f -O fio.tar.gz + - name: Untar fio shell: cd $HOME/fio/ && sudo tar -zxvf fio.tar.gz + - name: configure shell: cd $HOME/fio/fio-2.1.10 && sudo ./configure && sudo make - name: Fetching fio job - copy: src=../etc/fio_test_job dest={{home_dir.stdout}}/fio/fio-2.1.10/ + copy: src=./etc/fio_test_job dest={{home_dir.stdout}}/fio/fio-2.1.10/ - name: Benchmarking block storage through fio shell: cd $HOME/fio/fio-2.1.10 && sudo ./fio --output-format=json --output=$HOME/qtip_result/fio_result.json fio_test_job @@ -86,6 +88,12 @@ fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest={{workingdir}}/{{Dest_dir}}/fio/fio_temp with_items: "{{copy_log_results.stdout_lines}}" + - name: cleaning fio + file: path=$HOME/fio state=absent + + - name: cleaning_qtip_result + file: path=$HOME/qtip_result + - hosts: localhost connection: local gather_facts: no @@ -95,11 +103,10 @@ shell: ( find {{workingdir}}/{{Dest_dir}}/fio/fio_temp/ -name "*.json" | xargs cp -t {{workingdir}}/{{Dest_dir}}/fio/) - name: making_logs_folder - shell: mkdir -p {{workingdir}}/{{Dest_dir}}/fio/logs + file: path={{workingdir}}/{{Dest_dir}}/fio/logs state=directory - name: extracting_log shell: ( find {{workingdir}}/{{Dest_dir}}/fio/fio_temp/ -name "*.log" | xargs cp -t {{workingdir}}/{{Dest_dir}}/fio/logs) - name: removing fio_log - shell: rm -rf {{workingdir}}/{{Dest_dir}}/fio/fio_temp - + file: {{workingdir}}/{{Dest_dir}}/fio/fio_temp state=absent diff --git a/benchmarks/perftest/iperf.yaml b/benchmarks/perftest/iperf.yaml index af5b836a..481a2e3e 100644 --- a/benchmarks/perftest/iperf.yaml +++ b/benchmarks/perftest/iperf.yaml @@ -39,21 +39,21 @@ shell: echo $HOME register: home_dir - - name: cleaning - shell: sudo rm -rf $HOME/iperf + - name: cleaning iperf directory + file: path=$HOME/iperf state=absent - name: cleaning previous results - shell: sudo rm -rf $HOME/qtip_result + file: path=$HOME/qtip_result state=absent - name: making Iperf temporary directory - shell: sudo mkdir $HOME/iperf + file: path=$HOME/iperf state=directory - name: making results temporary directory - shell: sudo mkdir $HOME/qtip_result + file: path=$HOME/qtip_result state=directory - - include: ../common/sys_proxy_pbook.yaml + - include: ./common/sys_proxy_pbook.yaml - - include: ../common/sys_info_pbook.yaml + - include: ./common/sys_info_pbook.yaml vars: network: true @@ -131,6 +131,11 @@ with_items: "{{copy_log_results.stdout_lines}}" when: rolename =="2-host" and "{{ip2}}" == '' + - name: cleaning iperf directory + file: path=$HOME/iperf state=absent + + - name: cleaning previous results + file: path=$HOME/qtip_result state=absent - hosts: localhost connection: local @@ -147,15 +152,15 @@ when: rolename == "2-host" - name: making_logs_folder - shell: mkdir -p {{workingdir}}/{{Dest_dir}}/iperf/logs + file: path={{workingdir}}/{{Dest_dir}}/iperf/logs state=directory - name: extracting_log shell: ( find {{workingdir}}/{{Dest_dir}}/iperf/iperf_temp/ -name "*.log" | xargs cp -t {{workingdir}}/{{Dest_dir}}/iperf/logs) when: rolename == "2-host" - name: removing iperf_raw file - shell: rm -rf {{workingdir}}/{{Dest_dir}}/iperf/iperf_raw.json + file: path={{workingdir}}/{{Dest_dir}}/iperf/iperf_raw.json state=absent when: rolename == "2-host" - name: removing iperf_temp - shell: rm -rf {{workingdir}}/{{Dest_dir}}/iperf/iperf_temp + file: path={{workingdir}}/{{Dest_dir}}/iperf/iperf_temp state=absent diff --git a/benchmarks/perftest/ramspeed.yaml b/benchmarks/perftest/ramspeed.yaml index f62c6056..fb624c85 100644 --- a/benchmarks/perftest/ramspeed.yaml +++ b/benchmarks/perftest/ramspeed.yaml @@ -19,19 +19,19 @@ shell: echo $HOME register: home_dir - - name: cleaning - shell: sudo rm -rf $HOME/ramspeed + - name: cleaning ramspeed directory + file: path=$HOME/ramspeed state=absent - name: cleaning previous results - shell: sudo rm -rf $HOME/qtip_result + file: path=$HOME/qtip_result state=absent - name: making ramspeed temporary directory - shell: sudo mkdir $HOME/ramspeed + file: path=$HOME/ramspeed state=directory - name: making results temporary directory - shell: sudo mkdir $HOME/qtip_result + file: path=$HOME/qtip_result state=directory - - include: ../common/sys_proxy_pbook.yaml + - include: ./common/sys_proxy_pbook.yaml - include: ./common/sys_info_pbook.yaml vars: @@ -67,7 +67,7 @@ copy: src={{workdingdir}}/utils/transform/ramspeed_transform.py dest={{home_dir.stdout}}/qtip_result - name: Transforming result - shell: cd $HOME/qtip_result && sudo python ramspd_transform.py + shell: cd $HOME/qtip_result && sudo python ramspeed_transform.py - name: copy report formation script copy: src={{workingdir}}/utils/transform/final_report.py dest={{home_dir.stdout}}/qtip_result @@ -91,6 +91,12 @@ fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest={{workingdir}}/{{Dest_dir}}/ramspeed/ramspeed_temp with_items: "{{copy_log_results.stdout_lines}}" + - name: cleaning ramspeed directory + file: path=$HOME/ramspeed state=absent + + - name: cleaning previous results + file: path=$HOME/qtip_result state=absent + - hosts: localhost connection: local gather_facts: no @@ -100,10 +106,10 @@ shell: ( find {{workingdir}}/{{Dest_dir}}/ramspeed/ramspeed_temp/ -name "*.json" | xargs cp -t {{workingdir}}/{{Dest_dir}}/ramspeed/) - name: making_logs_folder - shell: mkdir -p {{workingdir}}/{{Dest_dir}}/ramspeed/logs + file: path={{workingdir}}/{{Dest_dir}}/ramspeed/logs state=directory - name: extracting_log shell: ( find {{workingdir}}/{{Dest_dir}}/ramspeed/ramspeed_temp/ -name "*.log" | xargs cp -t {{workingdir}}/{{Dest_dir}}/ramspeed/logs) - name: removing ramspeed_log - shell: rm -rf {{workingdir}}/{{Dest_dir}}/ramspeed/ramspeed_temp + file: path={{workingdir}}/{{Dest_dir}}/ramspeed/ramspeed_temp state=absent diff --git a/benchmarks/perftest/ssl.yaml b/benchmarks/perftest/ssl.yaml index 4b5f036a..ef36265e 100644 --- a/benchmarks/perftest/ssl.yaml +++ b/benchmarks/perftest/ssl.yaml @@ -18,21 +18,21 @@ shell: sudo echo $HOME register: home_dir - - name: cleaning - shell: sudo rm -rf $HOME/Open_SSL + - name: cleaning Open_SSL directory + file: path=$HOME/Open_SSL state=absent - - name: cleaning previous results - shell: sudo rm -rf $HOME/qtip_result + - name: cleaning_qtip_result + file: path=$HOME/qtip_result state=absent - name: making OpenSSL temporary directory - shell: sudo mkdir $HOME/Open_SSL + file: path=$HOME/Open_SSL state=directory - name: making results temporary directory - shell: sudo mkdir $HOME/qtip_result + file: path=$HOME/qtip_result state=directory - - include: ../common/sys_proxy_pbook.yaml + - include: ./common/sys_proxy_pbook.yaml - - include: ../common/sys_info_pbook.yaml + - include: ./common/sys_info_pbook.yaml vars: network: false @@ -53,10 +53,10 @@ shell: cd $HOME/Open_SSL/openssl-1.0.2f && sudo ./config - name: make - shell: cd $HOME/Open_SSL/openssl-1.0.2f && sudo make + shell: cd $HOME/Open_SSL/openssl-1.0.2f && sudo make - name: make install - shell: cd $HOME/Open_SSL/openssl-1.0.2f && sudo make install + shell: cd $HOME/Open_SSL/openssl-1.0.2f && sudo make install - name: Benchmarking RSA signatures shell: cd $HOME/Open_SSL/openssl-1.0.2f/apps && sudo ./openssl speed rsa >> $HOME/qtip_result/RSA_dump @@ -92,6 +92,12 @@ fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest={{workingdir}}/{{Dest_dir}}/ssl/ssl_temp with_items: "{{copy_log_results.stdout_lines}}" + - name: cleaning Open_SSL directory + file: path=$HOME/Open_SSL state=absent + + - name: cleaning_qtip_result + file: path=$HOME/qtip_result state=absent + - hosts: localhost connection: local gather_facts: no @@ -104,10 +110,10 @@ shell: ( find {{workingdir}}/{{Dest_dir}}/ssl/ssl_temp/ -name "*.json" | xargs cp -t {{workingdir}}/{{Dest_dir}}/ssl/) - name: making_logs_folder - shell: mkdir -p {{workingdir}}/{{Dest_dir}}/ssl/logs + file: path={{workingdir}}/{{Dest_dir}}/ssl/logs state=directory - name: extracting_log shell: ( find {{workingdir}}/{{Dest_dir}}/ssl/ssl_temp/ -name "*.log" | xargs cp -t {{workingdir}}/{{Dest_dir}}/ssl/logs) - name: removing ssl_temp - shell: rm -rf {{workingdir}}/{{Dest_dir}}/ssl/ssl_temp + file: path={{workingdir}}/{{Dest_dir}}/ssl/ssl_temp state=absent diff --git a/benchmarks/perftest/summary b/benchmarks/perftest/summary new file mode 100644 index 00000000..5891408c --- /dev/null +++ b/benchmarks/perftest/summary @@ -0,0 +1,23 @@ +--- + + test_cases: + - name: fio + description: Storage performance benchmark + + - name: iperf + description: Measures the network throughput + + - name: dpi + description: Traffic classification rate provides a measure for CPU performance + + - name: ssl + description: CPU performance benchmark + + - name: dhrystone + description: Evaluate CPU's integer operation performance + + - name: whetstone + description: Evaluate CPU's floating point performance + + - name: ramspeed + description: Measures the memory performance of a machine diff --git a/benchmarks/perftest/whetstone.yaml b/benchmarks/perftest/whetstone.yaml index 32a22fe3..4dcddb99 100644 --- a/benchmarks/perftest/whetstone.yaml +++ b/benchmarks/perftest/whetstone.yaml @@ -18,18 +18,18 @@ shell: echo $HOME register: home_dir - - name: cleaning tempT - shell: sudo rm -rf $HOME/tempT + - name: cleaning tempT directory + file: path=$HOME/tempT state=absent - - name: cleaning_qtip_result - shell: sudo rm -rf $HOME/qtip_result + - name: cleaning qtip result directory + file: path=$HOME/qtip_result state=absent - - name: make directory - shell: sudo mkdir $HOME/qtip_result + - name: making qtip_result directory + file: path=$HOME/qtip_result state=directory - - include: ../common/sys_proxy_pbook.yaml + - include: ./common/sys_proxy_pbook.yaml - - include: ../common/sys_info_pbook.yaml + - include: ./common/sys_info_pbook.yaml vars: network: false @@ -41,7 +41,7 @@ shell: sudo apt-get install git gcc patch perl -y when: ansible_os_family == "Debian" - - include: ./git_proxy_pbook.yaml + - include: ./common/git_proxy_pbook.yaml - name: Clone unixbench git: repo=https://github.com/kdlucas/byte-unixbench.git @@ -87,11 +87,11 @@ fetch: src={{home_dir.stdout}}/qtip_result/log/{{item}} dest={{workingdir}}/{{Dest_dir}}/whetstone/whetstone_temp with_items: "{{copy_log_results.stdout_lines}}" - - name: cleaning tempT - shell: sudo rm -rf $HOME/tempT + - name: cleaning tempT directory + file: path=$HOME/tempT state=absent - - name: cleaning_qtip_result - shell: sudo rm -rf $HOME/qtip_result + - name: cleaning qtip result directory + file: path=$HOME/qtip_result state=absent - hosts: localhost connection: local @@ -102,10 +102,10 @@ shell: ( find {{workingdir}}/{{Dest_dir}}/whetstone/whetstone_temp/ -name "*.json" | xargs cp -t {{workingdir}}/{{Dest_dir}}/whetstone/) - name: making_logs_folder - shell: mkdir -p {{workingdir}}/{{Dest_dir}}/whetstone/logs + file: path={{workingdir}}/{{Dest_dir}}/whetstone/logs state=directory - name: extracting_log shell: ( find {{workingdir}}/{{Dest_dir}}/whetstone/whetstone_temp/ -name "*.log" | xargs cp -t {{workingdir}}/{{Dest_dir}}/whetstone/logs) - name: removing whetstone_temp - shell: rm -rf {{workingdir}}/{{Dest_dir}}/whetstone/whetstone_temp + file: {{workingdir}}/{{Dest_dir}}/whetstone/whetstone_temp state=absent diff --git a/cli/commands/perftest.py b/cli/commands/perftest.py new file mode 100644 index 00000000..c163070a --- /dev/null +++ b/cli/commands/perftest.py @@ -0,0 +1,55 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +from prettytable import PrettyTable +import yaml +import click + + +class PerfTest: + + def __init__(self): + self.path = 'benchmarks/perftest/summary' + + def list(self): + table = PrettyTable(["Name", "Description"]) + table.align = 'l' + with open(self.path) as tests: + line = tests.read() + data = yaml.safe_load(line)['test_cases'] + for i in range(0, len(data)): + points = data[i] + table.add_row([points['name'], points['description']]) + click.echo(table) + + def run(self): + click.echo("Run a perftest") + + +@click.group() +def cli(): + pass + + +@cli.group() +@click.pass_context +def perftest(ctx): + pass + +_perftest = PerfTest() + + +@perftest.command("list", help="Lists all perftest benchmarks.") +def list(): + _perftest.list() + + +@perftest.command("run", help="Executes a single perftest benchmark.") +def execute(): + _perftest.run() diff --git a/cli/setup.py b/cli/setup.py deleted file mode 100644 index 1c2808b8..00000000 --- a/cli/setup.py +++ /dev/null @@ -1,23 +0,0 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -from setuptools import setup - -setup( - name='qtip', - py_modules=['cli_entry'], - include_package_data=True, - install_requires=[ - 'click', - ], - entry_points=''' - [console_scripts] - qtip=cli_entry:cli - ''', -) diff --git a/config/QtipKey b/config/QtipKey deleted file mode 100644 index 3f520775..00000000 --- a/config/QtipKey +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEAxVpAC+Q8LTyftATCgVHIKvDkXYzHI/8CZeKlVWHlHUA0/6Eu -qSrhzbpOwSaQi2C3x4Eszvh7/CfuVMql11yShsXwFD7aV6x0YG7q8IUScHILUFal -m0Fx52No7IpB3llluUV+kh8ay68V9OGvMZrQ+wWw4ieh2alUnpvFwteXmb8NQtXW -6pm2algUVixc4R3//YKLnGkj93pGFlQlDz5Q0vg+69jHNgodGewIrxCWeZk2pnn0 -vNAdWTQUTm7z/1uYz6AIeR/Wx66msmchFRmmDpP7aHXSSQEBySF7v/GEsZ1JwtBW -07WebNiTv9wYoOfyKVuZuTbBwjqlT4x5CpIJEwIDAQABAoIBAFHcDZThJtTcwKG7 -F7LsaUrmgNMNAc08iZIZYNr5sD9h0pn2EZS55M+g5+nWRT6K77AhNKTlDQiax5EE -PaYHEAA3Ok4rhAW50svtNiZuDCf4Jhk815R+oPCJm4wCDTBdhIRE/ys9G7BA/6qD -slexD94Pjj9AkTHnuuHPW0hmhMuQaSg96EO6QUWCG/HeMFDGcorBlM0s+NeR46Jm -dI8tOvt+rSa68oDtKEwhUbP5cATNdAITzo9+4We5EnYhW9/nRsaF/um2BPih6JnU -zG9udvZwj+YARmEfxhXzeRDpi30qYil/+CUF+qdyd8eoPBvhsW6rr1TA6XYGmPDN -SnlPLAECgYEA6RtgGqL34orxiqT3tDkA2Lb4aSq9Zntr09VAqwft4I3550XihHsa -lqJoy2macX36f60oRDZEL3v4cH22zyjojav4MXe2fLlCiApy45xCzsGkWizxTe5D -184jUIcRxb/sGbOulbXBdu8lmtNiyslvkAoj75bvL2MXhytbYgikhrMCgYEA2Lv1 -9Os+VXT9py/67dO/GY14NUpT1sFq2zxPYWpKxJD+j/NVZsflpPd/V5cGANZGovA0 -c8WNupJmCO1P20uldRX5dJ3EhhER2kn0yKhZuBBxmrELQZdnXGc9T8ub0xJVbo2u -K3Km3C1Dx7Us4BwzGOO6K2kYbT1ij/vAbZWmpCECgYBdkhqStqYwbrukfrHbyyH2 -3AN9G6XpdFOFNc3+mXE4OWV+G4Rgz5WNr+XG+T4AnqQmChjmwK3ALdA9P4lZQL+Q -1t4K5VYAXNFDEIarrPb4Tayucenu8VyUTO/KDF3q9i5M5t6Gw+3D0x1SN9YpNpCs -zhU8wGaErA8uuA23nWaFlwKBgQDXSf6MB5GnucqtZI/R5uCRNWIPLYISdZb7p/EC -R1912sHDpGdU7YREVkV8cFxaQH9yI0E3LyoWBo6sl28X2xDEOcvN91ncAuDFAWnS -WMimek8e5nbT9N2LgFH7Dbn/9NpAMySrq/vsAlqt6l5lUB3Bv5SSwpatDKj3dZA0 -Ss95gQKBgQCvwxW4PSd+hxdofwGd6KBwhA404GPdcVLLmASGVYRNcVNyfhUmMCn+ -UF1WsBCOcnWjJ/pT864S9Rp1bbqzpQS8Pz1vJwhkaHEJD0a5l+KhD4llsWDjUNfI -KfVffcmydNC3uRD59WPZEAgu+gOS8kRw7lybFTNuMS7B3dC5v9UtWg== ------END RSA PRIVATE KEY----- diff --git a/config/QtipKey.pub b/config/QtipKey.pub deleted file mode 100644 index 7a40f91c..00000000 --- a/config/QtipKey.pub +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDFWkAL5DwtPJ+0BMKBUcgq8ORdjMcj/wJl4qVVYeUdQDT/oS6pKuHNuk7BJpCLYLfHgSzO+Hv8J+5UyqXXXJKGxfAUPtpXrHRgburwhRJwcgtQVqWbQXHnY2jsikHeWWW5RX6SHxrLrxX04a8xmtD7BbDiJ6HZqVSem8XC15eZvw1C1dbqmbZqWBRWLFzhHf/9goucaSP3ekYWVCUPPlDS+D7r2Mc2Ch0Z7AivEJZ5mTamefS80B1ZNBRObvP/W5jPoAh5H9bHrqayZyEVGaYOk/toddJJAQHJIXu/8YSxnUnC0FbTtZ5s2JO/3Big5/IpW5m5NsHCOqVPjHkKkgkT root@foreman-jump.opnfv.com diff --git a/config/SampleHeat.yaml b/config/SampleHeat.yaml index ae9f5667..a42cdb79 100644 --- a/config/SampleHeat.yaml +++ b/config/SampleHeat.yaml @@ -1,36 +1,66 @@ -heat_template_version: 2014-10-16 -parameters: +heat_template_version: 2015-04-30 + +description: > + Used to run VMs for Qtip - private_net_name: +parameters: + image: type: string - default: 'private_network' - - availability_zone: + description: Name of the image + default: QTIP_CentOS + + external_net_name: type: string - description: The AvailZone. - default: compute1 - + description: Name of the external network which management network will connect to + default: admin_floating_net + resources: + flavor: + type: OS::Nova::Flavor + properties: + ram: 8192 + vcpus: 8 + disk: 80 - private_network: - type: OS::Neutron::Net - private_subnet: + network: + type: OS::Neutron::Net + properties: + name: qtip_net + + subnet: type: OS::Neutron::Subnet properties: - network_id: { get_resource: private_network } - cidr: '10.10.17.0/24' - dns_nameservers: [ '8.8.8.8' ] - gateway_ip: '10.10.17.1' - allocation_pools: [ {"start":'10.10.17.2', "end": '10.10.17.200'} ] - router_1: + name: qtip_subnet + ip_version: 4 + cidr: 192.168.0.0/24 + network: { get_resource: network } + dns_nameservers: [8.8.8.8] + + management_router: type: OS::Neutron::Router properties: + name: qtip_router external_gateway_info: - network: { get_param: public_network } - router_interface: + network: { get_param: external_net_name } + + management_router_interface: type: OS::Neutron::RouterInterface properties: - router_id: { get_resource: router_1 } - subnet: { get_resource: private_subnet } + router: { get_resource: management_router } + subnet: { get_resource: subnet } + + security_group: + type: OS::Neutron::SecurityGroup + properties: + name: qtip_security_group + rules: + - port_range_min: 22 + port_range_max: 5201 + protocol: tcp + - port_range_min: 22 + port_range_max: 5201 + protocol: udp + - protocol: icmp + outputs: description: 'none' diff --git a/docker/Dockerfile b/docker/Dockerfile index 204400cc..8d951fc5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -4,7 +4,7 @@ FROM ubuntu:14.04 -MAINTAINER Nauman Ahad <Nauman_Ahad@dell.com> +MAINTAINER Yujun Zhang <zhang.yujunz@zte.com.cn> LABEL version="0.1" description="OPNFV QTIP Docker container" diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000..35ac0935 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,11 @@ +# QTIP The Indices for Performance + +[QTIP] is an [OPNFV] project. + +It aims to build a platform for creating and sharing indices of [NFVI] performance. + +See the [project vision](https://wiki.opnfv.org/display/qtip/Vision) for more details. + +[QTIP]: https://wiki.opnfv.org/display/qtip +[OPNFV]: https://www.opnfv.org +[NFVI]: https://en.wikipedia.org/wiki/Network_function_virtualization diff --git a/docker/run_qtip.sh b/docker/run_qtip.sh index 62f97c88..a7a20501 100755 --- a/docker/run_qtip.sh +++ b/docker/run_qtip.sh @@ -1,4 +1,5 @@ #! /bin/bash + run_test_suite() { if [ "$TEST_CASE" == "compute" ]; then @@ -21,8 +22,16 @@ run_test_suite() fi } +rm -f ${QTIP_DIR}/config/QtipKey* + +echo "Generating ssh keypair" +ssh-keygen -t rsa -N "" -f ${QTIP_DIR}/config/QtipKey -q + source ${QTIP_DIR}/docker/prepare_qtip_image.sh run_test_suite source ${QTIP_DIR}/docker/cleanup_qtip_image.sh + +echo "Remove ssh keypair" +rm -f ${QTIP_DIR}/config/QtipKey* diff --git a/docs/configguide/configuration.rst b/docs/configguide/configuration.rst index e5228541..d6d2fd5d 100644 --- a/docs/configguide/configuration.rst +++ b/docs/configguide/configuration.rst @@ -7,39 +7,14 @@ Configuration ************* -QTIP currently supports by using a Docker image or by pulling the repo from -the upstream repository found at https://git.opnfv.org/qtip. Detailed steps -about setting up QTIP using both of these options can be found below. +QTIP currently supports by using a Docker image. Detailed steps +about setting up QTIP can be found below. To use QTIP you should have access to an OpenStack environment, with at least Nova, Neutron, Glance, Keystone and Heat installed. Add a brief introduction to configure OPNFV with this specific installer -Pre-configuration activities ----------------------------- - - -Setting QTIP framework on Ubuntu 14.04 -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Install dependencies: -:: - - sudo apt-get install python-dev - sudo apt-get install python-pip - sudo apt-get install build-essential - sudo apt-get install git wget - sudo pip install python-heatclient python-glanceclient python-neutronclient - - -Download source code and install python dependencies: -:: - - git clone https://git.opnfv.org/qtip - cd qtip - - Installing QTIP using Docker ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -47,18 +22,25 @@ QTIP has a Docker images on the docker hub. Pulling opnfv/qtip docker image from docker hub: :: - sudo docker pull opnfv/qtip + docker pull opnfv/qtip Verify that opnfv/qtip has been downloaded. It should be listed as an image by running the following command. :: - sudo docker images + docker images -Run the Docker instance: +Make dir to store the QTIP image which will be used to create vm in cloud. :: - docker run -it opnfv/qtip /bin/bash + mkdir $HOME/imgstore + +Run and enter the Docker instance: +:: + envs="INSTALLER_TYPE={INSTALLER_TYPE} -e INSTALLER_IP={INSTALLER_IP} +-e NODE_NAME={NODE_NAME}" + docker run --name qtip -id -e $envs -v "$HOME/imgstore:/home/opnfv/imgstore" opnfv/qtip + docker exec -i -t qtip /bin/bash Now you are in the container and QTIP can be found in the /repos/qtip and can be navigated to using the following command. @@ -79,22 +61,22 @@ from the OpenStack *openrc* file. This can be done by running the following command. :: - source get_env_info.sh -n {INSTALLER_TYPE} -i {INSTALLER_IP} + source scripts/get_env_info.sh -n {INSTALLER_TYPE} -i {INSTALLER_IP} source opnfv-creds.sh This provides a ``opnfv-creds.sh`` file which can be sources to get the -environment variables. For running QTIP manually, it is also necessary to -export the installer type. -:: - - export INSTALLER_TYPE="{installer-type}" +environment variables. QTIP default key pair """""""""""""""""""""" -QTIP uses a SSH key pair to connect to the guest image. This key pair can -be found in the ``config/`` directory. +QTIP uses a SSH key pair to connect to the guest image. You should generate key pair +before running QTIP test. And put key pair in the ``config/`` directory. +:: + + ssh-keygen -t rsa -N "" -f config/QtipKey -q + Hardware configuration diff --git a/docs/userguide/_01-compute.rst b/docs/userguide/_01-compute.rst index 7cd4c2ce..56be5488 100644 --- a/docs/userguide/_01-compute.rst +++ b/docs/userguide/_01-compute.rst @@ -58,7 +58,7 @@ OpenSSL Speed OpenSSL Speed can be used to benchmark compute performance of a machine. In QTIP, two OpenSSL Speed benchmarks are incorporated: 1. RSA signatunes/sec signed by a machine -2. AES 128-bit encnyption throught for a machine for cipher block sizes +2. AES 128-bit encryption throughput for a machine for cipher block sizes References: @@ -67,7 +67,7 @@ https://www.openssl.org/docs/manmaster/apps/speed.html RAMSpeed ^^^^^^^^ -RAMSpeed is used to measune a machine's memory perfomace. +RAMSpeed is used to measure a machine's memory perfomace. The problem(array)size is large enough to ensure Cache Misses so that the main machine memory is used. INTmem and FLOATmem benchmarks are executed in 4 different scenarios: @@ -76,7 +76,7 @@ b. Add: a(i)=b(i)+c(i) c. Scale: a(i)=b(i)*d d. Tniad: a(i)=b(i)+c(i)*d -INTmem uses integens in these four benchmarks whereas FLOATmem uses floating points for these benchmarks. +INTmem uses integers in these four benchmarks whereas FLOATmem uses floating points for these benchmarks. References: diff --git a/docs/userguide/annex.rst b/docs/userguide/annex.rst index ce1775d6..e8bf5555 100644 --- a/docs/userguide/annex.rst +++ b/docs/userguide/annex.rst @@ -15,4 +15,4 @@ Annex Downloads ========= -- :download:`Sample configuration <../download/sample_config.yaml>`
\ No newline at end of file +- :download:`Sample configuration <../download/sample_config.yaml>` diff --git a/docs/userguide/introduction.rst b/docs/userguide/introduction.rst index 7a202a45..4876d0e2 100644 --- a/docs/userguide/introduction.rst +++ b/docs/userguide/introduction.rst @@ -9,16 +9,16 @@ Introduction ************ This guide will serve as a first step to familiarize the user with how to -run QTIP the first time when the user clones QTIP on to their host machine. -In order to clone QTIP please follow the instructions in the -installation.rst located in docs/userguide/installation.rst. +run QTIP the first time when the user pull QTIP image on to their host machine. +In order to install and config QTIP please follow the instructions in the +configuration.rst located in docs/configguide/configuration.rst. QTIP Directory structure: ------------------------- The QTIP directory has been sectioned off into multiple folders to facilitate segmenting information into relevant categories. The folders that concern - the end user are `test_cases/` and `test_list/`. + the end user are `test_cases/` and `benchmarks/suite/`. **test_cases/:** @@ -58,7 +58,7 @@ distinguishes between a test to be run on the Virtual Machine or the compute node itself, respectively. -**test_list/:** +**benchmarks/suite/:** This folder contains three files, namely `compute`, `network` and `storage`. These files list the benchmarks are to be run by the QTIP framework. Sample @@ -132,11 +132,13 @@ The `Context` tag helps the user list the number of compute nodes they want to run dhrystone on. The user can list all the compute nodes under the `Host_Machines` tag. All the machines under test must be listed under the `Host_Machines` and naming it incrementally higher. The `ip:` tag is used - to specify the IP of the particular compute node. The `pw:` tag can be left - blank because QTIP uses its own key for ssh. In order to run dhrystone on - one compute node at a time the user needs to edit the `role:` tag. `role: - host` for machine_1 and `role: server` for machine_2 will allow for - dhrystone to be run on machine_1 and then run on machine_2. + to specify the IP of the particular compute node.The `ip:` tag can be left + blank when installer type is 'fuel',because QTIP will get ip + from installer. The `pw:` tag can be left blank because QTIP uses its own + key for ssh. In order to run dhrystone on one compute node at a time the user + needs to edit the `role:` tag. `role: host` for machine_1 and `role: server` + for machine_2 will allow for dhrystone to be run on machine_1 and then run + on machine_2. :: @@ -312,71 +314,67 @@ Sample dhrystone_vm.yaml file: Commands to run the Framework: ------------------------------ -In order to start QTIP on the default lab please use the following commands (asssuming you have prepared the config files in the test_cases/default/ directory and listed the intended suite in the test_list/<RELEVANT-SUITE-FILE>): +In order to start QTIP on the default lab please use the following commands (asssuming your installer +is 'fuel' or 'compass', you use the config files in the test_cases/default/ directory and listed the +intended suite in the benchmarks/suite/<RELEVANT-SUITE-FILE>): -First step is to export the necessary information to the environment. -:: - - source get_env_info.sh -n <INSTALLER_TYPE> -i <INSTALLER_IP> - -for running qtip on an openstack deployed using FUEL with the Installer IP 10.20.0.2 -:: - - source get_env_info.sh -n fuel -i 10.20.0.2 +First step is to export the necessary information to the environment and generate QTIP key pair. +Please follow the instructions in the configuration.rst. -This will generate the `opnfv-creds.sh` file needed to use the python clients for keystone, glance, nova, and neutron. +Secondary step download the QTIP image and upload it to the Cloud.QTIP will use this image +to create VM when test VM performance. :: - source opnfv-creds.sh + source docker/prepare_qtip_image.sh -Running QTIP on the using `default` as the pod name and for the `compute` suite by cli +Running QTIP on the using `default` as the pod name and for the `compute` suite by cli. :: python qtip.py -l default -f compute -Running QTIP on the using 'default' as the pod name and for the 'compute' suite 'bm' type by restful api +Running QTIP on the using 'default' as the pod name and for the 'compute' suite 'bm' type by restful api. :: - curl --trace-ascii debug.txt -X POST -d '{ "installer_ip": "10.20.6.2","installer_type":"fuel", "suite_name":"compute", "type": "BM"}' -H "Content-Type: application/json" http://qtip_server_ip:5000/api/v1.0/jobs + curl --trace-ascii debug.txt -X POST -d '{ "installer_ip": "10.20.6.2","installer_type":"fuel", "suite_name":"compute", "type": "BM"}' -H "Content-Type: application/json" http://127.0.0.1:5000/api/v1.0/jobs -Running QTIP on the using 'default' as the pod name and for the 'compute' suite 'vm' type by restful api +Running QTIP on the using 'default' as the pod name and for the 'compute' suite 'vm' type by restful api. :: - curl --trace-ascii debug.txt -X POST -d '{ "installer_ip": "10.20.6.2","installer_type":"fuel", "suite_name":"compute", "type": "VM"}' -H "Content-Type: application/json" http://qtip_server_ip:5000/api/v1.0/jobs + curl --trace-ascii debug.txt -X POST -d '{ "installer_ip": "10.20.6.2","installer_type":"fuel", "suite_name":"compute", "type": "VM"}' -H "Content-Type: application/json" http://127.0.0.1:5000/api/v1.0/jobs -Running QTIP on the using `default` as the pod name and for the `network` suite by cli +Running QTIP on the using `default` as the pod name and for the `network` suite by cli. :: python qtip.py -l default -f network -Running QTIP on the using 'default' as the pod name and for the 'network' suite 'bm' type by restful api +Running QTIP on the using 'default' as the pod name and for the 'network' suite 'bm' type by restful api. :: - curl --trace-ascii debug.txt -X POST -d '{ "installer_ip": "10.20.6.2","installer_type":"fuel", "suite_name":"network", "type": "BM"}' -H "Content-Type: application/json" http://qtip_server_ip:5000/api/v1.0/jobs + curl --trace-ascii debug.txt -X POST -d '{ "installer_ip": "10.20.6.2","installer_type":"fuel", "suite_name":"network", "type": "BM"}' -H "Content-Type: application/json" http://127.0.0.1:5000/api/v1.0/jobs -Running QTIP on the using `default` as the pod name and for the `storage` suite by cli +Running QTIP on the using `default` as the pod name and for the `storage` suite by cli. :: python qtip.py -l default -f network -Running QTIP on the using 'default' as the pod name and for the 'storage' suite 'bm' type by restful api +Running QTIP on the using 'default' as the pod name and for the 'storage' suite 'bm' type by restful api. :: - curl --trace-ascii debug.txt -X POST -d '{ "installer_ip": "10.20.6.2","installer_type":"fuel", "suite_name":"storage", "type": "BM"}' -H "Content-Type: application/json" http://qtip_server_ip:5000/api/v1.0/jobs + curl --trace-ascii debug.txt -X POST -d '{ "installer_ip": "10.20.6.2","installer_type":"fuel", "suite_name":"storage", "type": "BM"}' -H "Content-Type: application/json" http://127.0.0.1:5000/api/v1.0/jobs Get running QTIP job status by restful api :: - curl --trace-ascii debug.txt -X GET http://qtip_server_ip:5000/api/v1.0/jobs/job-id + curl --trace-ascii debug.txt -X GET http://127.0.0.1:5000/api/v1.0/jobs/job-id For example: - curl --trace-ascii debug.txt -X GET http://172.37.0.3:5000/api/v1.0/jobs/5b71f035-3fd6-425c-9cc7-86acd3a04214 + curl --trace-ascii debug.txt -X GET http://127.0.0.1:5000/api/v1.0/jobs/5b71f035-3fd6-425c-9cc7-86acd3a04214 Stop running QTIP job by restful api.The job will finish the current benchmark test and stop. :: - curl --trace-ascii debug.txt -X DELTET http://qtip_server_ip:5000/api/v1.0/jobs/job-id + curl --trace-ascii debug.txt -X DELTET http://127.0.0.1:5000/api/v1.0/jobs/job-id For example: - curl --trace-ascii debug.txt -X DELETE http://172.37.0.3:5000/api/v1.0/jobs/5b71f035-3fd6-425c-9cc7-86acd3a04214q + curl --trace-ascii debug.txt -X DELETE http://127.0.0.1:5000/api/v1.0/jobs/5b71f035-3fd6-425c-9cc7-86acd3a04214q Results: -------- diff --git a/func/args_handler.py b/func/args_handler.py index 2f7c8957..624f90c4 100644 --- a/func/args_handler.py +++ b/func/args_handler.py @@ -59,10 +59,12 @@ def prepare_ansible_env(benchmark_test_case): def run_benchmark(installer_type, pwd, benchmark, benchmark_details, proxy_info, env_setup, benchmark_test_case): driver = Driver() - return driver.drive_bench(installer_type, pwd, benchmark, - env_setup.roles_dict.items(), - _get_f_name(benchmark_test_case), - benchmark_details, env_setup.ip_pw_dict.items(), proxy_info) + result = driver.drive_bench(installer_type, pwd, benchmark, + env_setup.roles_dict.items(), + _get_f_name(benchmark_test_case), + benchmark_details, env_setup.ip_pw_dict.items(), proxy_info) + env_setup.cleanup_authorized_keys() + return result def prepare_and_run_benchmark(installer_type, pwd, benchmark_test_case): diff --git a/func/env_setup.py b/func/env_setup.py index 9e21a5b6..6027f904 100644 --- a/func/env_setup.py +++ b/func/env_setup.py @@ -208,3 +208,9 @@ class Env_setup: def call_ssh_test(self): self.ssh_test(self.ip_pw_list) + + def cleanup_authorized_keys(self): + for ip, pw in self.ip_pw_list: + cmd = './scripts/cleanup_creds.sh %s' % ip + logger.info("cleanup authorized_keys: %s " % cmd) + os.system(cmd) diff --git a/func/spawn_vm.py b/func/spawn_vm.py index 3a16e02d..0a24d7a4 100644 --- a/func/spawn_vm.py +++ b/func/spawn_vm.py @@ -1,169 +1,138 @@ ##############################################################################
-# Copyright (c) 2015 Dell Inc and others.
+# Copyright (c) 2016 Dell Inc, ZTE and others.
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-
import os
import sys
-from collections import defaultdict
-from func.env_setup import Env_setup
import yaml
import heatclient.client
import keystoneclient
-from novaclient import client
import time
+from func.env_setup import Env_setup
from func.create_zones import AvailabilityZone
+from utils import logger_utils
+
+logger = logger_utils.QtipLogger('spawn_vm').get
class SpawnVM(Env_setup):
- vm_role_ip_dict = defaultdict(list)
- installer = ''
def __init__(self, vm_info):
- print 'SpawnVM Class initiated'
- print 'vm_info: %s' % vm_info
+ logger.info('vm_info: %s' % vm_info)
vm_role_ip_dict = vm_info.copy()
- print 'Generating Heat Template\n'
self._keystone_client = None
self._heat_client = None
self._glance_client = None
self._nova_client = None
- self. _get_nova_client()
self.azone = AvailabilityZone()
# TODO: it should clean up aggregates and stack after test case finished.
self.azone.clean_all_aggregates()
self.azone.create_aggs(vm_info['availability_zone'])
- installer = self.get_installer_type()
- self.Heat_template1 = self.heat_template_vm(vm_info, installer)
- self.create_stack(vm_role_ip_dict, self.Heat_template1)
+ self.heat_template = self.generate_heat_template(vm_info)
+ self.create_stack(vm_role_ip_dict)
@staticmethod
- def get_installer_type():
- print 'Getting Installer Name'
- return os.environ['INSTALLER_TYPE']
-
- @staticmethod
- def get_public_network(installer_detected):
+ def get_public_network():
"""
TODO: GET THE NAMES OF THE PUBLIC NETWORKS for OTHER PROJECTS
"""
- print 'Getting Public Network'
- if installer_detected.lower() == 'fuel':
+ installer = os.environ['INSTALLER_TYPE']
+
+ if installer.lower() == 'fuel':
return 'admin_floating_net'
- if installer_detected.lower() == 'apex':
+ if installer.lower() == 'apex':
return 'external'
- if installer_detected.lower() == 'compass':
+ if installer.lower() == 'compass':
return 'ext-net'
- if installer_detected.lower() == 'joid':
+ if installer.lower() == 'joid':
return 'ext-net'
- def heat_template_vm(self, vm_params, installer):
- Heat_Dic = {}
+ def generate_heat_template(self, vm_params):
+ logger.info('Generating Heat Template')
+ heat_dict = {}
try:
with open('./config/SampleHeat.yaml', 'r+') as H_temp:
- Heat_Dic = yaml.safe_load(H_temp)
+ heat_dict = yaml.safe_load(H_temp)
except yaml.YAMLError as exc:
if hasattr(exc, 'problem_mark'):
mark = exc.problem_mark
- print 'Error in qtip/config/SampleHeat.yaml at: (%s,%s)' % (mark.line + 1, mark.column + 1)
- print 'EXITING PROGRAM. Correct File and restart'
+ logger.error(
+ 'Error in qtip/config/SampleHeat.yaml at: (%s,%s)' % (mark.line + 1,
+ mark.column + 1))
+ logger.error('EXITING PROGRAM. Correct File and restart')
sys.exit(1)
+
fopen = open('./config/QtipKey.pub', 'r')
fopenstr = fopen.read()
fopenstr = fopenstr.rstrip()
scriptcmd = '#!/bin/bash \n echo {0} >> foo.txt \n echo {1} >> /root/.ssh/authorized_keys'.format(
fopenstr, fopenstr)
- netName = self.get_public_network(installer)
- print netName
- Heat_Dic['heat_template_version'] = '2014-10-16'
- Heat_Dic['resources']['KeyPairSavePrivate'] = {
- 'type': 'OS::Nova::KeyPair',
- 'properties': {
- 'save_private_key': 'true',
- 'name': 'my_key'
- }
- }
- Heat_Dic['parameters']['public_network'] = {
+ netName = self.get_public_network()
+ heat_dict['heat_template_version'] = '2015-04-30'
+
+ heat_dict['parameters']['public_network'] = {
'type': 'string',
'default': netName
}
+
for x in range(1, len(vm_params['availability_zone']) + 1):
avail_zone = vm_params['availability_zone'][x - 1]
- img = vm_params['OS_image'][x - 1]
- flavor = vm_params['flavor'][x - 1]
- Heat_Dic['parameters']['availability_zone_' + str(x)] = \
+ heat_dict['parameters']['availability_zone_' + str(x)] = \
{'description': 'Availability Zone of the instance',
'default': avail_zone,
'type': 'string'}
- Heat_Dic['resources']['public_port_' + str(x)] = \
+ heat_dict['resources']['public_port_' + str(x)] = \
{'type': 'OS::Neutron::Port',
- 'properties': {'network': {'get_resource': 'private_network'},
- 'security_groups': [{'get_resource': 'demo1_security_Group'}],
- 'fixed_ips': [{'subnet_id':
- {'get_resource': 'private_subnet'}}]}}
+ 'properties': {'network': {'get_resource': 'network'},
+ 'security_groups': [{'get_resource': 'security_group'}],
+ 'fixed_ips': [{'subnet_id': {'get_resource': 'subnet'}}]}}
- Heat_Dic['resources']['floating_ip_' + str(x)] = {
+ heat_dict['resources']['floating_ip_' + str(x)] = {
'type': 'OS::Neutron::FloatingIP',
- 'properties': {
- 'floating_network': {'get_param': 'public_network'}}}
+ 'properties': {'floating_network': {'get_param': 'external_net_name'}}}
- Heat_Dic['resources']['floating_ip_assoc_' + str(x)] = {
+ heat_dict['resources']['floating_ip_assoc_' + str(x)] = {
'type': 'OS::Neutron::FloatingIPAssociation',
'properties': {
'floatingip_id': {'get_resource': 'floating_ip_' + str(x)},
'port_id': {'get_resource': 'public_port_' + str(x)}}}
- Heat_Dic['resources']['my_instance_' + str(x)] = \
+ heat_dict['resources']['my_instance_' + str(x)] = \
{'type': 'OS::Nova::Server',
- 'properties': {'image': img,
+ 'properties': {'image': {'get_param': 'image'},
'networks':
[{'port': {'get_resource': 'public_port_' + str(x)}}],
- 'flavor': flavor,
+ 'flavor': {'get_resource': 'flavor'},
'availability_zone': avail_zone,
+ 'security_groups': [{'get_resource': 'security_group'}],
'name': 'instance' + str(x),
- 'key_name': {'get_resource': 'KeyPairSavePrivate'},
'user_data_format': 'RAW',
'user_data': scriptcmd}}
- Heat_Dic['resources']['demo1_security_Group'] = {
- 'type': 'OS::Neutron::SecurityGroup',
- 'properties': {
- 'name': 'demo1_security_Group',
- 'rules': [{
- 'protocol': 'tcp',
- 'port_range_min': 22,
- 'port_range_max': 5201},
- {'protocol': 'udp',
- 'port_range_min': 22,
- 'port_range_max': 5201},
- {'protocol': 'icmp'}]}}
-
- Heat_Dic['outputs']['instance_PIP_' + str(x)] = {
+ heat_dict['outputs']['instance_PIP_' + str(x)] = {
'description': 'IP address of the instance',
'value': {'get_attr': ['my_instance_' + str(x), 'first_address']}}
- Heat_Dic['outputs']['instance_ip_' + str(x)] = {
+
+ heat_dict['outputs']['instance_ip_' + str(x)] = {
'description': 'IP address of the instance',
'value': {'get_attr': ['floating_ip_' + str(x), 'floating_ip_address']}}
- Heat_Dic['outputs']['availability_instance_' + str(x)] = {
+ heat_dict['outputs']['availability_instance_' + str(x)] = {
'description': 'Availability Zone of the Instance',
'value': {'get_param': 'availability_zone_' + str(x)}}
- Heat_Dic['outputs']['KeyPair_PublicKey'] = {
- 'description': 'Private Key',
- 'value': {'get_attr': ['KeyPairSavePrivate', 'private_key']}
- }
- del Heat_Dic['outputs']['description']
- print Heat_Dic
- return Heat_Dic
+ del heat_dict['outputs']['description']
+ logger.info(heat_dict)
+
+ return heat_dict
def _get_keystone_client(self):
"""returns a keystone client instance"""
@@ -176,12 +145,6 @@ class SpawnVM(Env_setup): tenant_name=os.environ.get('OS_TENANT_NAME'))
return self._keystone_client
- def _get_nova_client(self):
- if self._nova_client is None:
- keystone = self._get_keystone_client()
- self._nova_client = client.Client('2', token=keystone.auth_token)
- return self._nova_client
-
def _get_heat_client(self):
"""returns a heat client instance"""
if self._heat_client is None:
@@ -192,45 +155,29 @@ class SpawnVM(Env_setup): '1', endpoint=heat_endpoint, token=keystone.auth_token)
return self._heat_client
- def create_stack(self, vm_role_ip_dict, heat_template):
-
- global sshkey
+ def create_stack(self, vm_role_ip_dict):
stackname = 'QTIP'
heat = self._get_heat_client()
- for checks in range(3):
- print "Try to delete heats %s" % checks
- for prev_stacks in heat.stacks.list():
- if prev_stacks.stack_name == 'QTIP':
- print 'QTIP Stacks exists.\nDeleting Existing Stack'
- heat.stacks.delete('QTIP')
- time.sleep(10)
- print '\nStack Creating Started\n'
+ self.delete_stack(stackname)
- try:
- heat.stacks.create(stack_name=stackname, template=heat_template)
- except Exception:
- print 'Create Failed :( '
-
- cluster_detail = heat.stacks.get(stackname)
- while cluster_detail.status != 'COMPLETE':
- if cluster_detail.status == 'IN_PROGRESS':
- print 'Stack Creation in Progress'
- cluster_detail = heat.stacks.get(stackname)
- time.sleep(10)
- print 'Stack Created'
- print 'Getting Public IP(s)'
- zone = []
- s = 0
- for vm in range(len(vm_role_ip_dict['OS_image'])):
+ logger.info('Start to create stack %s' % stackname)
+ heat.stacks.create(stack_name=stackname, template=self.heat_template)
+
+ stack_status = "IN_PROGRESS"
+ while stack_status != 'COMPLETE':
+ if stack_status == 'IN_PROGRESS':
+ logger.debug('Create in Progress')
+ if stack_status == 'CREATE_FAILED':
+ raise RuntimeError("Stack %s created failed!" % stackname)
+ stack_status = heat.stacks.get(stackname).status
+ time.sleep(15)
+ logger.info('Stack %s Created Complete!' % stackname)
- for I in cluster_detail.outputs:
- availabilityKey = 'availability_instance_' + str(vm + 1)
+ stack_outputs = heat.stacks.get(stackname).outputs
- if I['output_key'] == availabilityKey:
- zone.insert(s, str(I['output_value']))
- s = s + 1
- for i in cluster_detail.outputs:
+ for vm in range(len(vm_role_ip_dict['OS_image'])):
+ for i in stack_outputs:
instanceKey = "instance_ip_" + str(vm + 1)
privateIPkey = 'instance_PIP_' + str(vm + 1)
if i['output_key'] == instanceKey:
@@ -240,10 +187,20 @@ class SpawnVM(Env_setup): if i['output_key'] == privateIPkey:
Env_setup.ip_pw_dict[vm_role_ip_dict['role'][vm]] = str(i['output_value'])
- if i['output_key'] == 'KeyPair_PublicKey':
- sshkey = str(i['output_value'])
- with open('./config/my_key.pem', 'w') as fopen:
- fopen.write(sshkey)
- fopen.close()
- print Env_setup.ip_pw_list
+ logger.info('Getting Public IP(s): %s' % Env_setup.ip_pw_list)
+
+ def delete_stack(self, stack_name):
+ heat = self._get_heat_client()
+
+ stacks = heat.stacks.list()
+ exists = map(lambda x: x.stack_name, stacks)
+ if stack_name in exists:
+ logger.info("Delete stack %s" % stack_name)
+ heat.stacks.delete(stack_name)
+ while stack_name in exists:
+ time.sleep(10)
+ stacks = heat.stacks.list()
+ exists = map(lambda x: x.stack_name, stacks)
+ logger.debug("exists_stacks: %s" % exists)
+ logger.info("%s doesn't exist" % stack_name)
@@ -7,11 +7,9 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## from func.cli import Cli -import os def main(): - os.system('./scripts/file_permission.sh') Cli() diff --git a/scripts/cleanup_creds.sh b/scripts/cleanup_creds.sh new file mode 100644 index 00000000..9bf44305 --- /dev/null +++ b/scripts/cleanup_creds.sh @@ -0,0 +1,14 @@ +#! /bin/bash + +DEST_IP=$1 +HOSTNAME=$(hostname) +sshoptions="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" + +case "$INSTALLER_TYPE" in + fuel) + ssh $sshoptions -i ./config/QtipKey root@$DEST_IP "sed -i '/root@$HOSTNAME/d' /root/.ssh/authorized_keys" + ;; +esac + + + diff --git a/scripts/file_permission.sh b/scripts/file_permission.sh deleted file mode 100755 index a8af957e..00000000 --- a/scripts/file_permission.sh +++ /dev/null @@ -1,3 +0,0 @@ -#! /bin/bash -chmod 0600 config/QtipKey -chmod 0600 config/QtipKey.pub diff --git a/tests/spawn_vm_test.py b/tests/spawn_vm_test.py index 7890abd1..fca7dd00 100644 --- a/tests/spawn_vm_test.py +++ b/tests/spawn_vm_test.py @@ -17,9 +17,7 @@ class StackMock(MagicMock): {'output_key': 'instance_ip_1', "output_value": "172.10.0.154"}, {"output_key": "instance_PIP_1", - "output_value": "10.10.17.5"}, - {'output_key': 'KeyPair_PublicKey', - "output_value": "-----BEGIN RSA PRIVATE KEY-----\nMIIEpwIBAAKCAQEAqCiHcrLBXtxG0LhnKndU7VIVpYxORmv0d4tvujkWOkYuagiW\nU/MTRk0zhRvFQDVPEs0Jrj/BIecqm6fjjT6dZ/H7JLYGaqJitRkoupKgBsMSIqUz\nrR0ekOlfXZ6N+Ud8k6s+qjc7BO4b1ezz78jHisC5o0GCkUV0ECx64Re1fO+oKs1c\nfL9aaexahJUYN3J48pazQz+imc2x/G9nuqHX3cqEszmxnT4jwv//In1GjHy2AyXw\n1oA5F6wZoQCSrXc2BditU+1tlVhEkPFt5JgiHUpY8T8mYbroT7JH6xjcGSKUN+HG\nN8PXNUTD1VAQfwHpkfsGMfDyzjytCXsoTEOqnwIDAQABAoIBAAEL/4vfQQTuKiKy\ngzHofEbd8/SL4xDdKzBzVca7BEBon3FZjFYJdV1CrcduXNQBgPSFAkJrczBa2BEQ\nAoKmmSREhWO9Hl0blbG67l36+7QPEtXUYXX6cG5Ghal3izq6DzR8JG+62Es3kETM\nrNgZT+S1PnKdvcpZvFc9b6ZnF2InuTbrmNVBZKrhdWOJ5tCwRGKKUl6BHoJH3yu0\nT5hUW277e1LYHx+hZtoZ98ToC+LGe6/M8a8y6VLYpcQlX2AtVXeGDalomunF+p3f\nuY6din6s4lq1gSJz03PTpUbwiuhYCTe8Xkseu74Y+XYYJXPHopFju0Ewd6p0Db9Q\nJzzxCoECggCBAM2ox9zyrDc/Vlc0bb9SciFGUd/nEJF89+UHy98bAkpo22zNZIDg\nfacSgkg/6faZD+KrOU0I5W7m2B5t6w2fNHHik6NYGSLQ1JhgbXELGV7X/qECDL02\nctPaf+8o+dYoZja2LdJNASq2nmEmPI3LSHhzAt4dWY4W+geXiHt4iWVHAoIAgQDR\nUdN09xv4U+stWqNcSfgjtx6boEUE8Ky7pyj+LrZKG0L61Jy9cSDP0x0rCtkW9vVR\n6RjidWM/DHQ5cl6aq+7pPy20/OqtqttFYT4R+C3AoAnRSaNzPD9a80C2gjv7WEz0\nPPFstWkI1gsN71KKRx7e6NIa9CNn5x9iE+SGfjgb6QKCAIBXylzG7LCnRNpOj4rp\nyP//RE1fDvv7nyUTF6jnrFfl+6zvXR4yBaKd10DWJrJxGhW15PGo+Ms39EL9el6E\nihmRI+9yIwFX411dToxpXRuPaRTBFmbpvnx2Ayfpp8w+pzA62rnktApzeVFSl0fy\nH3zoLfBjcJPyG8zPwNf6HRJJsQKCAIAE2S5asTaWo+r4m/bYtmXm/eDZnfa7TI/T\nsOWELbTPNp5wjOgsgyhNaAhu7MtmesXn5cxLwohP94vhoMKMNptMD8iRPqJ471Iw\n4zW62NLGeW6AyIHes3CMPMIs+AtHoR33MkotSG5sY/jRk8+HoGoYo6/qK+l+CJ5z\neR579wR5sQKCAIAvPWq+bvcPTDKUU1Fe/Y/GyWoUA+uSqmCdORBkK38lALFGphxj\nfDz9dXskimqW+A9hOPOS8dm8YcVvi/TLXVE5Vsx9VkOg6z6AZBQpgNXGfOgpju4W\nbjER7bQaASatuWQyCxbA9oNlAUdSeOhGTxeFLkLj7hNMd6tLjfd8w7A/hA==\n-----END RSA PRIVATE KEY-----\n"}] + "output_value": "10.10.17.5"}] class HeatMock(MagicMock): @@ -44,16 +42,15 @@ class TestClass: ]) @mock.patch('func.spawn_vm.Env_setup') @mock.patch('func.spawn_vm.AvailabilityZone') - @mock.patch('func.spawn_vm.client', autospec=True) @mock.patch('func.spawn_vm.keystoneclient.v2_0', autospec=True) @mock.patch('func.spawn_vm.heatclient.client', autospec=True) def test_create_zones_success(self, mock_heat, mock_keystone, - mock_nova_client, mock_zone, - mock_setup, test_input, expected): - mock_nova_client.Client.return_value = Mock() + mock_zone, mock_setup, test_input, expected): + open('./config/QtipKey.pub', 'a').close() mock_heat.Client.return_value = Mock(stacks=HeatMock()) k = mock.patch.dict(os.environ, {'INSTALLER_TYPE': 'fuel'}) k.start() SpawnVM(test_input) k.stop() + os.remove('./config/QtipKey.pub') mock_setup.ip_pw_list.append.assert_called_with(expected[0]) |