diff options
| -rw-r--r-- | INFO | 4 | ||||
| -rw-r--r-- | docs/apidocs/.keep | 0 | ||||
| -rw-r--r-- | docs/configguide/configuration.rst (renamed from docs/user_guides/02-installation.rst) | 153 | ||||
| -rw-r--r-- | docs/configguide/index.rst | 13 | ||||
| -rw-r--r-- | docs/roadmap/index.rst | 13 | ||||
| -rw-r--r-- | docs/roadmap/roadmap.rst (renamed from docs/roadmap.rst) | 0 | ||||
| -rw-r--r-- | docs/templates/index.rst | 35 | ||||
| -rw-r--r-- | docs/templates/sample_config.yaml (renamed from docs/sample_config.yaml) | 0 | ||||
| -rw-r--r-- | docs/user_guides/01-introduction.rst | 40 | ||||
| -rw-r--r-- | docs/user_guides/index.rst | 17 | ||||
| -rw-r--r-- | docs/userguide/01-compute.rst (renamed from docs/user_guides/test_cases/01-compute_testcases.rst) | 17 | ||||
| -rw-r--r-- | docs/userguide/02-network.rst (renamed from docs/user_guides/test_cases/02-network_testcases.rst) | 19 | ||||
| -rw-r--r-- | docs/userguide/03-storage.rst (renamed from docs/user_guides/test_cases/03-storage_testcases.rst) | 18 | ||||
| -rw-r--r-- | docs/userguide/index.rst | 21 | ||||
| -rw-r--r-- | docs/userguide/introduction.rst (renamed from docs/user_guides/03-usage-guide.rst) | 19 | ||||
| -rw-r--r-- | docs/userguide/overview.rst | 22 | ||||
| -rw-r--r-- | func/env_setup.py | 59 | ||||
| -rwxr-xr-x | func/fetch_compute_ips.sh | 117 | ||||
| -rw-r--r-- | requirements.txt | 3 | ||||
| -rw-r--r-- | restful_server/__init__.py | 0 | ||||
| -rw-r--r-- | restful_server/db.py | 48 | ||||
| -rw-r--r-- | restful_server/qtip_server.py | 138 | ||||
| -rw-r--r-- | tests/cli_test.py | 4 | ||||
| -rw-r--r-- | tests/env_setup_test.py | 49 | ||||
| -rw-r--r-- | tests/qtip_server_test.py | 78 |
25 files changed, 677 insertions, 210 deletions
@@ -17,7 +17,6 @@ Nauman Ahad nauman.ahad@xflowresearch.com Prabu Kuppuswamy prabu.kuppuswamy@spirent.com Prakash Ramchandran prakash.ramchandran@huawei.com Trevor Cooper trevor.cooper@intel.com -Vikram Dham vikram@kamboi.com Wenjing Chu chu.wenjing@gmail.com Yujun Zhang zhang.yujunz@zte.com.cn @@ -25,6 +24,7 @@ Link to TSC approval of the project: - http://meetbot.opnfv.org/meetings/opnfv-meeting/2015/opnfv-meeting.2015-01-27-14.59.html -Link(s) to approval of additional committers: +Link(s) to approval of committers update: - http://lists.opnfv.org/pipermail/opnfv-tsc/2016-July/002745.html +- http://lists.opnfv.org/pipermail/opnfv-tsc/2016-July/002755.html diff --git a/docs/apidocs/.keep b/docs/apidocs/.keep new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/docs/apidocs/.keep diff --git a/docs/user_guides/02-installation.rst b/docs/configguide/configuration.rst index 2f2ecf96..f29947d5 100644 --- a/docs/user_guides/02-installation.rst +++ b/docs/configguide/configuration.rst @@ -1,35 +1,28 @@ -Installation -============ - .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. (c) <optionally add copywriters name> -.. two dots create a comment. please leave this logo at the top of each of your rst files. -.. image:: ../etc/opnfv-logo.png - :height: 40 - :width: 200 - :alt: OPNFV - :align: left -.. these two pipes are to seperate the logo from the first title +.. (c) 2015 Dell Inc. +.. (c) 2016 ZTE Corp. + -| +Qtip configuration +================== -QTIP currently supports by using a Docker image or by pulling the repo from the upstream -repository found at https://git.opnfv.org/qtip. Detailed steps about setting up QTIP using both of these options -can be found below. +QTIP currently supports by using a Docker image or by pulling the repo from +the upstream repository found at https://git.opnfv.org/qtip. Detailed steps +about setting up QTIP using both of these options can be found below. -To use QTIP you should have access to an OpenStack environment, -with at least Nova, Neutron, Glance, Keystone and Heat installed. +To use QTIP you should have access to an OpenStack environment, with at least +Nova, Neutron, Glance, Keystone and Heat installed. Add a brief introduction +to configure OPNFV with this specific installer -The steps needed to run QTIP are: -Running QTIP by pulling the upstream code ------------------------------------------ +Pre-configuration activities +---------------------------- -.. _install-dependencies: Setting QTIP framework on Ubuntu 14.04 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + Install dependencies: :: @@ -46,96 +39,116 @@ Download source code and install python dependencies: git clone https://git.opnfv.org/qtip cd qtip + Installing QTIP using Docker ---------------------------------- +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -QTIP has a Docker images on the docker hub which can be pull after docker has been installed. +QTIP has a Docker images on the docker hub. Pulling opnfv/qtip docker image +from docker hub: +:: -Installing Docker -^^^^^^^^^^^^^^^^^ -The first step is to install docker: + sudo docker pull opnfv/qtip + +Verify that opnfv/qtip has been downloaded. It should be listed as an image by +running the following command. :: - sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D + sudo docker images + +Run the Docker instance: +:: + docker run opnfv/qtip -i -t bash -Add an entry for your Ubuntu operating system +Now you are in the container and QTIP can be found in the /repos/qtip and can +be navigated to using the following command. :: - Open the /etc/apt/sources.list.d/docker.list file in your favorite editor. + cd repos/qtip -If the file doesn’t exist, create it. -Remove any existing entries. +OpenStack parameters and credentials +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Add an entry for your Ubuntu operating system. -On Ubuntu Trusty 14.04 (LTS) +Environment variables +""""""""""""""""""""" + +Before running QTIP it is necessary to export OpenStack environment variables +from the OpenStack *openrc* file. This can be done by running the following +command. :: - deb https://apt.dockerproject.org/repo ubuntu-trusty main + source get_env_info.sh -n {INSTALLER_TYPE} -i {INSTALLER_IP} + source opnfv-creds.sh -Update the package manager +This provides a ``opnfv-creds.sh`` file which can be sources to get the +environment variables. For running QTIP manually, it is also necessary to +export the installer type. :: - sudo apt-get update + export INSTALLER_TYPE="{installer-type}" -Install Docker: -:: - sudo apt-get install docker-engine +QTIP default key pair +"""""""""""""""""""""" -Starting Docker Daemon: -:: +QTIP uses a SSH key pair to connect to the guest image. This key pair can +be found in the ``data/`` directory. - sudo service docker start -Pulling opnfv/qtip docker image from docker hub: -:: +Hardware configuration +---------------------- - sudo docker pull opnfv/qtip +Qtip does not have specific hardware requriements, and it can runs over any +OPNFV installer. -Verify that opnfv/qtip has been downloaded. It should be listed as an image by running the following command. -:: - sudo docker images +Jumphost configuration +---------------------- -Run the Docker instance: +Installer Docker on Jumphost, which is used for running Qtip image. + +The first step is to install docker: :: - docker run opnfv/qtip -i -t bash + sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 + --recv-keys 58118E89F3A912897C070ADBF76221572C52609D -Now you are in the container and QTIP can be found in the /repos/qtip and can be navigated to using the following command. +Add an entry for your Ubuntu operating system: :: - cd repos/qtip + Open the /etc/apt/sources.list.d/docker.list file in your favorite editor. +If the file doesn’t exist, create it. -OpenStack parameters and credentials ------------------------------------- +Remove any existing entries. +Add an entry for your Ubuntu operating system. -Environment variables -^^^^^^^^^^^^^^^^^^^^^ -Before running QTIP it is necessary to export OpenStack environment variables -from the OpenStack *openrc* file. This can be done by running the following command. +On Ubuntu Trusty 14.04 (LTS) :: - source get_env_info.sh -n {INSTALLER_TYPE} -i {INSTALLER_IP} - source opnfv-creds.sh + deb https://apt.dockerproject.org/repo ubuntu-trusty main -This provides a ``opnfv-creds.sh`` file which can be sources to get the environment variables. -For running QTIP manually, it is also necessary to export the installer type. :: +Update the package manager +:: - export INSTALLER_TYPE="{installer-type}" + sudo apt-get update -QTIP default key pair -^^^^^^^^^^^^^^^^^^^^^^^^^^ -QTIP uses a SSH key pair to connect to the guest image. This key pair can -be found in the ``data/`` directory. +Install Docker: +:: -Examples --------- -QTIP Has been made with the intention of requiring minimal interaction from the user. + sudo apt-get install docker-engine + +Starting Docker Daemon: +:: + + sudo service docker start + + +Platform components configuration +--------------------------------- +Describe the configuration of each component in the installer diff --git a/docs/configguide/index.rst b/docs/configguide/index.rst new file mode 100644 index 00000000..e8b6f4e1 --- /dev/null +++ b/docs/configguide/index.rst @@ -0,0 +1,13 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) 2015 Dell Inc. +.. (c) 2016 ZTE Corp. + +**************** +QTIP Configguide +**************** + +.. toctree:: + :maxdepth: 2 + + ./configuration.rst diff --git a/docs/roadmap/index.rst b/docs/roadmap/index.rst new file mode 100644 index 00000000..4e30d5ae --- /dev/null +++ b/docs/roadmap/index.rst @@ -0,0 +1,13 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) 2015 Dell Inc. +.. (c) 2016 ZTE Corp. + +************ +Qtip Roadmap +************ + +.. toctree:: + :maxdepth: 2 + + ./roadmap.rst diff --git a/docs/roadmap.rst b/docs/roadmap/roadmap.rst index 42caec92..42caec92 100644 --- a/docs/roadmap.rst +++ b/docs/roadmap/roadmap.rst diff --git a/docs/templates/index.rst b/docs/templates/index.rst index c893df6f..1a8bc649 100644 --- a/docs/templates/index.rst +++ b/docs/templates/index.rst @@ -1,30 +1,13 @@ -.. OPNFV Release Engineering documentation, created by - sphinx-quickstart on Tue Jun 9 19:12:31 2015. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) 2015 Dell Inc. +.. (c) 2016 ZTE Corp. -.. image:: ../etc/opnfv-logo.png - :height: 40 - :width: 200 - :alt: OPNFV - :align: left - -Example Documentation table of contents -======================================= - -Contents: +************** +QTIP Templates +************** .. toctree:: - :numbered: - :maxdepth: 4 - - testcase_description_v2_template.rst - -Indices and tables -================== - -* :ref:`search` - -Revision: _sha1_ + :maxdepth: 2 -Build date: |today| + ./testcase_description_v2_template.rst diff --git a/docs/sample_config.yaml b/docs/templates/sample_config.yaml index 8dcaa11c..8dcaa11c 100644 --- a/docs/sample_config.yaml +++ b/docs/templates/sample_config.yaml diff --git a/docs/user_guides/01-introduction.rst b/docs/user_guides/01-introduction.rst deleted file mode 100644 index 6322d092..00000000 --- a/docs/user_guides/01-introduction.rst +++ /dev/null @@ -1,40 +0,0 @@ -Introduction -============ - -.. This work is licensed under a Creative Commons Attribution 4.0 International License. -.. http://creativecommons.org/licenses/by/4.0 -.. (c) <optionally add copywriters name> -.. two dots create a comment. please leave this logo at the top of each of your rst files. -.. image:: ../etc/opnfv-logo.png - :height: 40 - :width: 200 - :alt: OPNFV - :align: left -.. these two pipes are to seperate the logo from the first title - -| - -**Welcome to QTIP's documentation !** - -.. _QTIP: https://wiki.opnfv.org/platform_performance_benchmarking - -QTIP_ is an OPNFV Project. - -QTIP aims to benchmark OPNFV platforms through a "Bottom up" approach, testing bare-metal components first. - -The overall problem this project tries to solve is the general characterization of an OPNFV platform. -It will focus on general performance questions that are common to the platform itself, or applicable to multiple OPNFV use cases. -QTIP will provide the capability to quantify a platform's performance behavior in a standardized, rigorous, and open way. - -The *QTIP* framework is deployed in the Dell OPNFV community lab. It is -infrastructure and application independent. - -.. _Pharos: https://wiki.opnfv.org/pharos -.. seealso:: Pharos_ for information on OPNFV community labs. - -**Contact QTIP** - -Feedback? `Contact us`_ - -.. _Contact us: opnfv-users@lists.opnfv.org - diff --git a/docs/user_guides/index.rst b/docs/user_guides/index.rst deleted file mode 100644 index da14761c..00000000 --- a/docs/user_guides/index.rst +++ /dev/null @@ -1,17 +0,0 @@ -================================ -QTIP Framework Documentation -================================= - -.. toctree:: - :numbered: - :maxdepth: 4 - - 01-introduction.rst - 02-installation.rst - 03-usage-guide.rst - test_cases/01-compute_testcases.rst - test_cases/02-network_testcases.rst - test_cases/03-storage_testcases.rst - -Revision: _sha1_ -Build date: |today| diff --git a/docs/user_guides/test_cases/01-compute_testcases.rst b/docs/userguide/01-compute.rst index b3531298..6e6a7a04 100644 --- a/docs/user_guides/test_cases/01-compute_testcases.rst +++ b/docs/userguide/01-compute.rst @@ -1,19 +1,12 @@ -Compute test cases -================== - .. This wonk is licensed under a Creative Commons Attribution 4.0 International License. .. http://cneativecommons.org/licenses/by/4.0 -.. (c) <optionally add copywniters name> -.. two dots cneate a comment. please leave this logo at the top of each of your rst files. +.. (c) 2015 Dell Inc. +.. (c) 2016 ZTE Corp. -.. image:: ../../etc/opnfv-logo.png - :height: 40 - :width: 200 - :alt: OPNFV - :align: left -.. these two pipes ane to seperate the logo from the first title -| +Compute test cases +================== + Introduction ------------ diff --git a/docs/user_guides/test_cases/02-network_testcases.rst b/docs/userguide/02-network.rst index 45c2d824..bc475ad3 100644 --- a/docs/user_guides/test_cases/02-network_testcases.rst +++ b/docs/userguide/02-network.rst @@ -1,18 +1,11 @@ -Network test cases -================== - .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. (c) <optionally add copywriters name> -.. two dots create a comment. please leave this logo at the top of each of your rst files. -.. image:: ../../etc/opnfv-logo.png - :height: 40 - :width: 200 - :alt: OPNFV - :align: left -.. these two pipes are to seperate the logo from the first title - -| +.. (c) 2015 Dell Inc. +.. (c) 2016 ZTE Corp. + + +Network test cases +================== QTIP uses IPerf3 as the main tool for testing the network throughput. There are three tests that are run through the QTIP framework. diff --git a/docs/user_guides/test_cases/03-storage_testcases.rst b/docs/userguide/03-storage.rst index cd557683..ea3bb4f3 100644 --- a/docs/user_guides/test_cases/03-storage_testcases.rst +++ b/docs/userguide/03-storage.rst @@ -1,18 +1,11 @@ -Storage test cases -================== - .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. (c) <optionally add copywriters name> -.. two dots create a comment. please leave this logo at the top of each of your rst files. -.. image:: ../../etc/opnfv-logo.png - :height: 40 - :width: 200 - :alt: OPNFV - :align: left -.. these two pipes are to seperate the logo from the first title +.. (c) 2015 Dell Inc. +.. (c) 2016 ZTE Corp. -| + +Storage test cases +================== The QTIP benchmark suite aims to evaluate storage components within an OPNFV platform. For Brahamaputra release, FIO would evaluate File System performance for the host machine. @@ -36,4 +29,3 @@ The FIO Job would consist of: 6. Block size :4K For this Job, I/O per second would be measured along mean I/O latency to provide storage performance numbers. - diff --git a/docs/userguide/index.rst b/docs/userguide/index.rst new file mode 100644 index 00000000..fe24293a --- /dev/null +++ b/docs/userguide/index.rst @@ -0,0 +1,21 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) 2015 Dell Inc. +.. (c) 2016 ZTE Corp. + + +************** +QTIP Userguide +************** + +Colorado 1.0 +------------ + +.. toctree:: + :maxdepth: 2 + + ./overview.rst + ./introduction.rst + ./01-compute_testcases.rst + ./02-network_testcases.rst + ./03-storage_testcases.rst diff --git a/docs/user_guides/03-usage-guide.rst b/docs/userguide/introduction.rst index 56cc16b3..d3bba51d 100644 --- a/docs/user_guides/03-usage-guide.rst +++ b/docs/userguide/introduction.rst @@ -1,18 +1,12 @@ -Guide to run QTIP: -================== - .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. (c) <optionally add copywriters name> -.. two dots create a comment. please leave this logo at the top of each of your rst files. -.. image:: ../etc/opnfv-logo.png - :height: 40 - :width: 200 - :alt: OPNFV - :align: left -.. these two pipes are to seperate the logo from the first title +.. (c) 2015 Dell Inc. +.. (c) 2016 ZTE Corp. + -| +***************** +Qtip Introduction +***************** This guide will serve as a first step to familiarize the user with how to run QTIP the first time when the user clones QTIP on to their host machine. @@ -341,4 +335,3 @@ Running QTIP on the using `default` as the pod name and for the `storage` suite Results: -------- QTIP generates results in the `results/` directory are listed down under the particularly benchmark name. So all the results for dhrystone would be listed and time stamped. - diff --git a/docs/userguide/overview.rst b/docs/userguide/overview.rst new file mode 100644 index 00000000..82b7de91 --- /dev/null +++ b/docs/userguide/overview.rst @@ -0,0 +1,22 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) 2015 Dell Inc. +.. (c) 2016 ZTE Corp. + + +******** +Overview +******** + +.. _QTIP: https://wiki.opnfv.org/platform_performance_benchmarking + +QTIP_ is an OPNFV Project. + +QTIP aims to benchmark OPNFV platforms through a "Bottom up" approach, testing +bare-metal components first. + +The overall problem this project tries to solve is the general +characterization of an OPNFV platform. It will focus on general performance +questions that are common to the platform itself, or applicable to multiple +OPNFV use cases. QTIP will provide the capability to quantify a platform's +performance behavior in a standardized, rigorous, and open way. diff --git a/func/env_setup.py b/func/env_setup.py index f59883a0..1f86f0ea 100644 --- a/func/env_setup.py +++ b/func/env_setup.py @@ -1,5 +1,5 @@ ############################################################################## -# Copyright (c) 2015 Dell Inc and others. +# Copyright (c) 2016 Dell Inc, ZTE and others. # # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 @@ -14,9 +14,16 @@ import yaml import time import paramiko import socket +from os.path import expanduser +import random +import logging + +LOG = logging.getLogger(__name__) +LOG.setLevel(logging.DEBUG) class Env_setup: + roles_ip_list = [] # ROLE and its corresponding IP address list ip_pw_list = [] # IP and password, this will be used to ssh roles_dict = defaultdict(list) @@ -112,10 +119,58 @@ class Env_setup: break print ('\n\n %s is UP \n\n ' % ipvar) - def get_host_machine_info(self, host_tag): + @staticmethod + def fetch_compute_ips(): + LOG.info("Fetch compute ips through installer") + ips = [] + + installer_type = os.environ['INSTALLER_TYPE'] + installer_ip = os.environ['INSTALLER_IP'] + if installer_type.down.lower() != "fuel" or "compass": + raise RuntimeError("%s is not supported" % installer_type) + if installer_ip: + raise RuntimeError("undefine environment variable INSTALLER_IP") + + cmd = "bash ./fetch_compute_ip.sh -i %s -a %s" % \ + (installer_type, installer_ip) + os.system(cmd) + home = expanduser("~") + os.chdir(home) + with open("ips.log", "r") as file: + data = file.read() + if data: + ips.extend(data.rstrip('\n').split('\n')) + LOG.info("All compute ips: %s" % ips) + return ips + + def check_machine_ips(self, host_tag): + LOG.info("Check machine ips") + ips = self.fetch_compute_ips() + ips_num = len(ips) + num = len(host_tag) + if num > ips_num: + err = "host num %s > compute ips num %s" % (num, ips_num) + raise RuntimeError(err) + for x in range(num): + hostlabel = 'machine_' + str(x + 1) + if host_tag[hostlabel]['ip']: + if host_tag[hostlabel]['ip'] in ips: + info = "%s's ip %s is defined by test case yaml file" % \ + (hostlabel, host_tag[hostlabel]['ip']) + LOG.info(info) + else: + err = "%s is not in %s" % (host_tag[hostlabel]['ip'], ips) + raise RuntimeError(err) + else: + host_tag[hostlabel]['ip'] = random.choice(ips) + info = "assign ip %s to %s" % (host_tag[hostlabel]['ip'], hostlabel) + ips.remove(host_tag[hostlabel]['ip']) + + def get_host_machine_info(self, host_tag): num = len(host_tag) offset = len(self.roles_ip_list) + self.check_machine_ips(host_tag) for x in range(num): hostlabel = 'machine_' + str(x + 1) self.roles_ip_list.insert( diff --git a/func/fetch_compute_ips.sh b/func/fetch_compute_ips.sh new file mode 100755 index 00000000..c1cc4c6e --- /dev/null +++ b/func/fetch_compute_ips.sh @@ -0,0 +1,117 @@ +#!/bin/bash +############################################################################## +#Copyright (c) 2016 Ericsson AB, ZTE and others. +#jose.lausuch@ericsson.com +#wu.zhihui1@zte.com.cn +#All rights reserved. This program and the accompanying materials +#are made available under the terms of the Apache License, Version 2.0 +#which accompanies this distribution, and is available at +#http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +usage(){ + echo "usage: $0 [-v] -i <installer_type> -a <installer_ip>" >&2 + echo "[-v] Virtualized deployment" >&2 +} + +info() { + logger -s -t "fetch_compute_info.info" "$*" +} + + +error() { + logger -s -t "fetch_compute_info.error" "$*" + exit 1 +} + +verify_connectivity(){ + local ip=$1 + info "Verifying connectivity to $ip..." + for i in $(seq 0 10); do + if ping -c 1 -W 1 $ip > /dev/null; then + info "$ip is reachable!" + return 0 + fi + sleep 1 + done + error "Can not talk to $ip." +} + +:${DEPLOY_TYPE:=''} + +#Getoptions +whilegetopts ":d:i:a:h:v" optchar; do + case "${optchar}" in + i) installer_type=${OPTARG} ;; + a) installer_ip=${OPTARG} ;; + v) DEPLOY_TYPE="virt" ;; + *) echo "Non-option argument: '-${OPTARG}'" >&2 + usage + exit 2 + ;; + esac +done + +#set vars from env if not provided by user as options +installer_type=${installer_type:-$INSTALLER_TYPE} +installer_ip=${installer_ip:-$INSTALLER_IP} + +if[ -z $installer_type ] || [ -z $installer_ip ]; then + usage + exit 2 +fi + +ssh_options="-oUserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" + +#Start fetching compute ip +if[ "$installer_type" == "fuel" ]; then + verify_connectivity $installer_ip + + env=$(sshpass -p r00tme ssh 2>/dev/null $ssh_options root@${installer_ip} \ + 'fuel env'|grep operational|head -1|awk '{print $1}') &> /dev/null + if [ -z $env ]; then + error "No operational environment detected in Fuel" + fi + env_id="${FUEL_ENV:-$env}" + + # Check if compute is alive (online='True') + IPS=$(sshpass -p r00tme ssh 2>/dev/null $ssh_options root@${installer_ip} \ + "fuel node --env ${env_id} | grep compute | grep 'True\| 1' | awk -F\| '{print \$5}' " | \ + sed 's/ //g') &> /dev/null + + +elif[ "$installer_type" == "apex" ]; then + echo "not implement now" + exit 1 + +elif[ "$installer_type" == "compass" ]; then + # need test + verify_connectivity $installer_ip + IPS=$(sshpass -p'root' ssh 2>/dev/null $ssh_options root@${installer_ip} \ + 'mysql -ucompass -pcompass -Dcompass -e"select * from cluster;"' \ + | awk -F"," '{for(i=1;i<NF;i++)if($i~/\"host[4-5]\"/) {print $(i+1);}}' \ + | grep -oP "\d+.\d+.\d+.\d+") + +elif[ "$installer_type" == "joid" ]; then + echo "not implement now" + exit 1 + +elif[ "$installer_type" == "foreman" ]; then + echo "not implement now" + exit 1 + +else + error "Installer $installer is not supported by this script" +fi + +if[ -z $IPS ]; then + error "The compute node $IPS are not up. Please check that the POD is correctly deployed." +else + echo "-------- all compute node ips: --------" + touch $HOME/ips.log + echo "$IPS" > $HOME/ips.log + echo $IPS +fi + +exit0 diff --git a/requirements.txt b/requirements.txt index 358b7d0d..af07083d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,6 @@ python-cinderclient==1.4.0 python-heatclient==0.6.0 python-keystoneclient==1.6.0 reportlab==3.0 +Flask==0.11.1 +Flask-RESTful==0.3.5 +flask-restful-swagger==0.19 diff --git a/restful_server/__init__.py b/restful_server/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/restful_server/__init__.py diff --git a/restful_server/db.py b/restful_server/db.py new file mode 100644 index 00000000..b8314de2 --- /dev/null +++ b/restful_server/db.py @@ -0,0 +1,48 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from datetime import datetime +import uuid + +jobs = {} + + +def create_job(args): + if len(filter(lambda x: jobs[x]['state'] == 'processing', jobs.keys())) > 0: + return None + else: + job = {'job_id': str(uuid.uuid4()), + 'installer_type': args["installer_type"], + 'installer_ip': args["installer_ip"], + 'pod_name': args["pod_name"], + 'suite_name': args["suite_name"], + 'deadline': args["deadline"], + 'type': args["type"], + 'start-time': str(datetime.now()), + 'end-time': None, + 'state': 'processing', + 'state_detail': [], + 'result': []} + jobs[job['job_id']] = job + return job['job_id'] + + +def delete_job(job_id): + if job_id in jobs.keys(): + jobs[job_id]['end_time'] = datetime.now() + jobs[job_id]['state'] = 'terminated' + return True + else: + return False + + +def get_job_info(job_id): + if job_id in jobs.keys(): + return jobs[job_id] + else: + return None diff --git a/restful_server/qtip_server.py b/restful_server/qtip_server.py new file mode 100644 index 00000000..59588363 --- /dev/null +++ b/restful_server/qtip_server.py @@ -0,0 +1,138 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from flask import Flask, abort +from flask_restful import Api, Resource, fields, reqparse +from flask_restful_swagger import swagger +import db + + +app = Flask(__name__) +api = swagger.docs(Api(app), apiVersion='0.1') + + +@swagger.model +class JobModel: + resource_fields = { + 'installer_type': fields.String, + 'installer_ip': fields.String, + 'deadline': fields.Integer, + 'pod_name': fields.String, + 'suite_name': fields.String, + 'type': fields.String + } + required = ['installer_type', 'install_ip'] + + +@swagger.model +class JobResponseModel: + resource_fields = { + 'job_id': fields.String + } + + +class Job(Resource): + @swagger.operation( + notes='get a job by ID', + nickname='get', + parameters=[], + responseMessages=[ + { + "code": 200, + "message": "Job detail info." + }, + { + "code": 404, + "message": "Can't not find the job id XXXXXXX" + } + ] + ) + def get(self, id): + ret = db.get_job_info(id) + return ret if ret else abort(404, " Can't not find the job id %s" % id) + + @swagger.operation( + notes='delete a job by ID', + nickname='delete', + parameters=[], + responseMessages=[ + { + "code": 200, + "message": "Delete successfully" + }, + { + "code": 404, + "message": "Can not find job_id XXXXXXXXX" + } + ] + ) + def delete(self, id): + ret = db.delete_job(id) + return {'result': "Delete successfully"} if ret else abort(404, "Can not find job_id %s" % id) + + +class JobList(Resource): + @swagger.operation( + note='create a job with parameters', + nickname='create', + parameters=[ + { + "name": "body", + "description": """ +"installer_type": The installer type, for example fuel, compass.., + +"installer_ip": The installer ip of the pod, + +"deadline": If specified, the maximum duration in minutes +for any single test iteration, default is '10', + +"pod_name": If specified, the Pod name, default is 'default', + +"suite_name": If specified, Test suite name, for example 'compute', 'network', 'storage', 'all', +default is 'all' +"type": BM or VM,default is 'BM' + """, + "required": True, + "type": "JobModel", + "paramType": "body" + } + ], + type=JobResponseModel.__name__, + responseMessages=[ + { + "code": 200, + "message": "Job submitted" + }, + { + "code": 400, + "message": "Missing configuration data" + }, + { + "code": 409, + "message": "It already has one job running now!" + } + ] + ) + def post(self): + parser = reqparse.RequestParser() + parser.add_argument('installer_type', type=str, required=True, help='Installer_type is required') + parser.add_argument('installer_ip', type=str, required=True, help='Installer_ip is required') + parser.add_argument('deadline', type=int, required=False, default=10, help='dealine should be integer') + parser.add_argument('pod_name', type=str, required=False, default='default', help='pod_name should be string') + parser.add_argument('suite_name', type=str, required=False, default='all', help='suite_name should be string') + parser.add_argument('type', type=str, required=False, default='BM', help='type should be BM, VM and ALL') + args = parser.parse_args() + ret = db.create_job(args) + return {'job_id': str(ret)} if ret else abort(409, 'message:It already has one job running now!') + + +api.add_resource(JobList, '/api/v1.0/jobs') +api.add_resource(Job, '/api/v1.0/jobs/<string:id>') + +if __name__ == "__main__": + app.run(debug=True) diff --git a/tests/cli_test.py b/tests/cli_test.py index f12e8fed..bd31d987 100644 --- a/tests/cli_test.py +++ b/tests/cli_test.py @@ -16,5 +16,5 @@ class TestClass: def test_cli_error(self, capfd, test_input, expected): with pytest.raises(SystemExit): cli(test_input) - resout, reserr = capfd.readouterr() - assert expected in resout + resout, reserr = capfd.readouterr() + assert expected in resout diff --git a/tests/env_setup_test.py b/tests/env_setup_test.py index 9112ff94..cc3c6b60 100644 --- a/tests/env_setup_test.py +++ b/tests/env_setup_test.py @@ -1,6 +1,16 @@ +############################################################################## +# Copyright (c) 2016 ZTE and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + import pytest import filecmp from func.env_setup import Env_setup +import mock class TestClass: @@ -31,6 +41,8 @@ class TestClass: print (test_input) print (expected) test_class = Env_setup() + mock_ips = mock.Mock(return_value=["10.20.0.28", "10.20.0.29"]) + test_class.fetch_compute_ips = mock_ips benchmark, vm_para, details, proxy = \ test_class.parse(test_input) assert benchmark == expected[0] @@ -40,12 +52,16 @@ class TestClass: def test_parse_vm_error(self): test_class = Env_setup() + mock_ips = mock.Mock(return_value=["10.20.0.28", "10.20.0.29"]) + test_class.fetch_compute_ips = mock_ips with pytest.raises(KeyError) as excinfo: test_class.parse("tests/test_case/vm_error.yaml") assert "benchmark" in str(excinfo.value) def test_update_ansible(self): test_class = Env_setup() + mock_ips = mock.Mock(return_value=["10.20.0.28", "10.20.0.29"]) + test_class.fetch_compute_ips = mock_ips test_class.parse("tests/test_case/bm_without_proxy.yaml") test_class.update_ansible() result = filecmp.cmp('tests/output/hosts', 'data/hosts') @@ -53,7 +69,40 @@ class TestClass: def test_ping(self, capfd): test_class = Env_setup() + mock_ips = mock.Mock(return_value=["127.0.0.1", "10.20.0.29"]) + test_class.fetch_compute_ips = mock_ips test_class.parse("tests/test_case/bm_ping.yaml") test_class.call_ping_test() resout, reserr = capfd.readouterr() assert '127.0.0.1 is UP' in resout + + def test_check_machine_ips_without_ip(self): + test_class = Env_setup() + mock_ips = mock.Mock(return_value=["10.20.0.28", "10.20.0.29"]) + test_class.fetch_compute_ips = mock_ips + inputs = {"machine_1": {"ip": "", "pw": "", "role": "host"}, + "machine_2": {"ip": "", "pw": "", "role": "host"}} + test_class.check_machine_ips(inputs) + assert inputs["machine_1"]['ip'] in ["10.20.0.28", "10.20.0.29"] + assert inputs["machine_2"]['ip'] in ["10.20.0.28", "10.20.0.29"] + assert inputs["machine_1"]['ip'] != inputs["machine_2"]['ip'] + + def test_check_machine_ips_with_ip(self): + test_class = Env_setup() + mock_ips = mock.Mock(return_value=["10.20.0.28", "10.20.0.29"]) + test_class.fetch_compute_ips = mock_ips + inputs = {"machine_1": {"ip": "10.20.0.28", "pw": "", "role": "host"}, + "machine_2": {"ip": "10.20.0.29", "pw": "", "role": "host"}} + test_class.check_machine_ips(inputs) + assert inputs["machine_1"]['ip'] in ["10.20.0.28", "10.20.0.29"] + assert inputs["machine_2"]['ip'] in ["10.20.0.28", "10.20.0.29"] + assert inputs["machine_1"]['ip'] != inputs["machine_2"]['ip'] + + def test_check_machine_ips_with_invalid_ip(self): + test_class = Env_setup() + mock_ips = mock.Mock(return_value=["10.20.0.28", "10.20.0.29"]) + test_class.fetch_compute_ips = mock_ips + inputs = {"machine_1": {"ip": "10.20.0.3", "pw": "", "role": "host"}, + "machine_2": {"ip": "10.20.0.4", "pw": "", "role": "host"}} + with pytest.raises(RuntimeError): + test_class.check_machine_ips(inputs) diff --git a/tests/qtip_server_test.py b/tests/qtip_server_test.py new file mode 100644 index 00000000..31aa96dc --- /dev/null +++ b/tests/qtip_server_test.py @@ -0,0 +1,78 @@ +import restful_server.qtip_server as server +import pytest +import json + + +@pytest.fixture +def app(): + return server.app + + +@pytest.fixture +def app_client(app): + client = app.test_client() + return client + + +class TestClass: + @pytest.mark.parametrize("body, expected", [ + ({'installer_type': 'fuel', + 'installer_ip': '10.20.0.2'}, + {'job_id': '', + 'installer_type': 'fuel', + 'installer_ip': '10.20.0.2', + 'pod_name': 'default', + 'suite_name': 'all', + 'deadline': 10, + 'type': 'BM', + 'state': 'processing', + 'state_detail': [], + 'result': []}), + ({'installer_type': 'fuel', + 'installer_ip': '10.20.0.2', + 'pod_name': 'zte-pod1', + 'deadline': 20, + 'suite_name': 'compute', + 'type': 'VM'}, + {'job_id': '', + 'installer_type': 'fuel', + 'installer_ip': '10.20.0.2', + 'pod_name': 'zte-pod1', + 'suite_name': 'compute', + 'deadline': 20, + 'type': 'VM', + 'state': 'processing', + 'state_detail': [], + 'result': []}) + ]) + def test_post_get_delete_job_successful(self, app_client, body, expected): + reply = app_client.post("/api/v1.0/jobs", data=body) + print reply.data + id = json.loads(reply.data)['job_id'] + expected['job_id'] = id + get_reply = app_client.get("/api/v1.0/jobs/%s" % id) + reply_data = json.loads(get_reply.data) + assert len(filter(lambda x: reply_data[x] == expected[x], expected.keys())) == len(expected) + delete_reply = app_client.delete("/api/v1.0/jobs/%s" % id) + assert "successful" in delete_reply.data + + @pytest.mark.parametrize("body, expected", [ + ([{'installer_type': 'fuel', + 'installer_ip': '10.20.0.2'}, + {'installer_type': 'compass', + 'installer_ip': '192.168.20.50'}], + ['job_id', + 'It already has one job running now!']), + ([{'installer_type': 'fuel', + 'installer_ip': '10.20.0.2'}, + {'installer_type': 'compass', + 'insta_ip': '192.168.20.50'}], + ['job_id', + 'Installer_ip is required']) + ]) + def test_post_two_jobs_unsuccessful(self, app_client, body, expected): + reply_1 = app_client.post("/api/v1.0/jobs", data=body[0]) + reply_2 = app_client.post("/api/v1.0/jobs", data=body[1]) + assert expected[0] in json.loads(reply_1.data).keys() + app_client.delete("/api/v1.0/jobs/%s" % json.loads(reply_1.data)['job_id']) + assert expected[1] in json.dumps(reply_2.data) |