diff options
23 files changed, 387 insertions, 256 deletions
diff --git a/docker/Dockerfile b/docker/Dockerfile index bab503fc..0405aa45 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -8,11 +8,17 @@ LABEL version="0.1" description="OPNFV QTIP Docker container" ARG BRANCH=master -ENV REPOS_DIR /home/opnfv/repos -ENV PYTHONPATH /home/opnfv/repos/qtip -ENV USER root +ENV REPOS_DIR=/home/opnfv/repos \ + PYTHONPATH=/home/opnfv/repos/qtip \ + USER=root + WORKDIR /home/opnfv +RUN mkdir -p ${REPOS_DIR} \ + && mkdir -p /root/qtip/logs \ + && mkdir -p /root/.ssh \ + && chmod 700 /root/.ssh + # Packaged Dependencies RUN apt-get update && apt-get install -y \ software-properties-common \ @@ -28,26 +34,19 @@ RUN apt-get update && apt-get install -y \ python-setuptools \ rsync \ iputils-ping \ + wget \ + curl \ + openssh-client \ --no-install-recommends \ -&& rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* RUN pip install -U pip && pip install -U setuptools -RUN apt-add-repository ppa:ansible/ansible -y -RUN apt-key update -y -RUN apt-get update && apt-get install ansible -y - -RUN mkdir -p ${REPOS_DIR} -RUN mkdir -p /root/.ssh -RUN mkdir -p /root/qtip/logs - -RUN chmod 700 /root/.ssh - #Cloning Repos -RUN git config --global http.sslVerify false -RUN git clone -b $BRANCH https://gerrit.opnfv.org/gerrit/qtip $REPOS_DIR/qtip - -RUN cd $REPOS_DIR/qtip && pip install -U -e . +RUN git config --global http.sslVerify false \ + && git clone -b $BRANCH https://gerrit.opnfv.org/gerrit/qtip $REPOS_DIR/qtip \ + && cd $REPOS_DIR/qtip \ + && pip install -U -e . RUN echo 'eval $(ssh-agent)' >> /root/.bashrc @@ -55,8 +54,7 @@ RUN echo 'eval $(ssh-agent)' >> /root/.bashrc EXPOSE 5000 #Config supervisor -RUN mkdir -p /var/log/supervisor -RUN locale-gen en_US en_US.UTF-8 +RUN mkdir -p /var/log/supervisor && locale-gen en_US en_US.UTF-8 COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf CMD ["/usr/bin/supervisord"]
\ No newline at end of file diff --git a/integration/storperf/README.md b/integration/storperf/README.md index 137e15be..bdcc7e70 100644 --- a/integration/storperf/README.md +++ b/integration/storperf/README.md @@ -1,13 +1,45 @@ -# usage +# Usage Please make sure pip, docker and docker-compose are installer on your environment. -``` -storperf.sh -t installer_type -i installer_ip -s stack_json_file -j job_json_file +1. Launch qtip and storperf containers. -options: - -t : installer type. For now only supports Apex. - -i : installer ip address. - -s : Stack configuration json file. If not given, default_stack.json will be used. - -j : Storperf job configuration json file. If not given, default_job.json will be used. -```
\ No newline at end of file + ``` + $ cd qtip/integration/storperf + $ bash launch_containers.sh -t apex -n "" + + Arguments: + + -t : Installer type. For now only supports Apex. + -n : Node name. + ``` + + Then you will get 5 containers: + + ``` + CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES + 2c079d5d88bc opnfv/qtip "/usr/bin/supervisord" 38 minutes ago Up 38 minutes 5000/tcp qtip + 2032a16dab17 opnfv/storperf-httpfrontend "nginx -g 'daemon ..." 38 minutes ago Up 38 minutes 80/tcp, 0.0.0.0:5000->5000/tcp storperf-httpfrontend + c0d3e2763d35 schickling/swagger-ui "sh run.sh" 39 minutes ago Up 38 minutes 80/tcp storperf-swaggerui + 0e08a1968829 opnfv/storperf-reporting "python app.py" 39 minutes ago Up 38 minutes 0.0.0.0:5080->5000/tcp storperf-reporting + b92967139d8e opnfv/storperf-master "/usr/bin/supervisord" 39 minutes ago Up 39 minutes 5000/tcp, 0.0.0.0:8000->8000/tcp storperf-master + ``` + +2. Prepare environment. + + ``` + $ docker exec qtip bash -c "/home/opnfv/repos/qtip/integration/storperf/prepare.sh" + ``` + + This command can create a flavor and upload a Ubuntu 16.04 image on OpenStack. + +3. Run storperf job. + + ``` + $ docker exec qtip bash -c "/home/opnfv/repos/qtip/integration/storperf/start_job.sh -s stack.json -j job.json" + + Arguments: + + -s : Stack configuration json file. If not given, default_stack.json will be used. + -j : Storperf job configuration json file. If not given, default_job.json will be used. + ``` diff --git a/integration/storperf/cleanup.sh b/integration/storperf/cleanup.sh new file mode 100755 index 00000000..1aee1658 --- /dev/null +++ b/integration/storperf/cleanup.sh @@ -0,0 +1,14 @@ +#! /bin/bash +############################################################################## +# Copyright (c) 2017 ZTE and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +echo "Deleting image" +openstack image delete "Ubuntu 16.04 x86_64" + +echo "Deteing flavor" +openstack flavor delete storperf diff --git a/integration/storperf/storperf-docker-compose.yaml b/integration/storperf/docker-compose.yaml index ec121802..8cef5bc3 100644 --- a/integration/storperf/storperf-docker-compose.yaml +++ b/integration/storperf/docker-compose.yaml @@ -1,5 +1,5 @@ ############################################################################## -# Copyright (c) 2017 Dell EMC and others. +# Copyright (c) 2017 Dell EMC, ZTE and others. # # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 @@ -10,6 +10,7 @@ version: '2' services: + storperf-master: container_name: "storperf-master" image: opnfv/storperf-master @@ -18,15 +19,12 @@ services: env_file: ${ENV_FILE} volumes: - ${CARBON_DIR}:/opt/graphite/storage/whisper - - ./storperf-master/:/home/opnfv/repos/storperf storperf-reporting: container_name: "storperf-reporting" image: opnfv/storperf-reporting ports: - "5080:5000" - volumes: - - ./storperf-reporting/:/home/opnfv/storperf-reporting storperf-swaggerui: container_name: "storperf-swaggerui" @@ -40,4 +38,11 @@ services: links: - storperf-master - storperf-reporting - - storperf-swaggerui
\ No newline at end of file + - storperf-swaggerui + + qtip: + container_name: qtip + image: opnfv/qtip + env_file: ${ENV_FILE} + links: + - storperf-httpfrontend diff --git a/integration/storperf/launch_containers.sh b/integration/storperf/launch_containers.sh new file mode 100755 index 00000000..3fcf8052 --- /dev/null +++ b/integration/storperf/launch_containers.sh @@ -0,0 +1,100 @@ +#! /bin/bash +############################################################################## +# Copyright (c) 2017 ZTE and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +usage(){ + echo "usage: $0 -t <installer_type> -n <node_name>" >&2 +} + +#Get options +while getopts ":t:n:he" optchar; do + case "${optchar}" in + t) installer_type=${OPTARG} ;; + n) node_name=${OPTARG} ;; + h) usage + exit 0 + ;; + *) echo "Non-option argument: '-${OPTARG}'" >&2 + usage + exit 2 + ;; + esac +done + + +#set vars from env if not provided by user as options +installer_type=${installer_type:-$INSTALLER_TYPE} +node_name=${node_name:-$NODE_NAME} + +case $installer_type in + apex) + installer_ip=`sudo virsh domifaddr undercloud | grep ipv4 | awk '{print $4}' | cut -d/ -f1` + ;; + *) + + echo "Unsupported $installer_type. Cannot continue!" + exit 1 +esac + +if [[ -z $WORKSPACE ]];then + WORKSPACE=`pwd` +fi + +git clone --depth 1 https://gerrit.opnfv.org/gerrit/releng $WORKSPACE/releng + +$WORKSPACE/releng/utils/fetch_os_creds.sh -i ${installer_type} -a ${installer_ip} -d $WORKSPACE/openrc + +grep "export" $WORKSPACE/openrc | sed "s/export //" > $WORKSPACE/admin.rc +echo "INSTALLER_TYPE=${installer_type}" >> $WORKSPACE/admin.rc +echo "INSTALLER_IP=${installer_ip}" >> $WORKSPACE/admin.rc +echo "NODE_NAME=${node_name}" >> $WORKSPACE/admin.rc +export ENV_FILE=$WORKSPACE/admin.rc + +export CARBON_DIR=$WORKSPACE/carbon/ +WWW_DATA_UID=33 +WWW_DATA_GID=33 +sudo install --owner=${WWW_DATA_UID} --group=${WWW_DATA_GID} -d "${CARBON_DIR}" + +clean_storperf_container() +{ + docker-compose -f docker-compose.yaml down + + for name in qtip storperf-master storperf-swaggerui storperf-httpfrontend storperf-reporting + do + container=`docker ps -a -q -f name=$name` + if [[ ! -z $container ]];then + echo "Stopping any existing $name container" + docker rm -fv $container + fi + + image=`docker images opnfv/$name` + if [[ ! -z $image ]];then + echo "Deleteing any existing opnfv/$name image" + docker rmi -f opnfv/$container_name + fi + done +} + +launch_storperf_container() +{ + docker-compose pull + docker-compose -f docker-compose.yaml up -d + + echo "Waiting for StorPerf to become active" + + while [ $(curl -s -o /dev/null -I -w "%{http_code}" -X GET http://127.0.0.1:5000/api/v1.0/configurations) != "200" ] + do + sleep 1 + done +} + +echo "Clean existing storperf containers" +clean_storperf_container + +echo "Launch new storperf containers" +launch_storperf_container
\ No newline at end of file diff --git a/integration/storperf/openstack.sh b/integration/storperf/prepare.sh index 99701871..345a80bc 100755 --- a/integration/storperf/openstack.sh +++ b/integration/storperf/prepare.sh @@ -7,6 +7,16 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## +# See https://stackoverflow.com/questions/59895/getting-the-source-directory-of-a-bash-script-from-within +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +pip install -r $script_dir/storperf_requirements.txt + +if [[ -z $WORKSPACE ]];then + WORKSPACE=`pwd` +fi + + delete_storperf_stack() { echo "Checking for an existing stack" @@ -16,6 +26,7 @@ delete_storperf_stack() fi } + load_ubuntu_image() { echo "Checking for Ubuntu 16.04 image in Glance" @@ -24,8 +35,8 @@ load_ubuntu_image() cd $WORKSPACE if [[ ! -f ubuntu-16.04-server-cloudimg-amd64-disk1.img ]];then echo "download Ubuntu 16.04 image" - wget https://cloud-images.ubuntu.com/releases/16.04/release/ubuntu-16.04-server-cloudimg-amd64-disk1.img - wget https://cloud-images.ubuntu.com/releases/16.04/release/MD5SUMS + wget -q https://cloud-images.ubuntu.com/releases/16.04/release/ubuntu-16.04-server-cloudimg-amd64-disk1.img + wget -q https://cloud-images.ubuntu.com/releases/16.04/release/MD5SUMS checksum=$(cat ./MD5SUMS |grep ubuntu-16.04-server-cloudimg-amd64-disk1.img | md5sum -c) if [[ $checksum =~ 'FAILED' ]];then echo "Check image md5sum failed. Exit!" @@ -58,8 +69,6 @@ create_storperf_flavor() } -nova_vm_mapping() -{ - rm ./nova_vm.json - openstack server list --name storperf-agent -c ID -c Host --long -f json > nova_vm.json -}
\ No newline at end of file +delete_storperf_stack +load_ubuntu_image +create_storperf_flavor diff --git a/integration/storperf/run_ci.sh b/integration/storperf/run_ci.sh new file mode 100755 index 00000000..ca67ac62 --- /dev/null +++ b/integration/storperf/run_ci.sh @@ -0,0 +1,21 @@ +#! /bin/bash +############################################################################## +# Copyright (c) 2017 ZTE and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +source $WORKSPACE/integration/storperf/launch_containers.sh -t $INSTALLER_TYPE -n $NODE_NAME + +docker exec qtip bash -c "/home/opnfv/repos/qtip/integration/storperf/prepare.sh" + +docker exec qtip bash -c "/home/opnfv/repos/qtip/integration/storperf/start_job.sh" + +docker exec qtip bash -c "/home/opnfv/repos/qtip/integration/storperf/cleanup.sh" + +exit 0 + + diff --git a/integration/storperf/start_job.sh b/integration/storperf/start_job.sh index e94356e5..672dabef 100755 --- a/integration/storperf/start_job.sh +++ b/integration/storperf/start_job.sh @@ -29,11 +29,22 @@ done # See https://stackoverflow.com/questions/59895/getting-the-source-directory-of-a-bash-script-from-within script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +stack_json=${stack_json:-"$script_dir/default_stack.json"} +job_json=${job_json:-"$script_dir/default_job.json"} + if [[ -z $WORKSPACE ]];then WORKSPACE=`pwd` fi -source $script_dir/openstack.sh +nova_vm_mapping() +{ + rm $WORKSPACE/nova_vm.json + openstack server list --name storperf-agent -c ID -c Host --long -f json > $WORKSPACE/nova_vm.json + + echo ========================================================================== + cat $WORKSPACE/nova_vm.json + echo ========================================================================== +} echo ========================================================================== echo "Start to create storperf stack" @@ -42,7 +53,7 @@ echo ========================================================================== curl -X POST --header 'Content-Type: application/json' \ --header 'Accept: application/json' -d @${stack_json} \ - 'http://127.0.0.1:5000/api/v1.0/configurations' + 'http://storperf-httpfrontend:5000/api/v1.0/configurations' nova_vm_mapping @@ -54,23 +65,26 @@ echo ========================================================================== JOB=$(curl -s -X POST --header 'Content-Type: application/json' \ --header 'Accept: application/json' \ - -d @${job_json} 'http://127.0.0.1:5000/api/v1.0/jobs' | \ + -d @${job_json} 'http://storperf-httpfrontend:5000/api/v1.0/jobs' | \ awk '/job_id/ {print $2}' | sed 's/"//g') echo "JOB ID: $JOB" if [[ -z "$JOB" ]]; then echo "Oops, JOB ID is empty!" else - echo "Loop: check job status" - curl -s -X GET "http://127.0.0.1:5000/api/v1.0/jobs?id=$JOB&type=status" \ + echo "checking job status..." + curl -s -X GET "http://storperf-httpfrontend:5000/api/v1.0/jobs?id=$JOB&type=status" \ -o $WORKSPACE/status.json + cat $WORKSPACE/status.json + JOB_STATUS=`cat $WORKSPACE/status.json | awk '/Status/ {print $2}' | cut -d\" -f2` + while [ "$JOB_STATUS" != "Completed" ] do - sleep 30 + sleep 180 mv $WORKSPACE/status.json $WORKSPACE/old-status.json - curl -s -X GET "http://127.0.0.1:5000/api/v1.0/jobs?id=$JOB&type=status" \ + curl -s -X GET "http://storperf-httpfrontend:5000/api/v1.0/jobs?id=$JOB&type=status" \ -o $WORKSPACE/status.json JOB_STATUS=`cat $WORKSPACE/status.json | awk '/Status/ {print $2}' | cut -d\" -f2` diff $WORKSPACE/status.json $WORKSPACE/old-status.json >/dev/null @@ -80,19 +94,16 @@ else fi done - echo ========================================================================== + echo echo "Storperf test completed!" - echo ========================================================================== - - curl -s -X GET "http://127.0.0.1:5000/api/v1.0/jobs?id=$JOB&type=metadata" \ - -o $WORKSPACE/report.json echo ========================================================================== echo Final report echo ========================================================================== + curl -s -X GET "http://storperf-httpfrontend:5000/api/v1.0/jobs?id=$JOB&type=metadata" \ + -o $WORKSPACE/report.json cat $WORKSPACE/report.json fi echo "Deleting stack for cleanup" -curl -s -X DELETE --header 'Accept: application/json' 'http://127.0.0.1:5000/api/v1.0/configurations' - +curl -s -X DELETE --header 'Accept: application/json' 'http://storperf-httpfrontend:5000/api/v1.0/configurations'
\ No newline at end of file diff --git a/integration/storperf/storperf.sh b/integration/storperf/storperf.sh deleted file mode 100755 index 31e059d7..00000000 --- a/integration/storperf/storperf.sh +++ /dev/null @@ -1,88 +0,0 @@ -#! /bin/bash -############################################################################## -# Copyright (c) 2017 ZTE and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -usage(){ - echo "usage: $0 -t <installer_type> -i <installer_ip> -s <stack_json_path> -j <job_json_path> " >&2 -} - -#Get options -while getopts ":t:i:s:j:he" optchar; do - case "${optchar}" in - t) installer_type=${OPTARG} ;; - i) installer_ip=${OPTARG} ;; - s) stack_json=${OPTARG} ;; - j) job_json=${OPTARG} ;; - h) usage - exit 0 - ;; - *) echo "Non-option argument: '-${OPTARG}'" >&2 - usage - exit 2 - ;; - esac -done - -# See https://stackoverflow.com/questions/59895/getting-the-source-directory-of-a-bash-script-from-within -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -if [[ -z $WORKSPACE ]];then - WORKSPACE=`pwd` -fi - -#set vars from env if not provided by user as options -installer_type=${installer_type:-$INSTALLER_TYPE} -installer_ip=${installer_ip:-$INSTALLER_IP} -stack_json=${stack_json:-"$script_dir/default_stack.json"} -job_json=${job_json:-"$script_dir/default_job.json"} - -source $script_dir/openstack.sh -source $script_dir/storperf_docker.sh - -git clone --depth 1 https://gerrit.opnfv.org/gerrit/storperf $WORKSPACE/storperf -git clone --depth 1 https://gerrit.opnfv.org/gerrit/releng $WORKSPACE/releng - -virtualenv $WORKSPACE/storperf_venv -source $WORKSPACE/storperf_venv/bin/activate - -pip install -r $script_dir/storperf_requirements.txt - -$WORKSPACE/releng/utils/fetch_os_creds.sh -i ${installer_type} -a ${installer_ip} -d $WORKSPACE/openrc -source $WORKSPACE/openrc - -grep "export" $WORKSPACE/openrc | sed "s/export //" > $WORKSPACE/admin.rc -echo "INSTALLER_TYPE=${installer_type}" >> $WORKSPACE/admin.rc -export ENV_FILE=$WORKSPACE/admin.rc - -if [[ ! -d $WORKSPACE/carbon ]];then - mkdir -p $WORKSPACE/carbon - sudo chown 33:33 $WORKSPACE/carbon -fi -export CARBON_DIR=$WORKSPACE/carbon/ - -delete_storperf_stack -load_ubuntu_image -create_storperf_flavor - -cd $WORKSPACE/storperf/docker -cp $script_dir/storperf-docker-compose.yaml ./ -echo "Clean existing storperf containers" -clean_storperf_container -echo "Launch new storperf containers" -launch_storperf_container - -$script_dir/start_job.sh -s $stack_json -j $job_json - -echo "Clean up environment" -cd $WORKSPACE/storperf/docker -clean_storperf_container -openstack flavor delete storperf -openstack image delete "Ubuntu 16.04 x86_64" - -echo "Done!" - diff --git a/integration/storperf/storperf_docker.sh b/integration/storperf/storperf_docker.sh deleted file mode 100755 index 629a4edb..00000000 --- a/integration/storperf/storperf_docker.sh +++ /dev/null @@ -1,35 +0,0 @@ -#! /bin/bash -############################################################################## -# Copyright (c) 2017 ZTE and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -clean_storperf_container() -{ - docker-compose -f storperf-docker-compose.yaml down - - for container_name in storperf storperf-master storperf-swaggerui storperf-httpfrontend storperf-reporting - do - container=`docker ps -a -q -f name=$container_name` - if [[ ! -z $container ]];then - echo "Stopping any existing $container_name container" - docker rm -fv $container - fi - done -} - - -launch_storperf_container() -{ - docker-compose -f storperf-docker-compose.yaml up -d - - echo "Waiting for StorPerf to become active" - - while [ $(curl -s -o /dev/null -I -w "%{http_code}" -X GET http://127.0.0.1:5000/api/v1.0/configurations) != "200" ] - do - sleep 1 - done -}
\ No newline at end of file diff --git a/integration/storperf/storperf_requirements.txt b/integration/storperf/storperf_requirments.txt index 4f8b07c5..4a279ae5 100644 --- a/integration/storperf/storperf_requirements.txt +++ b/integration/storperf/storperf_requirments.txt @@ -1,6 +1,5 @@ -setuptools==33.1.1 functools32==3.2.3.post2 pytz==2016.10 osc_lib==1.3.0 python-openstackclient==3.7.0 -python-heatclient==1.7.0
\ No newline at end of file +python-heatclient==1.7.0 diff --git a/qtip/ansible_library/modules/apex_generate_inventory.py b/qtip/ansible_library/modules/apex_generate_inventory.py index b1cc976d..0c9500f1 100644 --- a/qtip/ansible_library/modules/apex_generate_inventory.py +++ b/qtip/ansible_library/modules/apex_generate_inventory.py @@ -79,7 +79,8 @@ def generate_inventory(baremetal_info, server_info): node_ip = re.findall('.+=(\d+.\d+.\d+.\d+)$', server['Networks'])[0] hosts[role].append(node_ip) # To match ssh.cfg.j2 template - hosts_meta[node_ip] = {'ansible_ssh_host': node_ip} + hosts_meta[node_ip] = {'ansible_ssh_host': node_ip, + 'ansible_user': 'heat-admin'} for host in hosts: hosts[host].sort() diff --git a/qtip/ansible_library/modules/fuel.py b/qtip/ansible_library/modules/fuel.py index 5ec45dd1..04154e34 100644 --- a/qtip/ansible_library/modules/fuel.py +++ b/qtip/ansible_library/modules/fuel.py @@ -98,7 +98,8 @@ def generate_inventory(nodes): 'ip': node['ip'], 'mac': node['mac'], 'cluster': cluster_id, - 'ansible_ssh_host': node['ip'] + 'ansible_ssh_host': node['ip'], + 'ansible_user': 'root' } hosts_meta[hostname] = node_meta diff --git a/qtip/cli/commands/cmd_qpi.py b/qtip/cli/commands/cmd_qpi.py index d08842a4..4865b7ae 100644 --- a/qtip/cli/commands/cmd_qpi.py +++ b/qtip/cli/commands/cmd_qpi.py @@ -9,47 +9,41 @@ import click -from colorama import Fore import os +from os import path +from prettytable import PrettyTable +import yaml -from qtip.base.error import InvalidContentError -from qtip.base.error import NotFoundError -from qtip.cli import utils -from qtip.loader.qpi import QPISpec +QPI_PATH = path.join(path.dirname(__file__), '..', '..', '..', + 'resources', 'QPI') @click.group() def cli(): - ''' Collection of performance tests ''' + """ Collection of performance tests """ pass @cli.command('list', help='List all the QPI specs') def cmd_list(): - qpis = QPISpec.list_all() - table = utils.table('QPIs', qpis) + table = PrettyTable(['QPI', 'Description']) + table.align = 'l' + for qpi in os.listdir(QPI_PATH): + if qpi.endswith('yaml'): + with open('{}/{}'.format(QPI_PATH, qpi)) as conf: + details = yaml.safe_load(conf) + table.add_row([details['name'], details['description']]) click.echo(table) @cli.command('show', help='View details of a QPI') @click.argument('name') def show(name): - try: - qpi = QPISpec('{}.yaml'.format(name)) - except NotFoundError as nf: - click.echo(Fore.RED + "ERROR: qpi spec: " + nf.message) - except InvalidContentError as ice: - click.echo(Fore.RED + "ERROR: qpi spec: " + ice.message) - else: - cnt = qpi.content - output = utils.render('qpi', cnt) - click.echo(output) - - -@cli.command('run', help='Run performance tests for the specified QPI') -@click.argument('name') -@click.option('-p', '--path', help='Path to store results') -def run(name, path): - runner_path = path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir, - 'runner/runner.py') - os.system('python {0} -b all -d {1}'.format(runner_path, path)) + table = PrettyTable(['QPI', 'Description', 'Formula']) + table.align = 'l' + with open('{}/{}.yaml'.format(QPI_PATH, name)) as conf: + qpi = yaml.safe_load(conf) + for section in qpi['sections']: + table.add_row([section['name'], section['description'], + section['formula']]) + click.echo(table) diff --git a/requirements.txt b/requirements.txt index 3a13361a..3b1d7498 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,4 +11,4 @@ six PyYAML humanfriendly asq -Django +Django
\ No newline at end of file diff --git a/resources/QPI/compute.yaml b/resources/QPI/compute.yaml index 8169da4f..50c1cfbf 100644 --- a/resources/QPI/compute.yaml +++ b/resources/QPI/compute.yaml @@ -29,7 +29,7 @@ sections: # split based on different application metrics: - name: ssl_rsa description: performance of cryptographic using RSA cipher algorithm - formual: geometric mean + formula: geometric mean workloads: - name: rsa_sign_512 description: rsa 512 bits sign per second @@ -50,7 +50,7 @@ sections: # split based on different application - name: ssl_aes description: > performance of advanced encryption standard (AES) cipher algorithm in cipher block chaining (CBC) mode - formual: geometric mean + formula: geometric mean workloads: - name: aes_128_cbc_16_bytes description: aes 128 bits key cbc on 16 bytes blocks @@ -64,6 +64,7 @@ sections: # split based on different application description: aes 128 bits key cbc on 8192 bytes blocks - name: DPI description: deep packet inspection + formula: geometric mean metrics: - name: dpi_throughput description: deep packet inspection throughput @@ -74,6 +75,7 @@ sections: # split based on different application description: DPI bits per second - name: memory description: cache and memory performance + formula: geometric mean metrics: - name: floatmem description: > @@ -123,6 +125,7 @@ sections: # split based on different application (A = m*B + C). - name: arithmetic description: arithmetic computing speed + formula: geometric mean metrics: - name: integer description: > diff --git a/resources/ansible_roles/ceph-disk-info/tasks/main.yml b/resources/ansible_roles/ceph-disk-info/tasks/main.yml index 3186e3dd..6bb88e69 100644 --- a/resources/ansible_roles/ceph-disk-info/tasks/main.yml +++ b/resources/ansible_roles/ceph-disk-info/tasks/main.yml @@ -8,8 +8,9 @@ ############################################################################## -- name: get ceph disk information - shell: ceph-disk list +- name: get ceph disk information + become: true + command: ceph-disk list register: ceph_disk_out - name: saving output to log @@ -19,10 +20,11 @@ delegate_to: localhost - name: get ceph journal information + become: true shell: lsblk -d -o name,rota |awk '{if ($2 == 0) printf "%s SSD \n",$1 }' register: ceph_journal_out -- name: append +- name: append lineinfile: dest: "{{ logfile }}" line: "{{ ceph_journal_out.stdout }}" @@ -30,6 +32,7 @@ delegate_to: localhost - name: get ceph report + become: true command: ceph report register: ceph_report_out diff --git a/resources/ansible_roles/qtip-generator/files/compute/templates/ssh.cfg b/resources/ansible_roles/qtip-generator/files/compute/templates/ssh.cfg index f0e33fd4..1b7c0183 100644 --- a/resources/ansible_roles/qtip-generator/files/compute/templates/ssh.cfg +++ b/resources/ansible_roles/qtip-generator/files/compute/templates/ssh.cfg @@ -9,7 +9,7 @@ {% for (name, host) in hosts_meta.items() %} Host {{ name }} HostName {{ host.ansible_ssh_host }} - User root + User {{ host.ansible_user }} ProxyCommand ssh -o 'ForwardAgent yes' {{ installer_host }} 'ssh-add && nc %h %p' {% endfor %} diff --git a/tests/ci/periodic.sh b/tests/ci/periodic.sh new file mode 100755 index 00000000..4235cdee --- /dev/null +++ b/tests/ci/periodic.sh @@ -0,0 +1,77 @@ +#! /bin/bash +############################################################################## +# Copyright (c) 2017 ZTE and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +set -e + + +case $INSTALLER_TYPE in + apex) + INSTALLER_IP=`sudo virsh domifaddr undercloud | grep ipv4 | awk '{print $4}' | cut -d/ -f1` + ;; +esac + +cat << EOF > $WORKSPACE/env_file +INSTALLER_TYPE=$INSTALLER_TYPE +INSTALLER_IP=$INSTALLER_IP +POD_NAME=$NODE_NAME +SCENARIO=$DEPLOY_SCENARIO +TESTAPI_URL=$TESTAPI_URL +EOF + +echo "--------------------------------------------------------" +cat $WORKSPACE/env_file +echo "--------------------------------------------------------" + +# Remove previous running containers if exist +if [[ ! -z $(docker ps -a | grep "opnfv/qtip:$DOCKER_TAG") ]]; then + echo "Removing existing opnfv/qtip containers..." + # workaround: sometimes it throws an error when stopping qtip container. + # To make sure ci job unblocked, remove qtip container by force without stopping it. + docker rm -f $(docker ps -a | grep "opnfv/qtip:$DOCKER_TAG" | awk '{print $1}') +fi + +# Remove existing images if exist +if [[ $(docker images opnfv/qtip:${DOCKER_TAG} | wc -l) -gt 1 ]]; then + echo "Removing docker image opnfv/qtip:$DOCKER_TAG..." + docker rmi -f opnfv/qtip:$DOCKER_TAG +fi + +echo "Qtip: Pulling docker image: opnfv/qtip:${DOCKER_TAG}" +docker pull opnfv/qtip:$DOCKER_TAG >/dev/null + +envs="--env-file $WORKSPACE/env_file" +docker_volume="-v /var/run/docker.sock:/var/run/docker.sock" + +cmd="docker run -id ${envs} ${docker_volume} opnfv/qtip:${DOCKER_TAG} /bin/bash" +echo "Qtip: Running docker command: ${cmd}" +${cmd} + +container_id=$(docker ps | grep "opnfv/qtip:${DOCKER_TAG}" | awk '{print $1}' | head -1) +if [ $(docker ps | grep 'opnfv/qtip' | wc -l) == 0 ]; then + echo "The container opnfv/qtip with ID=${container_id} has not been properly started. Exiting..." + exit 1 +fi + +echo "The container ID is: ${container_id}" +QTIP_REPO=/home/opnfv/repos/qtip + +if [[ "$INSTALLER_TYPE" == "apex" ]];then + if [ -f /root/.ssh/id_rsa ]; then + sudo chmod 600 /root/.ssh/id_rsa + sudo docker cp /root/.ssh/id_rsa ${container_id}:/root/.ssh/ + fi +fi + +case $TEST_SUITE in + compute) + docker exec -t ${container_id} bash -c "bash ${QTIP_REPO}/tests/ci/run_compute_qpi.sh" + ;; +esac + +echo "Qtip done!" +exit 0
\ No newline at end of file diff --git a/tests/ci/run_ci.sh b/tests/ci/run_compute_qpi.sh index 02dfca09..09762b15 100644 --- a/tests/ci/run_ci.sh +++ b/tests/ci/run_compute_qpi.sh @@ -50,21 +50,21 @@ pod_name=${pod_name:-$POD_NAME} scenario=${scenario:-$SCENARIO} testapi_url=${testapi_url:-$TESTAPI_URL} +# we currently support ericsson, intel, lf and zte labs +if [[ ! "$installer_type" =~ (fuel|apex) ]]; then + echo "Unsupported/unidentified installer $installer_type. Cannot continue!" + exit 1 +fi + sshoptions="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" verify_connectivity ${installer_ip} -ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa -q - -# ssh-copy-id publickey to installer case "$installer_type" in fuel) + ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa -q sshpass -p r00tme ssh-copy-id $sshoptions ${installer_ip} ;; - *) - echo "Unkown installer $installer_type specified" - exit 1 - ;; esac cd /home/opnfv @@ -89,11 +89,7 @@ case "$installer_type" in publickey=$(sed -r 's/\//\\\//g' /root/.ssh/id_rsa.pub) ssh $sshoptions root@${installer_ip} "sed -i '/$publickey/d' /root/.ssh/authorized_keys" ;; - *) - echo "Not support $installer_type." - exit 1 - ;; esac -echo "Qtip done!" +echo "Compute QPI done!" exit 0
\ No newline at end of file diff --git a/tests/unit/ansible_library/modules/apex_generate_inventory_test.py b/tests/unit/ansible_library/modules/apex_generate_inventory_test.py index 7df9d35c..668a0155 100644 --- a/tests/unit/ansible_library/modules/apex_generate_inventory_test.py +++ b/tests/unit/ansible_library/modules/apex_generate_inventory_test.py @@ -23,8 +23,8 @@ def test_generate_inventory(data_root): u'compute': [u'192.0.2.5', u'192.0.2.6'], u'control': [u'192.0.2.7', u'192.0.2.8', u'192.0.2.9']} assert dict(inventory['hosts_meta']) == { - u'192.0.2.5': {'ansible_ssh_host': u'192.0.2.5'}, - u'192.0.2.6': {'ansible_ssh_host': u'192.0.2.6'}, - u'192.0.2.7': {'ansible_ssh_host': u'192.0.2.7'}, - u'192.0.2.8': {'ansible_ssh_host': u'192.0.2.8'}, - u'192.0.2.9': {'ansible_ssh_host': u'192.0.2.9'}} + u'192.0.2.5': {'ansible_ssh_host': u'192.0.2.5', 'ansible_user': 'heat-admin'}, + u'192.0.2.6': {'ansible_ssh_host': u'192.0.2.6', 'ansible_user': 'heat-admin'}, + u'192.0.2.7': {'ansible_ssh_host': u'192.0.2.7', 'ansible_user': 'heat-admin'}, + u'192.0.2.8': {'ansible_ssh_host': u'192.0.2.8', 'ansible_user': 'heat-admin'}, + u'192.0.2.9': {'ansible_ssh_host': u'192.0.2.9', 'ansible_user': 'heat-admin'}} diff --git a/tests/unit/ansible_library/modules/fuel_test.py b/tests/unit/ansible_library/modules/fuel_test.py index e004fc17..9ec1806f 100644 --- a/tests/unit/ansible_library/modules/fuel_test.py +++ b/tests/unit/ansible_library/modules/fuel_test.py @@ -34,21 +34,21 @@ def test_generate_inventory(data_root): u'node-27'], u'mongo': [u'node-24']} assert dict(inventory['hosts_meta']) == { - u'node-23': {'ansible_ssh_host': u'10.20.11.10', 'cluster': 4, 'ip': u'10.20.11.10', + u'node-23': {'ansible_ssh_host': u'10.20.11.10', 'ansible_user': 'root', 'cluster': 4, 'ip': u'10.20.11.10', 'mac': u'74:4a:a4:01:71:61', 'name': u'Untitled (71:61)', 'online': True, 'os_platform': u'ubuntu', 'status': u'ready'}, - u'node-24': {'ansible_ssh_host': u'10.20.11.11', 'cluster': 4, 'ip': u'10.20.11.11', + u'node-24': {'ansible_ssh_host': u'10.20.11.11', 'ansible_user': 'root', 'cluster': 4, 'ip': u'10.20.11.11', 'mac': u'74:4a:a4:01:73:50', 'name': u'Untitled (73:50)', 'online': True, 'os_platform': u'ubuntu', 'status': u'ready'}, - u'node-25': {'ansible_ssh_host': u'10.20.11.12', 'cluster': 4, 'ip': u'10.20.11.12', + u'node-25': {'ansible_ssh_host': u'10.20.11.12', 'ansible_user': 'root', 'cluster': 4, 'ip': u'10.20.11.12', 'mac': u'74:4a:a4:00:d8:76', 'name': u'Untitled (d8:76)', 'online': True, 'os_platform': u'ubuntu', 'status': u'ready'}, - u'node-26': {'ansible_ssh_host': u'10.20.11.15', 'cluster': 4, 'ip': u'10.20.11.15', + u'node-26': {'ansible_ssh_host': u'10.20.11.15', 'ansible_user': 'root', 'cluster': 4, 'ip': u'10.20.11.15', 'mac': u'74:4a:a4:01:61:ae', 'name': u'Untitled (61:ae)', 'online': True, 'os_platform': u'ubuntu', 'status': u'ready'}, - u'node-27': {'ansible_ssh_host': u'10.20.11.13', 'cluster': 4, 'ip': u'10.20.11.13', + u'node-27': {'ansible_ssh_host': u'10.20.11.13', 'ansible_user': 'root', 'cluster': 4, 'ip': u'10.20.11.13', 'mac': u'74:4a:a4:01:82:c0', 'name': u'Untitled (82:c0)', 'online': True, 'os_platform': u'ubuntu', 'status': u'ready'}, - u'node-28': {'ansible_ssh_host': u'10.20.11.14', 'cluster': 4, 'ip': u'10.20.11.14', + u'node-28': {'ansible_ssh_host': u'10.20.11.14', 'ansible_user': 'root', 'cluster': 4, 'ip': u'10.20.11.14', 'mac': u'74:4a:a4:01:74:63', 'name': u'Untitled (74:63)', 'online': True, 'os_platform': u'ubuntu', 'status': u'ready'}} diff --git a/tests/unit/cli/cmd_qpi_test.py b/tests/unit/cli/cmd_qpi_test.py index e7823c9b..ba5f814e 100644 --- a/tests/unit/cli/cmd_qpi_test.py +++ b/tests/unit/cli/cmd_qpi_test.py @@ -20,24 +20,14 @@ def runner(): def test_list(runner): result = runner.invoke(cli, ['qpi', 'list']) - assert 'QPIs' and 'compute' in result.output - - -def test_run(runner): - result = runner.invoke(cli, ['qpi', 'run', 'fake-qpi']) - assert result.output == '' - - result = runner.invoke(cli, ['qpi', 'run']) - assert 'Missing argument "name".' in result.output + assert 'QPI' in result.output + assert 'compute' in result.output + assert 'Description' in result.output + assert 'QTIP Performance Index of compute' in result.output def test_show(runner): result = runner.invoke(cli, ['qpi', 'show', 'compute']) - assert 'Name: compute' in result.output - assert 'Description: sample performance index of computing' in result.output - - result = runner.invoke(cli, ['qpi', 'show']) - assert 'Missing argument "name".' in result.output - - result = runner.invoke(cli, ['qpi', 'show', 'xyz']) - assert "ERROR: qpi spec: xyz not found" in result.output + assert 'QPI' in result.output + assert 'Description' in result.output + assert 'Formula' in result.output |