From 6ecd5a9aab2d2d04c9f913fa9d9ee84f945ed788 Mon Sep 17 00:00:00 2001 From: Yujun Zhang Date: Sun, 29 Jan 2017 19:49:06 +0800 Subject: Refactor qtip jobs - rename jobs to verify and validate - add job for validate-setup - move all scripts to helpers folder - use auto variables to merge job templates Change-Id: Ie1199e35e48b6a95861d9b7281fa7bdcc94212d9 Signed-off-by: Yujun Zhang --- jjb/qtip/helpers/cleanup-deploy.sh | 26 +++++++ jjb/qtip/helpers/validate-deploy.sh | 38 ++++++++++ jjb/qtip/helpers/validate-setup.sh | 24 +++++++ jjb/qtip/qtip-ci-jobs.yml | 100 -------------------------- jjb/qtip/qtip-cleanup.sh | 26 ------- jjb/qtip/qtip-daily-ci.sh | 38 ---------- jjb/qtip/qtip-project-jobs.yml | 70 ------------------- jjb/qtip/qtip-validate-jobs.yml | 136 ++++++++++++++++++++++++++++++++++++ jjb/qtip/qtip-verify-jobs.yml | 71 +++++++++++++++++++ 9 files changed, 295 insertions(+), 234 deletions(-) create mode 100644 jjb/qtip/helpers/cleanup-deploy.sh create mode 100644 jjb/qtip/helpers/validate-deploy.sh create mode 100644 jjb/qtip/helpers/validate-setup.sh delete mode 100644 jjb/qtip/qtip-ci-jobs.yml delete mode 100644 jjb/qtip/qtip-cleanup.sh delete mode 100644 jjb/qtip/qtip-daily-ci.sh delete mode 100644 jjb/qtip/qtip-project-jobs.yml create mode 100644 jjb/qtip/qtip-validate-jobs.yml create mode 100644 jjb/qtip/qtip-verify-jobs.yml (limited to 'jjb/qtip') diff --git a/jjb/qtip/helpers/cleanup-deploy.sh b/jjb/qtip/helpers/cleanup-deploy.sh new file mode 100644 index 000000000..95babb318 --- /dev/null +++ b/jjb/qtip/helpers/cleanup-deploy.sh @@ -0,0 +1,26 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2016 ZTE and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +# Remove previous running containers if exist +if [[ ! -z $(docker ps -a | grep opnfv/qtip) ]]; then + echo "Removing existing opnfv/qtip containers..." + # workaround: sometimes it throws an error when stopping qtip container. + # To make sure ci job unblocked, remove qtip container by force without stopping it. + docker rm -f $(docker ps -a | grep opnfv/qtip | awk '{print $1}') +fi + +# Remove existing images if exist +if [[ ! -z $(docker images | grep opnfv/qtip) ]]; then + echo "Docker images to remove:" + docker images | head -1 && docker images | grep opnfv/qtip + image_tags=($(docker images | grep opnfv/qtip | awk '{print $2}')) + for tag in "${image_tags[@]}"; do + echo "Removing docker image opnfv/qtip:$tag..." + docker rmi opnfv/qtip:$tag + done +fi diff --git a/jjb/qtip/helpers/validate-deploy.sh b/jjb/qtip/helpers/validate-deploy.sh new file mode 100644 index 000000000..4fdc04345 --- /dev/null +++ b/jjb/qtip/helpers/validate-deploy.sh @@ -0,0 +1,38 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2016 ZTE and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +set -e + +envs="INSTALLER_TYPE=${INSTALLER_TYPE} -e INSTALLER_IP=${INSTALLER_IP} -e NODE_NAME=${NODE_NAME}" +suite="TEST_CASE=all" +dir_imgstore="${HOME}/imgstore" +img_volume="${dir_imgstore}:/home/opnfv/imgstore" + +echo "Qtip: Pulling docker image: opnfv/qtip:${DOCKER_TAG}" +docker pull opnfv/qtip:$DOCKER_TAG + +cmd=" docker run -id -e $envs -e $suite -v ${img_volume} opnfv/qtip:${DOCKER_TAG} /bin/bash" +echo "Qtip: Running docker command: ${cmd}" +${cmd} + +container_id=$(docker ps | grep "opnfv/qtip:${DOCKER_TAG}" | awk '{print $1}' | head -1) +if [ $(docker ps | grep 'opnfv/qtip' | wc -l) == 0 ]; then + echo "The container opnfv/qtip with ID=${container_id} has not been properly started. Exiting..." + exit 1 +else + echo "The container ID is: ${container_id}" + QTIP_REPO=/home/opnfv/repos/qtip + + echo "Run Qtip test" + docker exec -t ${container_id} $QTIP_REPO/docker/run_qtip.sh + + echo "Pushing available results to DB" + docker exec -t ${container_id} $QTIP_REPO/docker/push_db.sh +fi + +echo "Qtip done!" diff --git a/jjb/qtip/helpers/validate-setup.sh b/jjb/qtip/helpers/validate-setup.sh new file mode 100644 index 000000000..8d84e120c --- /dev/null +++ b/jjb/qtip/helpers/validate-setup.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +############################################################################## +# Copyright (c) 2017 ZTE and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -e + +# setup virtualenv +sudo pip install -u virtualenv virtualenvwrapper +export WORKON_HOME=$HOME/.virtualenvs +source /usr/local/bin/virtualenvwrapper.sh +mkvirtualenv qtip +workon qtip + +# setup qtip +sudo pip install $HOME/repos/qtip + +# testing +qtip --version +qtip --help diff --git a/jjb/qtip/qtip-ci-jobs.yml b/jjb/qtip/qtip-ci-jobs.yml deleted file mode 100644 index 38f9955ec..000000000 --- a/jjb/qtip/qtip-ci-jobs.yml +++ /dev/null @@ -1,100 +0,0 @@ -#################################### -# job configuration for qtip -#################################### -- project: - name: qtip - - project: 'qtip' - -#-------------------------------- -# BRANCH ANCHORS -#-------------------------------- - master: &master - stream: master - branch: '{stream}' - gs-pathname: '' - docker-tag: 'latest' -#-------------------------------- -# POD, INSTALLER, AND BRANCH MAPPING -#-------------------------------- -# master -#-------------------------------- - pod: - - zte-pod2: - installer: fuel - auto-trigger-name: 'qtip-daily-zte-pod2-trigger' - <<: *master - - zte-pod3: - installer: fuel - auto-trigger-name: 'qtip-daily-zte-pod3-trigger' - <<: *master - -#-------------------------------- - jobs: - - 'qtip-{installer}-{pod}-daily-{stream}' - -################################ -# job templates -################################ -- job-template: - name: 'qtip-{installer}-{pod}-daily-{stream}' - - disabled: false - - parameters: - - project-parameter: - project: '{project}' - branch: '{branch}' - - '{installer}-defaults' - - '{pod}-defaults' - - string: - name: DEPLOY_SCENARIO - default: 'os-nosdn-nofeature-ha' - - string: - name: DOCKER_TAG - default: '{docker-tag}' - description: 'Tag to pull docker image' - - scm: - - git-scm - - triggers: - - '{auto-trigger-name}' - - builders: - - description-setter: - description: "POD: $NODE_NAME" - - 'qtip-cleanup' - - 'qtip-daily-ci' - - publishers: - - email: - recipients: wu.zhihui1@zte.com.cn, zhang.yujunz@zte.com.cn - -########################### -#biuilder macros -########################### -- builder: - name: qtip-daily-ci - builders: - - shell: - !include-raw: ./qtip-daily-ci.sh - -- builder: - name: qtip-cleanup - builders: - - shell: - !include-raw: ./qtip-cleanup.sh - -################# -#trigger macros -################# -- trigger: - name: 'qtip-daily-zte-pod2-trigger' - triggers: - - timed: '0 7 * * *' - -- trigger: - name: 'qtip-daily-zte-pod3-trigger' - triggers: - - timed: '0 1 * * *' diff --git a/jjb/qtip/qtip-cleanup.sh b/jjb/qtip/qtip-cleanup.sh deleted file mode 100644 index 95babb318..000000000 --- a/jjb/qtip/qtip-cleanup.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -############################################################################## -# Copyright (c) 2016 ZTE and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -# Remove previous running containers if exist -if [[ ! -z $(docker ps -a | grep opnfv/qtip) ]]; then - echo "Removing existing opnfv/qtip containers..." - # workaround: sometimes it throws an error when stopping qtip container. - # To make sure ci job unblocked, remove qtip container by force without stopping it. - docker rm -f $(docker ps -a | grep opnfv/qtip | awk '{print $1}') -fi - -# Remove existing images if exist -if [[ ! -z $(docker images | grep opnfv/qtip) ]]; then - echo "Docker images to remove:" - docker images | head -1 && docker images | grep opnfv/qtip - image_tags=($(docker images | grep opnfv/qtip | awk '{print $2}')) - for tag in "${image_tags[@]}"; do - echo "Removing docker image opnfv/qtip:$tag..." - docker rmi opnfv/qtip:$tag - done -fi diff --git a/jjb/qtip/qtip-daily-ci.sh b/jjb/qtip/qtip-daily-ci.sh deleted file mode 100644 index 4fdc04345..000000000 --- a/jjb/qtip/qtip-daily-ci.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -############################################################################## -# Copyright (c) 2016 ZTE and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -set -e - -envs="INSTALLER_TYPE=${INSTALLER_TYPE} -e INSTALLER_IP=${INSTALLER_IP} -e NODE_NAME=${NODE_NAME}" -suite="TEST_CASE=all" -dir_imgstore="${HOME}/imgstore" -img_volume="${dir_imgstore}:/home/opnfv/imgstore" - -echo "Qtip: Pulling docker image: opnfv/qtip:${DOCKER_TAG}" -docker pull opnfv/qtip:$DOCKER_TAG - -cmd=" docker run -id -e $envs -e $suite -v ${img_volume} opnfv/qtip:${DOCKER_TAG} /bin/bash" -echo "Qtip: Running docker command: ${cmd}" -${cmd} - -container_id=$(docker ps | grep "opnfv/qtip:${DOCKER_TAG}" | awk '{print $1}' | head -1) -if [ $(docker ps | grep 'opnfv/qtip' | wc -l) == 0 ]; then - echo "The container opnfv/qtip with ID=${container_id} has not been properly started. Exiting..." - exit 1 -else - echo "The container ID is: ${container_id}" - QTIP_REPO=/home/opnfv/repos/qtip - - echo "Run Qtip test" - docker exec -t ${container_id} $QTIP_REPO/docker/run_qtip.sh - - echo "Pushing available results to DB" - docker exec -t ${container_id} $QTIP_REPO/docker/push_db.sh -fi - -echo "Qtip done!" diff --git a/jjb/qtip/qtip-project-jobs.yml b/jjb/qtip/qtip-project-jobs.yml deleted file mode 100644 index 00455f8b3..000000000 --- a/jjb/qtip/qtip-project-jobs.yml +++ /dev/null @@ -1,70 +0,0 @@ -- project: - name: qtip-project-jobs - - project: 'qtip' - - jobs: - - 'qtip-verify-{stream}' - - stream: - - master: - branch: '{stream}' - gs-pathname: '' - disabled: false - -################################ -## job templates -################################# -- job-template: - name: 'qtip-verify-{stream}' - - disabled: '{obj:disabled}' - - parameters: - - project-parameter: - project: '{project}' - branch: '{branch}' - - 'opnfv-build-ubuntu-defaults' - - scm: - - git-scm-gerrit - - triggers: - - gerrit: - server-name: 'gerrit.opnfv.org' - trigger-on: - - patchset-created-event: - exclude-drafts: 'false' - exclude-trivial-rebase: 'false' - exclude-no-code-change: 'false' - - draft-published-event - - comment-added-contains-event: - comment-contains-value: 'recheck' - - comment-added-contains-event: - comment-contains-value: 'reverify' - projects: - - project-compare-type: 'ANT' - project-pattern: '{project}' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - forbidden-file-paths: - - compare-type: ANT - pattern: 'docs/**|.gitignore' - - builders: - - qtip-unit-tests-and-docs-build - -################################ -## job builders -################################# -- builder: - name: qtip-unit-tests-and-docs-build - builders: - - shell: | - #!/bin/bash - set -o errexit - set -o pipefail - set -o xtrace - - tox diff --git a/jjb/qtip/qtip-validate-jobs.yml b/jjb/qtip/qtip-validate-jobs.yml new file mode 100644 index 000000000..161acb18c --- /dev/null +++ b/jjb/qtip/qtip-validate-jobs.yml @@ -0,0 +1,136 @@ +####################### +# validate after MERGE +####################### +- project: + name: qtip + project: qtip + +#-------------------------------- +# BRANCH ANCHORS +#-------------------------------- + master: &master + stream: master + branch: '{stream}' + gs-pathname: '' + docker-tag: latest +#-------------------------------- +# JOB VARIABLES +#-------------------------------- + pod: + - zte-pod2: + installer: fuel + <<: *master + - zte-pod3: + installer: fuel + <<: *master + task: + - daily: + auto-builder-name: qtip-validate-deploy + auto-trigger-name: 'qtip-daily-{pod}-trigger' + - validate: + auto-builder-name: qtip-validate-setup + auto-trigger-name: qtip-validate-trigger +#-------------------------------- +# JOB LIST +#-------------------------------- + jobs: + - 'qtip-{task}-{installer}-{pod}-{stream}' + +################################ +# job templates +################################ +- job-template: + name: 'qtip-{task}-{installer}-{pod}-{stream}' + disabled: false + parameters: + - qtip-common-parameters: + project: '{project}' + <<: *master + - '{installer}-defaults' + - '{pod}-defaults' + scm: + - git-scm + triggers: + - '{auto-trigger-name}' + builders: + - qtip-common-builders + - '{auto-builder-name}' + publishers: + - qtip-common-publishers + +################ +# MARCOS +################ + +#--------- +# builder +#--------- + +- builder: + name: qtip-common-builders + builders: + - description-setter: + description: "POD: $NODE_NAME" + +- builder: + name: qtip-validate-deploy + builders: + - shell: + !include-raw: ./helpers/validate-deploy.sh + - shell: + !include-raw: ./helpers/cleanup-deploy.sh + +- builder: + name: qtip-validate-setup + builders: + - shell: + !include-raw: ./helpers/validate-setup.sh + +#----------- +# parameter +#----------- + +- parameter: + name: qtip-common-parameters + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - string: + name: DEPLOY_SCENARIO + default: 'os-nosdn-nofeature-ha' + - string: + name: DOCKER_TAG + default: '{docker-tag}' + description: 'Tag to pull docker image' + +#----------- +# publisher +#----------- + +- publisher: + name: qtip-common-publishers + publishers: + - email: + recipients: wu.zhihui1@zte.com.cn, zhang.yujunz@zte.com.cn + +#--------- +# trigger +#--------- + +- trigger: + name: qtip-daily-zte-pod2-trigger + triggers: + - timed: '0 7 * * *' + +- trigger: + name: qtip-daily-zte-pod3-trigger + triggers: + - timed: '0 1 * * *' + +- trigger: + name: qtip-validate-trigger + triggers: + - gerrit-trigger-patch-merged: + project: '{project}' + branch: '{branch}' diff --git a/jjb/qtip/qtip-verify-jobs.yml b/jjb/qtip/qtip-verify-jobs.yml new file mode 100644 index 000000000..d1fc34d11 --- /dev/null +++ b/jjb/qtip/qtip-verify-jobs.yml @@ -0,0 +1,71 @@ +###################### +# verify before MERGE +###################### + +- project: + name: qtip-verify-jobs + project: qtip + jobs: + - 'qtip-verify-{stream}' + stream: + - master: + branch: '{stream}' + gs-pathname: '' + disabled: false + +################################ +## job templates +################################# +- job-template: + name: 'qtip-verify-{stream}' + + disabled: '{obj:disabled}' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - 'opnfv-build-ubuntu-defaults' + + scm: + - git-scm-gerrit + + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**|.gitignore' + + builders: + - qtip-unit-tests-and-docs-build + +################################ +## job builders +################################# +- builder: + name: qtip-unit-tests-and-docs-build + builders: + - shell: | + #!/bin/bash + set -o errexit + set -o pipefail + set -o xtrace + + tox -- cgit 1.2.3-korg