diff options
Diffstat (limited to 'jjb/dovetail')
| -rwxr-xr-x | jjb/dovetail/dovetail-artifacts-upload.sh | 71 | ||||
| -rw-r--r-- | jjb/dovetail/dovetail-artifacts-upload.yml | 130 | ||||
| -rw-r--r-- | jjb/dovetail/dovetail-ci-jobs.yml | 55 | ||||
| -rwxr-xr-x | jjb/dovetail/dovetail-cleanup.sh | 7 | ||||
| -rw-r--r-- | jjb/dovetail/dovetail-project-jobs.yml | 36 | ||||
| -rwxr-xr-x | jjb/dovetail/dovetail-run.sh | 41 |
6 files changed, 297 insertions, 43 deletions
diff --git a/jjb/dovetail/dovetail-artifacts-upload.sh b/jjb/dovetail/dovetail-artifacts-upload.sh new file mode 100755 index 000000000..94e4129fc --- /dev/null +++ b/jjb/dovetail/dovetail-artifacts-upload.sh @@ -0,0 +1,71 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +set -o pipefail + +echo "dovetail: pull and save the images" + +[[ -d ${CACHE_DIR} ]] || mkdir -p ${CACHE_DIR} + +cd ${CACHE_DIR} +sudo docker pull ${DOCKER_REPO_NAME}:${DOCKER_TAG} +sudo docker save -o ${STORE_FILE_NAME} ${DOCKER_REPO_NAME}:${DOCKER_TAG} + +importkey () { +# clone releng repository +echo "Cloning releng repository..." +[ -d releng ] && rm -rf releng +git clone https://gerrit.opnfv.org/gerrit/releng $WORKSPACE/releng/ &> /dev/null +#this is where we import the siging key +if [ -f $WORKSPACE/releng/utils/gpg_import_key.sh ]; then + source $WORKSPACE/releng/utils/gpg_import_key.sh +fi +} + +sign () { +gpg2 -vvv --batch --yes --no-tty \ + --default-key opnfv-helpdesk@rt.linuxfoundation.org \ + --passphrase besteffort \ + --detach-sig ${CACHE_DIR}/${STORE_FILE_NAME} + +gsutil cp ${CACHE_DIR}/${STORE_FILE_NAME}.sig ${STORE_URL}/${STORE_FILE_NAME}.sig +echo "signature Upload Complete!" +} + +upload () { +# log info to console +echo "Uploading to artifact. This could take some time..." +echo + +cd $WORKSPACE +# upload artifact and additional files to google storage +gsutil cp ${CACHE_DIR}/${STORE_FILE_NAME} ${STORE_URL}/${STORE_FILE_NAME} + +gsutil -m setmeta \ + -h "Cache-Control:private, max-age=0, no-transform" \ + ${STORE_URL}/${STORE_FILE_NAME} > /dev/null 2>&1 + +# disabled errexit due to gsutil setmeta complaints +# BadRequestException: 400 Invalid argument +# check if we uploaded the file successfully to see if things are fine +gsutil ls ${STORE_URL}/${STORE_FILE_NAME} > /dev/null 2>&1 +if [[ $? -ne 0 ]]; then + echo "Problem while uploading artifact!" + exit 1 +fi + +echo "dovetail: uploading Done!" +echo +echo "--------------------------------------------------------" +echo +} + +importkey +sign +upload diff --git a/jjb/dovetail/dovetail-artifacts-upload.yml b/jjb/dovetail/dovetail-artifacts-upload.yml new file mode 100644 index 000000000..dc2ae5aa2 --- /dev/null +++ b/jjb/dovetail/dovetail-artifacts-upload.yml @@ -0,0 +1,130 @@ +############################################ +# dovetail upload artifacts job +############################################ +- project: + name: dovetail-artifacts-upload + + project: 'dovetail' + + jobs: + - 'dovetail-{image}-artifacts-upload-{stream}' + + stream: + - master: + branch: '{stream}' + gs-pathname: '' + disabled: false + + image: + - 'dovetail' + - 'functest' + - 'yardstick' + +############################################# +# job template +############################################# + +- job-template: + name: 'dovetail-{image}-artifacts-upload-{stream}' + + + disabled: '{obj:disabled}' + + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 1 + max-per-node: 1 + option: 'project' + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - 'opnfv-build-ubuntu-defaults' + - dovetail-parameter: + gs-pathname: '{gs-pathname}' + image: '{image}' + branch: '{branch}' + + scm: + - git-scm + + builders: + - 'dovetail-builder-artifacts-upload' + - 'dovetail-workspace-cleanup' + +#################### +# parameter macros +#################### +- parameter: + name: dovetail-parameter + parameters: + - string: + name: CACHE_DIR + default: $WORKSPACE/cache{gs-pathname} + description: "the cache to store packages downloaded" + - string: + name: STORE_URL + default: gs://artifacts.opnfv.org/dovetail{gs-pathname} + description: "LF artifacts url for storage of dovetail packages" + - string: + name: DOCKER_REPO_NAME + default: opnfv/{image} + description: "docker repo name" + - string: + name: DOCKER_TAG + default: latest + description: "docker image tag of which will be uploaded to artifacts" + - string: + name: STORE_FILE_NAME + default: image_{image}_{branch}_$BUILD_ID.docker + description: "stored file name" + +#################################### +#builders for dovetail project +#################################### +- builder: + name: dovetail-builder-artifacts-upload + builders: + - shell: + !include-raw: ./dovetail-artifacts-upload.sh + +- builder: + name: dovetail-workspace-cleanup + builders: + - shell: | + #!/bin/bash + set -o errexit + + echo "Dovetail: cleanup cache used for storage downloaded packages" + + /bin/rm -rf $CACHE_DIR + + # Remove previous running containers if exist + if [[ -n "$(docker ps -a | grep $DOCKER_REPO_NAME)" ]]; then + echo "Removing existing $DOCKER_REPO_NAME containers..." + docker ps -a | grep $DOCKER_REPO_NAME | awk '{print $1}' | xargs docker rm -f + t=60 + # Wait max 60 sec for containers to be removed + while [[ $t -gt 0 ]] && [[ -n "$(docker ps| grep $DOCKER_REPO_NAME)" ]]; do + sleep 1 + let t=t-1 + done + fi + + # Remove existing images if exist + if [[ -n "$(docker images | grep $DOCKER_REPO_NAME)" ]]; then + echo "Docker images to remove:" + docker images | head -1 && docker images | grep $DOCKER_REPO_NAME + image_tags=($(docker images | grep $DOCKER_REPO_NAME | awk '{print $2}')) + for tag in "${image_tags[@]}"; do + if [[ -n "$(docker images|grep $DOCKER_REPO_NAME|grep $tag)" ]]; then + echo "Removing docker image $DOCKER_REPO_NAME:$tag..." + docker rmi -f $DOCKER_REPO_NAME:$tag + fi + done + fi diff --git a/jjb/dovetail/dovetail-ci-jobs.yml b/jjb/dovetail/dovetail-ci-jobs.yml index 29212005d..2b6ab7621 100644 --- a/jjb/dovetail/dovetail-ci-jobs.yml +++ b/jjb/dovetail/dovetail-ci-jobs.yml @@ -102,6 +102,27 @@ SUT: apex auto-trigger-name: 'daily-trigger-disabled' <<: *colorado +#armband CI PODs + - armband-baremetal: + slave-label: armband-baremetal + SUT: fuel + auto-trigger-name: 'daily-trigger-disabled' + <<: *master + - armband-virtual: + slave-label: armband-virtual + SUT: fuel + auto-trigger-name: 'daily-trigger-disabled' + <<: *master + - armband-baremetal: + slave-label: armband-baremetal + SUT: fuel + auto-trigger-name: 'daily-trigger-disabled' + <<: *colorado + - armband-virtual: + slave-label: armband-virtual + SUT: fuel + auto-trigger-name: 'daily-trigger-disabled' + <<: *colorado #-------------------------------- # None-CI PODs #-------------------------------- @@ -110,9 +131,21 @@ SUT: compass auto-trigger-name: 'daily-trigger-disabled' <<: *master + - arm-pod2: + slave-label: '{pod}' + SUT: fuel + auto-trigger-name: 'daily-trigger-disabled' + <<: *master + - arm-pod3: + slave-label: '{pod}' + SUT: fuel + auto-trigger-name: 'daily-trigger-disabled' + <<: *master #-------------------------------- testsuite: - - 'basic' + - 'debug' + - 'proposed_tests' + - 'compliance_set' jobs: - 'dovetail-{SUT}-{pod}-{testsuite}-{stream}' @@ -146,6 +179,8 @@ parameters: - project-parameter: project: '{project}' + - gerrit-parameter: + branch: '{branch}' - '{SUT}-defaults' - '{slave-label}-defaults' - string: @@ -159,18 +194,23 @@ name: CI_DEBUG default: 'true' description: "Show debug output information" + - string: + name: TESTSUITE + default: '{testsuite}' + description: "dovetail testsuite to run" + - string: + name: DOVETAIL_REPO_DIR + default: "/home/opnfv/dovetail" + description: "Directory where the dovetail repository is cloned" scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{dovetail-branch}' + - git-scm builders: - description-setter: description: "POD: $NODE_NAME" - 'dovetail-cleanup' - - 'dovetail-{testsuite}' + - 'dovetail-run' publishers: - archive: @@ -182,11 +222,12 @@ # builder macros ######################## - builder: - name: dovetail-basic + name: dovetail-run builders: - shell: !include-raw: ./dovetail-run.sh + - builder: name: dovetail-fetch-os-creds builders: diff --git a/jjb/dovetail/dovetail-cleanup.sh b/jjb/dovetail/dovetail-cleanup.sh index 297222bb3..f215278db 100755 --- a/jjb/dovetail/dovetail-cleanup.sh +++ b/jjb/dovetail/dovetail-cleanup.sh @@ -1,20 +1,21 @@ #!/bin/bash + [[ $CI_DEBUG == true ]] && redirect="/dev/stdout" || redirect="/dev/null" echo "Cleaning up docker containers/images..." # Remove previous running containers if exist if [[ ! -z $(docker ps -a | grep opnfv/dovetail) ]]; then echo "Removing existing opnfv/dovetail containers..." - docker ps -a | grep opnfv/dovetail | awk '{print $1}' | xargs docker rm -f >$redirect + docker ps -a | grep opnfv/dovetail | awk '{print $1}' | xargs docker rm -f >${redirect} fi # Remove existing images if exist if [[ ! -z $(docker images | grep opnfv/dovetail) ]]; then echo "Docker images to remove:" - docker images | head -1 && docker images | grep opnfv/dovetail + docker images | head -1 && docker images | grep opnfv/dovetail >${redirect} image_tags=($(docker images | grep opnfv/dovetail | awk '{print $2}')) for tag in "${image_tags[@]}"; do echo "Removing docker image opnfv/dovetail:$tag..." - docker rmi opnfv/dovetail:$tag >$redirect + docker rmi opnfv/dovetail:$tag >${redirect} done fi diff --git a/jjb/dovetail/dovetail-project-jobs.yml b/jjb/dovetail/dovetail-project-jobs.yml index 41fd8cdcb..904841396 100644 --- a/jjb/dovetail/dovetail-project-jobs.yml +++ b/jjb/dovetail/dovetail-project-jobs.yml @@ -33,13 +33,11 @@ - 'opnfv-build-ubuntu-defaults' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' @@ -72,13 +70,11 @@ - 'opnfv-build-ubuntu-defaults' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - choosing-strategy: 'default' + - git-scm triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - change-merged-event - comment-added-contains-event: @@ -97,25 +93,21 @@ #builders for dovetail project ############################### - builder: - name: dovetail-unit-tests + name: dovetail-hello-world builders: - shell: | #!/bin/bash set -o errexit - set -o pipefail - echo "Running unit tests..." - cd $WORKSPACE - virtualenv $WORKSPACE/dovetail_venv - source $WORKSPACE/dovetail_venv/bin/activate + echo "hello world" - #packages installation - easy_install -U setuptools - easy_install -U pip - pip install -r unittests/requirements.txt - pip install -e . - #unit tests - /bin/bash $WORKSPACE/unittests/unittest.sh +- builder: + name: dovetail-unit-tests + builders: + - shell: | + #!/bin/bash + set -o errexit + set -o pipefail - deactivate + tox diff --git a/jjb/dovetail/dovetail-run.sh b/jjb/dovetail/dovetail-run.sh index 098b7db0c..4082c34fe 100755 --- a/jjb/dovetail/dovetail-run.sh +++ b/jjb/dovetail/dovetail-run.sh @@ -22,7 +22,7 @@ if [[ ${INSTALLER_TYPE} == 'apex' ]]; then elif [[ ${INSTALLER_TYPE} == 'joid' ]]; then # If production lab then creds may be retrieved dynamically # creds are on the jumphost, always in the same folder - labconfig="-v $LAB_CONFIG/admin-openrc:/home/opnfv/openrc" + labconfig="-v $LAB_CONFIG/admin-openrc:/home/opnfv/functest/conf/openstack.creds" # If dev lab, credentials may not be the default ones, just provide a path to put them into docker # replace the default one by the customized one provided by jenkins config fi @@ -32,26 +32,45 @@ if ! sudo iptables -C FORWARD -j RETURN 2> ${redirect} || ! sudo iptables -L FOR sudo iptables -I FORWARD -j RETURN fi -opts="--privileged=true --rm" +opts="--privileged=true -id" envs="-e CI_DEBUG=${CI_DEBUG} \ -e INSTALLER_TYPE=${INSTALLER_TYPE} \ -e INSTALLER_IP=${INSTALLER_IP} \ -e DEPLOY_SCENARIO=${DEPLOY_SCENARIO} \ - -e DEPLOY_TYPE=${DEPLOY_TYPE} \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v /home/opnfv/dovetail/results:/home/opnfv/dovetail/results" + -e DEPLOY_TYPE=${DEPLOY_TYPE}" +results_envs="-v /var/run/docker.sock:/var/run/docker.sock \ + -v /home/opnfv/dovetail/results:/home/opnfv/dovetail/results" # Pull the image with correct tag echo "Dovetail: Pulling image opnfv/dovetail:${DOCKER_TAG}" docker pull opnfv/dovetail:$DOCKER_TAG >$redirect -# Run docker -echo "Dovetail: docker running..." -sudo docker run ${opts} ${envs} ${labconfig} ${sshkey} opnfv/dovetail:${DOCKER_TAG} \ -"/home/opnfv/dovetail/dovetail/run.py" +cmd="sudo docker run ${opts} ${envs} ${results_envs} ${labconfig} ${sshkey} \ + opnfv/dovetail:${DOCKER_TAG} /bin/bash" +echo "Dovetail: running docker run command: ${cmd}" +${cmd} >${redirect} +sleep 5 +container_id=$(docker ps | grep "opnfv/dovetail:${DOCKER_TAG}" | awk '{print $1}' | head -1) +echo "Container ID=${container_id}" +if [ -z ${container_id} ]; then + echo "Cannot find opnfv/dovetail container ID ${container_id}. Please check if it is existing." + docker ps -a + exit 1 +fi +echo "COntainer Start: docker start ${container_id}" +docker start ${container_id} +sleep 5 +docker ps >${redirect} +if [ $(docker ps | grep "opnfv/dovetail:${DOCKER_TAG}" | wc -l) == 0 ]; then + echo "The container opnfv/dovetail with ID=${container_id} has not been properly started. Exiting..." + exit 1 +fi + +exec_cmd="python ${DOVETAIL_REPO_DIR}/dovetail/run.py --testsuite ${TESTSUITE} -d true" +echo "Container exec command: ${exec_cmd}" +docker exec ${container_id} ${exec_cmd} -echo "Dovetail: store results..." -sudo cp -r /home/opnfv/dovetail/results ./ +sudo cp -r ${DOVETAIL_REPO_DIR}/results ./ #To make sure the file owner is jenkins, for the copied results files in the above line #if not, there will be error when next time to wipe workspace sudo chown -R jenkins:jenkins ${WORKSPACE}/results |