diff options
Diffstat (limited to 'jjb/releng')
| -rw-r--r-- | jjb/releng/automate.yml | 10 | ||||
| -rw-r--r-- | jjb/releng/docker-deploy.sh | 156 | ||||
| -rwxr-xr-x | jjb/releng/generate-job-list.sh | 70 | ||||
| -rw-r--r-- | jjb/releng/opnfv-docker-arm.yml | 146 | ||||
| -rw-r--r-- | jjb/releng/opnfv-docker.sh | 14 | ||||
| -rw-r--r-- | jjb/releng/opnfv-docker.yml | 149 | ||||
| -rw-r--r-- | jjb/releng/opnfv-repo-archiver.sh | 66 | ||||
| -rw-r--r-- | jjb/releng/opnfv-utils.yml | 50 | ||||
| -rw-r--r-- | jjb/releng/releng-ci-jobs.yml | 23 | ||||
| -rwxr-xr-x | jjb/releng/verify-releng.sh | 2 |
10 files changed, 595 insertions, 91 deletions
diff --git a/jjb/releng/automate.yml b/jjb/releng/automate.yml index 4f6044bb5..908e9a196 100644 --- a/jjb/releng/automate.yml +++ b/jjb/releng/automate.yml @@ -90,6 +90,7 @@ publishers: - publish-coverage + - email-jenkins-admins-on-failure - job-template: name: '{module}-automate-{stream}' @@ -170,6 +171,7 @@ publishers: - 'email-publisher' + - email-jenkins-admins-on-failure - job-template: name: '{module}-automate-{phase}-{stream}' @@ -235,17 +237,17 @@ name: 'testapi-automate-docker-deploy-macro' builders: - shell: | - bash ./jjb/releng/docker-deploy.sh "sudo docker run -dti -p 8082:8000 + sudo bash ./jjb/releng/docker-deploy.sh "sudo docker run -dti --name testapi -p 8082:8000 -e mongodb_url=mongodb://172.17.0.1:27017 -e base_url=http://testresults.opnfv.org/test opnfv/testapi" \ - "http://testresults.opnfv.org/test/swagger/APIs" "testapi" + "http://testresults.opnfv.org/test/" "testapi" - builder: name: 'reporting-automate-docker-deploy-macro' builders: - shell: | - bash ./jjb/releng/docker-deploy.sh "sudo docker run -itd -p 8084:8000 opnfv/reporting" \ - "http://testresults.opnfv.org/reporting2/reporting/index.html" "reporting" + sudo bash ./jjb/releng/docker-deploy.sh "sudo docker run -itd --name reporting -p 8084:8000 opnfv/reporting" \ + "http://testresults.opnfv.org/reporting/index.html" "reporting" - builder: name: mongodb-backup diff --git a/jjb/releng/docker-deploy.sh b/jjb/releng/docker-deploy.sh index 2a3e078ae..1e8357717 100644 --- a/jjb/releng/docker-deploy.sh +++ b/jjb/releng/docker-deploy.sh @@ -16,89 +16,137 @@ # specific language governing permissions and limitations * # under the License. * -# Assigning Variables + command=$1 url=$2 module=$3 -function check() { +REPO="opnfv" +latest_image=$REPO/$module:latest +old_image=$REPO/$module:old +latest_container_name=$module +old_container_name=$module"_old" +latest_container_id= +old_container_id= +new_start_container= + +function DEBUG() { + echo `date "+%Y-%m-%d %H:%M:%S.%N"` ": $1" +} - # Verify hosted +function check_connectivity() { + # check update status via test the connectivity of provide url sleep 5 cmd=`curl -s --head --request GET ${url} | grep '200 OK' > /dev/null` rc=$? - echo $rc - - if [[ $rc == 0 ]] - then + DEBUG $rc + if [[ $rc == 0 ]]; then return 0 else return 1 fi - } -echo "Getting contianer Id of the currently running one" -contId=$(sudo docker ps | grep "opnfv/${module}:latest" | awk '{print $1}') - -echo $contId -echo "Pulling the latest image" -sudo docker pull opnfv/${module}:latest +function pull_latest_image() { + DEBUG "pull latest image $latest_image" + docker pull $latest_image +} -echo "Deleting old containers of opnfv/${module}:old" -sudo docker ps -a | grep "opnfv/${module}" | grep "old" | awk '{print $1}' | xargs -r sudo docker rm -f +function get_latest_running_container() { + latest_container_id=`docker ps -q --filter name=^/$latest_container_name$` +} -echo "Deleting old images of opnfv/${module}:latest" -sudo docker images | grep "opnfv/${module}" | grep "old" | awk '{print $3}' | xargs -r sudo docker rmi -f +function get_old_running_container() { + old_container_id=`docker ps -q --filter name=^/$old_container_name$` +} +function delete_old_image() { + DEBUG "delete old image: $old_image" + docker rmi -f $old_image +} -if [[ -z "$contId" ]] -then - echo "No running ${module} container" +function delete_old_container() { + DEBUG "delete old container: $old_container_name" + docker ps -a -q --filter name=^/$old_container_name$ | xargs docker rm -f &>/dev/null +} - echo "Removing stopped ${module} containers in the previous iterations" - sudo docker ps -f status=exited | grep "opnfv_${module}" | awk '{print $1}' | xargs -r sudo docker rm -f -else - echo $contId +function delete_latest_container() { + DEBUG "delete latest container: $module" + docker ps -a -q --filter name=^/$latest_container_name$ | xargs docker rm -f &>/dev/null +} - echo "Get the image id of the currently running conatiner" - currImgId=$(sudo docker ps | grep "$contId" | awk '{print $2}') - echo $currImgId +function delete_latest_image() { + DEBUG "delete latest image: $REPO/$module:latest" + docker rmi -f $latest_image +} - if [[ -z "$currImgId" ]] - then - echo "No image id found for the container id" - exit 1 - fi +function change_image_tag_2_old() { + DEBUG "change image tag 2 old" + docker tag $latest_image $old_image + docker rmi -f $latest_image +} - echo "Changing current image tag to old" - sudo docker tag "$currImgId" opnfv/${module}:old +function mark_latest_container_2_old() { + DEBUG "mark latest container to be old" + docker rename "$latest_container_name" "$old_container_name" +} - echo "Removing stopped ${module} containers in the previous iteration" - sudo docker ps -f status=exited | grep "opnfv_${module}" | awk '{print $1}' | xargs -r sudo docker rm -f +function stop_old_container() { + DEBUG "stop old container" + docker stop "$old_container_name" +} - echo "Renaming the running container name to opnfv_${module} as to identify it." - sudo docker rename $contId opnfv_${module} +function run_latest_image() { + new_start_container=`$command` + DEBUG "run latest image: $new_start_container" +} - echo "Stop the currently running container" - sudo docker stop $contId +get_latest_running_container +get_old_running_container + +if [[ ! -z $latest_container_id ]]; then + DEBUG "latest container is running: $latest_container_id" + delete_old_container + delete_old_image + change_image_tag_2_old + mark_latest_container_2_old + pull_latest_image + stop_old_container + run_latest_image + +elif [[ ! -z $old_container_id ]]; then + DEBUG "old container is running: $old_container_id" + delete_latest_container + delete_latest_image + pull_latest_image + stop_old_container + run_latest_image +else + DEBUG "no container is running" + delete_old_container + delete_old_image + delete_latest_container + delete_latest_image + pull_latest_image + run_latest_image fi -echo "Running a container with the new image" -$command:latest - -if check; then - echo "TestResults Module Hosted." +if check_connectivity; then + DEBUG "CONGRATS: $module update successfully" else - echo "TestResults Module Failed" - if [[ $(sudo docker images | grep "opnfv/${module}" | grep "old" | awk '{print $3}') ]]; then - echo "Running old Image" - $command:old - exit 1 + DEBUG "ATTENTION: $module update failed" + id=`docker ps -a -q --filter name=^/$old_container_name$` + if [[ ! -z $id ]]; then + DEBUG "start old container instead" + docker stop $new_start_container + docker start $id + fi + if ! check_connectivity; then + DEBUG "BIG ISSUE: no container is running normally" fi + exit 1 fi -# Echo Images and Containers -sudo docker images -sudo docker ps -a +docker images +docker ps -a diff --git a/jjb/releng/generate-job-list.sh b/jjb/releng/generate-job-list.sh new file mode 100755 index 000000000..4bf8974e4 --- /dev/null +++ b/jjb/releng/generate-job-list.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o errexit +set -o pipefail + +# Job Number Formatter +function JOBS { + local NUMS=$1 + if [ $NUMS == 1 ]; then + echo -n "Job" + else + echo -n "Jobs" + fi +} + +# Activiate the virtualenv so we have access to 'jenkins-jobs' +source /opt/virtualenv/jenkins-job-builder/bin/activate + +echo "> Generating list of current JJB jobs..." +jenkins-jobs -l ERROR test -r jjb -o job_output + +echo "> Generating list of previous JJB jobs..." +git checkout -b previous-commit HEAD^ +jenkins-jobs -l ERROR test -r jjb -o job_output_prev +git checkout - && git branch -d previous-commit + +echo "> Finding job changes ..." +diff -r -q job_output job_output_prev &> job_diff.txt || true + +# Only in (job_output) = NEW +# Only in (job_output_prev) = DELETED +# Files ... differ = MODIFIED + +declare -a JOBS_ADDED=($(grep 'job_output:' job_diff.txt | cut -d':' -f2- | sed 's/^[ \t]*//;s/[ \t]*$//')) +declare -a JOBS_MODIFIED=($(grep 'differ$' job_diff.txt | sed "s/Files job_output\/\(.*\) and.*/\1/g")) +declare -a JOBS_REMOVED=($(grep 'job_output_prev:' job_diff.txt | cut -d ':' -f2- | sed 's/^[ \t]*//;s/[ \t]*$//')) + +NUM_JOBS_ADDED=${#JOBS_ADDED[@]} +NUM_JOBS_MODIFIED=${#JOBS_MODIFIED[@]} +NUM_JOBS_REMOVED=${#JOBS_REMOVED[@]} + +echo "> Writing gerrit comment ..." +if [ $NUM_JOBS_ADDED -gt 0 ]; then + JOB_STRING="$(JOBS $NUM_JOBS_ADDED)" + { printf "Added %s %s:\n\n" "${NUM_JOBS_ADDED}" "$JOB_STRING"; + printf '* %s\n' "${JOBS_ADDED[@]}"; + printf "\n"; } >> gerrit_comment.txt +fi + +if [ $NUM_JOBS_MODIFIED -gt 0 ]; then + JOB_STRING="$(JOBS $NUM_JOBS_MODIFIED)" + { printf "Modified %s %s:\n\n" "${NUM_JOBS_MODIFIED}" "$JOB_STRING"; + printf '* %s\n' "${JOBS_MODIFIED[@]}"; + printf "\n"; } >> gerrit_comment.txt +fi + +if [ $NUM_JOBS_REMOVED -gt 0 ]; then + JOB_STRING="$(JOBS $NUM_JOBS_REMOVED)" + { printf "Removed %s %s:\n\n" "${NUM_JOBS_REMOVED}" "$JOB_STRING"; + printf '* %s\n' "${JOBS_REMOVED[@]}"; + printf "\n"; } >> gerrit_comment.txt +fi diff --git a/jjb/releng/opnfv-docker-arm.yml b/jjb/releng/opnfv-docker-arm.yml index 417fc702c..7f2574ec4 100644 --- a/jjb/releng/opnfv-docker-arm.yml +++ b/jjb/releng/opnfv-docker-arm.yml @@ -10,40 +10,156 @@ stream: master branch: '{stream}' disabled: false - danube: &danube - stream: danube + euphrates: &euphrates + stream: euphrates branch: 'stable/{stream}' disabled: false functest-arm-receivers: &functest-arm-receivers receivers: > cristina.pauna@enea.com alexandru.avadanii@enea.com + delia.popescu@enea.com dovetail-arm-receivers: &dovetail-arm-receivers receivers: > cristina.pauna@enea.com alexandru.avadanii@enea.com alexandru.nemes@enea.com + storperf-arm-receivers: &storperf-arm-receivers + receivers: > + cristina.pauna@enea.com + alexandru.avadanii@enea.com + mark.beierl@emc.com + yardstick-arm-receivers: &yardstick-arm-receivers + receivers: > + cristina.pauna@enea.com + alexandru.avadanii@enea.com + alexandru.nemes@enea.com + catalina.focsa@enea.com + delia.popescu@enea.com other-receivers: &other-receivers receivers: '' - project: + dockerfile: "Dockerfile.aarch64" + dockerdir: "docker" + docker_repo_name: "opnfv/{project}_aarch64" + arch_tag: "" + + dockerrepo: # projects with jobs for master - 'functest': + project: 'functest' <<: *master <<: *functest-arm-receivers - 'dovetail': + project: 'dovetail' <<: *master <<: *dovetail-arm-receivers - # projects with jobs for stable + - 'storperf-master': + project: 'storperf' + dockerdir: 'docker/storperf-master' + dockerfile: 'Dockerfile' + docker_repo_name: 'opnfv/storperf-master' + arch_tag: 'aarch64' + <<: *master + <<: *storperf-arm-receivers + - 'storperf-graphite': + project: 'storperf' + dockerdir: 'docker/storperf-graphite' + dockerfile: 'Dockerfile' + docker_repo_name: 'opnfv/storperf-graphite' + arch_tag: 'aarch64' + <<: *master + <<: *storperf-arm-receivers + - 'storperf-httpfrontend': + project: 'storperf' + dockerdir: 'docker/storperf-httpfrontend' + dockerfile: 'Dockerfile' + docker_repo_name: 'opnfv/storperf-httpfrontend' + arch_tag: 'aarch64' + <<: *master + <<: *storperf-arm-receivers + - 'storperf-reporting': + project: 'storperf' + dockerdir: 'docker/storperf-reporting' + dockerfile: 'Dockerfile' + docker_repo_name: 'opnfv/storperf-reporting' + arch_tag: 'aarch64' + <<: *master + <<: *storperf-arm-receivers + - 'storperf-swaggerui': + project: 'storperf' + dockerdir: 'docker/storperf-swaggerui' + dockerfile: 'Dockerfile' + docker_repo_name: 'opnfv/storperf-swaggerui' + arch_tag: 'aarch64' + <<: *master + <<: *storperf-arm-receivers + - 'yardstick': + project: 'yardstick' + <<: *master + <<: *yardstick-arm-receivers + # projects with jobs for stable/euphrates + - 'functest': + project: 'functest' + <<: *euphrates + <<: *functest-arm-receivers + - 'dovetail': + project: 'dovetail' + <<: *euphrates + <<: *dovetail-arm-receivers + - 'storperf-master': + project: 'storperf' + dockerdir: 'docker/storperf-master' + dockerfile: 'Dockerfile' + docker_repo_name: 'opnfv/storperf-master' + arch_tag: 'aarch64' + <<: *euphrates + <<: *storperf-arm-receivers + - 'storperf-graphite': + project: 'storperf' + dockerdir: 'docker/storperf-graphite' + dockerfile: 'Dockerfile' + docker_repo_name: 'opnfv/storperf-graphite' + arch_tag: 'aarch64' + <<: *euphrates + <<: *storperf-arm-receivers + - 'storperf-httpfrontend': + project: 'storperf' + dockerdir: 'docker/storperf-httpfrontend' + dockerfile: 'Dockerfile' + docker_repo_name: 'opnfv/storperf-httpfrontend' + arch_tag: 'aarch64' + <<: *euphrates + <<: *storperf-arm-receivers + - 'storperf-reporting': + project: 'storperf' + dockerdir: 'docker/storperf-reporting' + dockerfile: 'Dockerfile' + docker_repo_name: 'opnfv/storperf-reporting' + arch_tag: 'aarch64' + <<: *euphrates + <<: *storperf-arm-receivers + - 'storperf-swaggerui': + project: 'storperf' + dockerdir: 'docker/storperf-swaggerui' + dockerfile: 'Dockerfile' + docker_repo_name: 'opnfv/storperf-swaggerui' + arch_tag: 'aarch64' + <<: *euphrates + <<: *storperf-arm-receivers + - 'yardstick': + project: 'yardstick' + <<: *euphrates + <<: *yardstick-arm-receivers jobs: - - '{project}-docker-build-arm-push-{stream}' + - '{dockerrepo}-docker-build-arm-push-{stream}' ######################## # job templates ######################## - job-template: - name: '{project}-docker-build-arm-push-{stream}' + name: '{dockerrepo}-docker-build-arm-push-{stream}' disabled: '{obj:disabled}' @@ -58,16 +174,29 @@ description: "To enable/disable pushing the image to Dockerhub." - string: name: DOCKER_REPO_NAME - default: "opnfv/{project}_aarch64" + default: "{docker_repo_name}" description: "Dockerhub repo to be pushed to." - string: name: RELEASE_VERSION default: "" description: "Release version, e.g. 1.0, 2.0, 3.0" - string: + name: DOCKER_DIR + default: "{dockerdir}" + description: "Directory containing files needed by the Dockerfile" + - string: name: DOCKERFILE - default: "Dockerfile.aarch64" + default: "{dockerfile}" description: "Dockerfile to use for creating the image." + - string: + name: ARCH_TAG + default: "{arch_tag}" + description: "If set, this value will be added to the docker image tag" + + properties: + - throttle: + max-per-node: 1 + option: 'project' scm: - git-scm @@ -83,3 +212,4 @@ publishers: - email: recipients: '{receivers}' + - email-jenkins-admins-on-failure diff --git a/jjb/releng/opnfv-docker.sh b/jjb/releng/opnfv-docker.sh index ebd0c9f3d..298d2af51 100644 --- a/jjb/releng/opnfv-docker.sh +++ b/jjb/releng/opnfv-docker.sh @@ -18,8 +18,8 @@ echo "--------------------------------------------------------" echo count=30 # docker build jobs might take up to ~30 min -while [[ -n `ps -ef|grep 'docker build'|grep -v grep` ]]; do - echo "Build in progress. Waiting..." +while [[ -n `ps -ef| grep 'docker build' | grep $DOCKER_REPO_NAME | grep -v grep` ]]; do + echo "Build or cleanup of $DOCKER_REPO_NAME in progress. Waiting..." sleep 60 count=$(( $count - 1 )) if [ $count -eq 0 ]; then @@ -54,7 +54,7 @@ if [[ -n "$(docker images | grep $DOCKER_REPO_NAME)" ]]; then done fi -cd $WORKSPACE/docker +cd $WORKSPACE/$DOCKER_DIR HOST_ARCH=$(uname -m) if [ ! -f "${DOCKERFILE}" ]; then # If this is expected to be a Dockerfile for other arch than x86 @@ -89,11 +89,19 @@ if [[ -n "${COMMIT_ID-}" && -n "${RELEASE_VERSION-}" ]]; then BUILD_BRANCH=$COMMIT_ID fi +ARCH_BUILD_ARG="" +ARCH_TAG=${ARCH_TAG:-} +if [[ -n "${ARCH_TAG}" ]]; then + DOCKER_TAG=${ARCH_TAG}-${DOCKER_TAG} + ARCH_BUILD_ARG="--build-arg ARCH=${ARCH_TAG}" +fi + # Start the build echo "Building docker image: $DOCKER_REPO_NAME:$DOCKER_TAG" echo "--------------------------------------------------------" echo cmd="docker build --no-cache -t $DOCKER_REPO_NAME:$DOCKER_TAG --build-arg BRANCH=$BUILD_BRANCH + $ARCH_BUILD_ARG -f $DOCKERFILE ." echo ${cmd} diff --git a/jjb/releng/opnfv-docker.yml b/jjb/releng/opnfv-docker.yml index 095ba4129..1a3db3a5f 100644 --- a/jjb/releng/opnfv-docker.yml +++ b/jjb/releng/opnfv-docker.yml @@ -14,71 +14,164 @@ stream: danube branch: 'stable/{stream}' disabled: false + euphrates: &euphrates + stream: euphrates + branch: 'stable/{stream}' + disabled: false functest-receivers: &functest-receivers receivers: > - jose.lausuch@ericsson.com morgan.richomme@orange.com + jalausuch@suse.com morgan.richomme@orange.com cedric.ollivier@orange.com feng.xiaowei@zte.com.cn - yaohelan@huawei.com helanyao@gmail.com - juha.kosonen@nokia.com + juha.kosonen@nokia.com wangwulin@huawei.com + storperf-receivers: &storperf-receivers + receivers: > + mark.beierl@emc.com other-receivers: &other-receivers receivers: '' - project: + dockerfile: "Dockerfile" + dockerdir: "docker" + arch_tag: "" + + # This is the dockerhub repo the image will be pushed to as + # 'opnfv/{dockerrepo}. See: DOCKER_REPO_NAME parameter. + # 'project' is the OPNFV repo we expect to contain the Dockerfile + dockerrepo: # projects with jobs for master - 'releng-anteater': + project: 'releng-anteater' <<: *master <<: *other-receivers - 'bottlenecks': + project: 'bottlenecks' <<: *master <<: *other-receivers - 'cperf': + project: 'cperf' <<: *master <<: *other-receivers - 'dovetail': + project: 'dovetail' <<: *master <<: *other-receivers - 'functest': + project: 'functest' <<: *master <<: *functest-receivers - - 'qtip': + - 'nfvbench': + project: 'nfvbench' <<: *master <<: *other-receivers - - 'storperf': + - 'qtip': + project: 'qtip' <<: *master <<: *other-receivers + - 'storperf-master': + project: 'storperf' + dockerdir: 'docker/storperf-master' + arch_tag: 'x86_64' + <<: *master + <<: *storperf-receivers + - 'storperf-graphite': + project: 'storperf' + dockerdir: 'docker/storperf-graphite' + arch_tag: 'x86_64' + <<: *master + <<: *storperf-receivers + - 'storperf-httpfrontend': + project: 'storperf' + dockerdir: 'docker/storperf-httpfrontend' + arch_tag: 'x86_64' + <<: *master + <<: *storperf-receivers + - 'storperf-reporting': + project: 'storperf' + dockerdir: 'docker/storperf-reporting' + arch_tag: 'x86_64' + <<: *master + <<: *storperf-receivers + - 'storperf-swaggerui': + project: 'storperf' + dockerdir: 'docker/storperf-swaggerui' + arch_tag: 'x86_64' + <<: *master + <<: *storperf-receivers - 'yardstick': + project: 'yardstick' <<: *master <<: *other-receivers - # projects with jobs for stable + # projects with jobs for Danube + - 'qtip': + project: 'qtip' + <<: *euphrates + <<: *other-receivers + - 'yardstick': + project: 'yardstick' + <<: *euphrates + <<: *other-receivers + # projects with jobs for euphrates - 'bottlenecks': - <<: *danube + project: 'bottlenecks' + <<: *euphrates <<: *other-receivers - 'functest': - <<: *danube + project: 'functest' + <<: *euphrates <<: *functest-receivers - - 'qtip': - <<: *danube - <<: *other-receivers - - 'storperf': - <<: *danube - <<: *other-receivers - - 'yardstick': - <<: *danube + - 'storperf-master': + project: 'storperf' + dockerdir: 'docker/storperf-master' + arch_tag: 'x86_64' + <<: *euphrates + <<: *storperf-receivers + - 'storperf-graphite': + project: 'storperf' + dockerdir: 'docker/storperf-graphite' + arch_tag: 'x86_64' + <<: *euphrates + <<: *storperf-receivers + - 'storperf-httpfrontend': + project: 'storperf' + dockerdir: 'docker/storperf-httpfrontend' + arch_tag: 'x86_64' + <<: *euphrates + <<: *storperf-receivers + - 'storperf-reporting': + project: 'storperf' + dockerdir: 'docker/storperf-reporting' + arch_tag: 'x86_64' + <<: *euphrates + <<: *storperf-receivers + - 'storperf-swaggerui': + project: 'storperf' + dockerdir: 'docker/storperf-swaggerui' + arch_tag: 'x86_64' + <<: *euphrates + <<: *storperf-receivers + - 'nfvbench': + project: 'nfvbench' + <<: *euphrates <<: *other-receivers jobs: - - '{project}-docker-build-push-{stream}' + - "{dockerrepo}-docker-build-push-{stream}" - project: name: opnfv-monitor-docker # projects which only monitor dedicated file or path + dockerfile: "Dockerfile" + dockerdir: "docker" + arch_tag: "" + project: # projects with jobs for master - 'daisy': + dockerrepo: 'daisy' <<: *master - 'escalator': + dockerrepo: 'escalator' <<: *master jobs: @@ -88,7 +181,7 @@ # job templates ######################## - job-template: - name: '{project}-docker-build-push-{stream}' + name: '{dockerrepo}-docker-build-push-{stream}' disabled: '{obj:disabled}' @@ -103,9 +196,13 @@ description: "To enable/disable pushing the image to Dockerhub." - string: name: DOCKER_REPO_NAME - default: "opnfv/{project}" + default: "opnfv/{dockerrepo}" description: "Dockerhub repo to be pushed to." - string: + name: DOCKER_DIR + default: "{dockerdir}" + description: "Directory containing files needed by the Dockerfile" + - string: name: COMMIT_ID default: "" description: "commit id to make a snapshot docker image" @@ -115,8 +212,17 @@ description: "Release version, e.g. 1.0, 2.0, 3.0" - string: name: DOCKERFILE - default: "Dockerfile" + default: "{dockerfile}" description: "Dockerfile to use for creating the image." + - string: + name: ARCH_TAG + default: "{arch_tag}" + description: "If set, this value will be added to the docker image tag as a prefix" + + properties: + - throttle: + max-per-node: 1 + option: 'project' scm: - git-scm @@ -132,6 +238,7 @@ publishers: - email: recipients: '{receivers}' + - email-jenkins-admins-on-failure - job-template: name: '{project}-docker-build-push-monitor-{stream}' diff --git a/jjb/releng/opnfv-repo-archiver.sh b/jjb/releng/opnfv-repo-archiver.sh new file mode 100644 index 000000000..c9fdba379 --- /dev/null +++ b/jjb/releng/opnfv-repo-archiver.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +set -o errexit +set -o pipefail +export PATH=$PATH:/usr/local/bin/ + +DATE="$(date +%Y%m%d)" + +declare -a PROJECT_LIST +EXCLUDE_PROJECTS="All-Projects|All-Users|securedlab" +CLONE_PATH="$WORKSPACE/opnfv-repos" + +# Generate project list from gerrit +PROJECT_LIST=($(ssh -p 29418 jenkins-ci@gerrit.opnfv.org gerrit ls-projects | egrep -v $EXCLUDE_PROJECTS)) + +echo "Cloning all OPNFV repositories" +echo "------------------------------" + +for PROJECT in "${PROJECT_LIST[@]}"; do + echo "> Cloning $PROJECT" + if [ ! -d "$CLONE_PATH/$PROJECT" ]; then + git clone "https://gerrit.opnfv.org/gerrit/$PROJECT.git" $CLONE_PATH/$PROJECT + else + pushd "$CLONE_PATH/$PROJECT" &>/dev/null + git pull -f + popd &> /dev/null + fi + + # Don't license scan kernel or qemu in kvmfornfv + if [ "$PROJECT" == "kvmfornfv" ]; then + rm -rf "$CLONE_PATH/$PROJECT/{kernel,qemu}" + fi +done + +echo "Finished cloning OPNFV repositories" +echo "-----------------------------------" + +# Copy repos and clear git data +echo "Copying repos to $WORKSPACE/opnfv-archive and removing .git files" +cp -R $CLONE_PATH $WORKSPACE/opnfv-archive +find $WORKSPACE/opnfv-archive -type d -iname '.git' -exec rm -rf {} + +find $WORKSPACE/opnfv-archive -type f -iname '.git*' -exec rm -rf {} + + +# Create archive +echo "Creating archive: opnfv-archive-$DATE.tar.gz" +echo "--------------------------------------" +cd $WORKSPACE +tar -czf "opnfv-archive-$DATE.tar.gz" opnfv-archive && rm -rf opnfv-archive +echo "Archiving Complete." + +echo "Uploading artifacts" +echo "--------------------------------------" + +gsutil cp "$WORKSPACE/opnfv-archive-$DATE.tar.gz" \ + "gs://opnfv-archive/opnfv-archive-$DATE.tar.gz" 2>&1 + +rm -f opnfv-archive-$DATE.tar.gz + +echo "Finished" diff --git a/jjb/releng/opnfv-utils.yml b/jjb/releng/opnfv-utils.yml index 717bb3cbc..721b5dede 100644 --- a/jjb/releng/opnfv-utils.yml +++ b/jjb/releng/opnfv-utils.yml @@ -4,6 +4,9 @@ jobs: - 'prune-docker-images' + - 'archive-repositories' + - 'check-status-of-slaves' + ######################## # job templates ######################## @@ -37,3 +40,50 @@ triggers: - timed: '@midnight' + +- job-template: + name: 'archive-repositories' + + disabled: false + + concurrent: true + + parameters: + - node: + name: SLAVE_NAME + description: Where to create the archive + default-slaves: + - master + allowed-multiselect: false + ignore-offline-nodes: true + + triggers: + - timed: '@monthly' + + builders: + - shell: + !include-raw-escape: opnfv-repo-archiver.sh + +- job-template: + name: 'check-status-of-slaves' + + disabled: false + + concurrent: true + + parameters: + - node: + name: SLAVE_NAME + description: We don't want workspace wiped. so I just threw the script on the master + default-slaves: + - master + allowed-multiselect: false + ignore-offline-nodes: true + + triggers: + - timed: '@midnight' + + builders: + - shell: | + cd /opt/jenkins-ci/slavemonitor + bash slave-monitor-0.1.sh | sort diff --git a/jjb/releng/releng-ci-jobs.yml b/jjb/releng/releng-ci-jobs.yml index ecc87303f..fd58ef2b8 100644 --- a/jjb/releng/releng-ci-jobs.yml +++ b/jjb/releng/releng-ci-jobs.yml @@ -3,6 +3,7 @@ jobs: - 'releng-verify-jjb' - 'releng-merge-jjb' + - 'releng-comment-jjb' - 'releng-generate-artifacts-api' project: 'releng' @@ -52,6 +53,28 @@ publishers: - archive-artifacts: artifacts: 'job_output/*' + - email-jenkins-admins-on-failure + +- job-template: + name: releng-comment-jjb + + parameters: + - project-parameter: + project: '{project}' + branch: 'master' + scm: + - git-scm-gerrit + + triggers: + - experimental: + project: '{project}' + branch: 'master' + files: 'jjb/**' + + builders: + - shell: + !include-raw-escape: generate-job-list.sh + - report-build-result-to-gerrit - job-template: name: 'releng-merge-jjb' diff --git a/jjb/releng/verify-releng.sh b/jjb/releng/verify-releng.sh index 682a8beda..a6eaa0f50 100755 --- a/jjb/releng/verify-releng.sh +++ b/jjb/releng/verify-releng.sh @@ -8,7 +8,7 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## #test for non-ascii characters, these can pass the test and end up breaking things in production -for x in $(find . -name *\.yml -or -name *\.yaml); do +for x in $(find . -name *\.yml -or -name *\.yaml -or -name *\.sh); do if LC_ALL=C grep -q '[^[:print:][:space:]]' "$x"; then echo "file "$x" contains non-ascii characters" |