From f8314a3e76bd4aac1ada4a174b7468f3604f86ea Mon Sep 17 00:00:00 2001 From: wutianwei Date: Thu, 16 Nov 2017 17:27:59 +0800 Subject: compass: Build Docker images with jjb JIRA: COMPASS-567 Add compass4nfv-docker.yml compass-tasks-k8s and compass-tasks-osa are dependent on compass-compass-tasks. So we have to phase to build docker images phase 1. execute the multijob 'build compass-tasks images' Only the phase 1 is successful then excute the phase 2 phase 2. execute the multijob 'build all compass images' Modify the opnfv-docker.sh 1. Add a function remove_containers_images 2. call the remove_containers_images when beginning and finishing build to solve the problem of rmove of images due to dependancy Change-Id: I66fb27260cf114da96417361e80c8183db3233bd Signed-off-by: wutianwei --- jjb/releng/compass4nfv-docker.yml | 177 ++++++++++++++++++++++++++++++++++++++ jjb/releng/opnfv-docker.sh | 60 +++++++------ 2 files changed, 212 insertions(+), 25 deletions(-) create mode 100644 jjb/releng/compass4nfv-docker.yml (limited to 'jjb') diff --git a/jjb/releng/compass4nfv-docker.yml b/jjb/releng/compass4nfv-docker.yml new file mode 100644 index 000000000..299908d66 --- /dev/null +++ b/jjb/releng/compass4nfv-docker.yml @@ -0,0 +1,177 @@ +--- +############################################## +# job configuration for docker build and push +############################################## +- project: + + name: compass-docker + + project: compass-containers + + stream: + - master: + branch: '{stream}' + disabled: false + - euphrates: + branch: 'stable/{stream}' + disabled: true + + arch_tag: + - 'amd64': + slave_label: 'opnfv-build-ubuntu' + + # yamllint disable rule:key-duplicates + image: + - 'tasks' + - 'cobbler' + - 'db' + - 'deck' + - 'tasks-k8s' + - 'tasks-osa' + + # settings for jobs run in multijob phases + build-job-settings: &build-job-settings + current-parameters: false + git-revision: true + node-parameters: false + predefined-parameters: | + PUSH_IMAGE=$PUSH_IMAGE + COMMIT_ID=$COMMIT_ID + RELEASE_VERSION=$RELEASE_VERSION + DOCKERFILE=$DOCKERFILE + kill-phase-on: FAILURE + abort-all-jobs: false + + # yamllint enable rule:key-duplicates + jobs: + - "compass-docker-{stream}" + - "compass-{image}-build-{arch_tag}-{stream}" + +######################## +# job templates +######################## +- job-template: + name: 'compass-docker-{stream}' + + project-type: multijob + + disabled: '{obj:disabled}' + + parameters: + - compass-job-parameters: + project: '{project}' + branch: '{branch}' + slave_label: 'opnfv-build-ubuntu' + arch_tag: 'amd64' + + properties: + - throttle: + max-per-node: 1 + option: 'project' + + scm: + - git-scm + + triggers: + - pollscm: + cron: "*/30 * * * *" + + builders: + - multijob: + name: 'build compass-tasks images' + execution-type: PARALLEL + projects: + - name: 'compass-tasks-build-amd64-{stream}' + <<: *build-job-settings + - multijob: + name: 'build all compass images' + condition: SUCCESSFUL + execution-type: PARALLEL + projects: + - name: 'compass-cobbler-build-amd64-{stream}' + <<: *build-job-settings + - name: 'compass-db-build-amd64-{stream}' + <<: *build-job-settings + - name: 'compass-deck-build-amd64-{stream}' + <<: *build-job-settings + - name: 'compass-tasks-k8s-build-amd64-{stream}' + <<: *build-job-settings + - name: 'compass-tasks-osa-build-amd64-{stream}' + <<: *build-job-settings + + publishers: + - 'compass-amd64-recipients' + +- job-template: + name: 'compass-{image}-build-{arch_tag}-{stream}' + disabled: '{obj:disabled}' + parameters: + - compass-job-parameters: + project: '{project}' + branch: '{branch}' + slave_label: '{slave_label}' + arch_tag: '{arch_tag}' + - string: + name: DOCKER_REPO_NAME + default: "opnfv/compass-{image}" + description: "Dockerhub repo to be pushed to." + - string: + name: DOCKER_DIR + default: "compass-{image}" + description: "Directory containing files needed by the Dockerfile" + scm: + - git-scm + builders: + - shell: + !include-raw-escape: ./opnfv-docker.sh + +# parameter macro +- parameter: + name: compass-job-parameters + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - label: + name: SLAVE_LABEL + default: '{slave_label}' + description: 'Slave label on Jenkins' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' + - string: + name: PUSH_IMAGE + default: "true" + description: "To enable/disable pushing the image to Dockerhub." + - string: + name: COMMIT_ID + default: "" + description: "commit id to make a snapshot docker image" + - string: + name: RELEASE_VERSION + default: "" + description: "Docker tag to be built, e.g. 5.0.0, opnfv-5.0.0, 5.0.RC1" + - string: + name: DOCKERFILE + default: "Dockerfile" + description: "Dockerfile to use for creating the image." + - string: + name: ARCH_TAG + default: "" + description: "If set, this value will be added to the docker image tag as a prefix" + - string: + name: PROJECT + default: "{project}" + description: "Project name used to enable job conditions" + +# publisher macros +- publisher: + name: 'compass-amd64-recipients' + publishers: + - email: + recipients: > + chigang@huawei.com + huangxiangyui5@huawei.com + xueyifei@huawei.com + wutianwei1@huawei.com diff --git a/jjb/releng/opnfv-docker.sh b/jjb/releng/opnfv-docker.sh index c179b1d8e..ec7b3fda3 100644 --- a/jjb/releng/opnfv-docker.sh +++ b/jjb/releng/opnfv-docker.sh @@ -17,6 +17,36 @@ echo "Starting opnfv-docker for $DOCKER_REPO_NAME ..." echo "--------------------------------------------------------" echo +function remove_containers_images() +{ + # Remove previous running containers if exist + if [[ -n "$(docker ps -a | grep $DOCKER_REPO_NAME)" ]]; then + echo "Removing existing $DOCKER_REPO_NAME containers..." + docker ps -a | grep $DOCKER_REPO_NAME | awk '{print $1}' | xargs docker rm -f + t=60 + # Wait max 60 sec for containers to be removed + while [[ $t -gt 0 ]] && [[ -n "$(docker ps| grep $DOCKER_REPO_NAME)" ]]; do + sleep 1 + let t=t-1 + done + fi + + + # Remove existing images if exist + if [[ -n "$(docker images | grep $DOCKER_REPO_NAME)" ]]; then + echo "Docker images to remove:" + docker images | head -1 && docker images | grep $DOCKER_REPO_NAME + image_ids=($(docker images | grep $DOCKER_REPO_NAME | awk '{print $3}')) + for id in "${image_ids[@]}"; do + if [[ -n "$(docker images|grep $DOCKER_REPO_NAME|grep $id)" ]]; then + echo "Removing docker image $DOCKER_REPO_NAME:$id..." + docker rmi -f $id + fi + done + fi +} + + count=30 # docker build jobs might take up to ~30 min while [[ -n `ps -ef| grep 'docker build' | grep $DOCKER_REPO_NAME | grep -v grep` ]]; do echo "Build or cleanup of $DOCKER_REPO_NAME in progress. Waiting..." @@ -28,31 +58,8 @@ while [[ -n `ps -ef| grep 'docker build' | grep $DOCKER_REPO_NAME | grep -v grep fi done -# Remove previous running containers if exist -if [[ -n "$(docker ps -a | grep $DOCKER_REPO_NAME)" ]]; then - echo "Removing existing $DOCKER_REPO_NAME containers..." - docker ps -a | grep $DOCKER_REPO_NAME | awk '{print $1}' | xargs docker rm -f - t=60 - # Wait max 60 sec for containers to be removed - while [[ $t -gt 0 ]] && [[ -n "$(docker ps| grep $DOCKER_REPO_NAME)" ]]; do - sleep 1 - let t=t-1 - done -fi - - -# Remove existing images if exist -if [[ -n "$(docker images | grep $DOCKER_REPO_NAME)" ]]; then - echo "Docker images to remove:" - docker images | head -1 && docker images | grep $DOCKER_REPO_NAME - image_ids=($(docker images | grep $DOCKER_REPO_NAME | awk '{print $3}')) - for id in "${image_ids[@]}"; do - if [[ -n "$(docker images|grep $DOCKER_REPO_NAME|grep $id)" ]]; then - echo "Removing docker image $DOCKER_REPO_NAME:$id..." - docker rmi -f $id - fi - done -fi +# Remove the existing containers and images before building +remove_containers_images cd "$WORKSPACE/$DOCKER_DIR" || exit 1 HOST_ARCH="$(uname -m)" @@ -117,3 +124,6 @@ if [[ "$PUSH_IMAGE" == "true" ]]; then echo docker push $DOCKER_REPO_NAME:$DOCKER_TAG fi + +# Remove the existing containers and images after building +remove_containers_images -- cgit 1.2.3-korg