compass: Build Docker images with jjb

JIRA: COMPASS-567

Add compass4nfv-docker.yml
compass-tasks-k8s and compass-tasks-osa are dependent on compass-compass-tasks.
So we have to phase to build docker images
phase 1. execute the multijob 'build compass-tasks images'
Only the phase 1 is successful then excute the phase 2
phase 2. execute the multijob 'build all compass images'

Modify the opnfv-docker.sh
1. Add a function remove_containers_images
2. call the remove_containers_images when beginning and finishing build
   to solve the problem of rmove of images due to dependancy

Change-Id: I66fb27260cf114da96417361e80c8183db3233bd
Signed-off-by: wutianwei <wutianwei1@huawei.com>

2 files changed, 212 insertions, 25 deletions

diff --git a/jjb/releng/compass4nfv-docker.yml b/jjb/releng/compass4nfv-docker.yml
new file mode 100644
index 000000000..299908d66
--- /dev/null
+++ b/jjb/releng/compass4nfv-docker.yml
@@ -0,0 +1,177 @@
+---
+##############################################
+# job configuration for docker build and push
+##############################################
+- project:
+
+    name: compass-docker
+
+    project: compass-containers
+
+    stream:
+      - master:
+          branch: '{stream}'
+          disabled: false
+      - euphrates:
+          branch: 'stable/{stream}'
+          disabled: true
+
+    arch_tag:
+      - 'amd64':
+          slave_label: 'opnfv-build-ubuntu'
+
+    # yamllint disable rule:key-duplicates
+    image:
+      - 'tasks'
+      - 'cobbler'
+      - 'db'
+      - 'deck'
+      - 'tasks-k8s'
+      - 'tasks-osa'
+
+    # settings for jobs run in multijob phases
+    build-job-settings: &build-job-settings
+      current-parameters: false
+      git-revision: true
+      node-parameters: false
+      predefined-parameters: |
+        PUSH_IMAGE=$PUSH_IMAGE
+        COMMIT_ID=$COMMIT_ID
+        RELEASE_VERSION=$RELEASE_VERSION
+        DOCKERFILE=$DOCKERFILE
+      kill-phase-on: FAILURE
+      abort-all-jobs: false
+
+    # yamllint enable rule:key-duplicates
+    jobs:
+      - "compass-docker-{stream}"
+      - "compass-{image}-build-{arch_tag}-{stream}"
+
+########################
+# job templates
+########################
+- job-template:
+    name: 'compass-docker-{stream}'
+
+    project-type: multijob
+
+    disabled: '{obj:disabled}'
+
+    parameters:
+      - compass-job-parameters:
+          project: '{project}'
+          branch: '{branch}'
+          slave_label: 'opnfv-build-ubuntu'
+          arch_tag: 'amd64'
+
+    properties:
+      - throttle:
+          max-per-node: 1
+          option: 'project'
+
+    scm:
+      - git-scm
+
+    triggers:
+      - pollscm:
+          cron: "*/30 * * * *"
+
+    builders:
+      - multijob:
+          name: 'build compass-tasks images'
+          execution-type: PARALLEL
+          projects:
+            - name: 'compass-tasks-build-amd64-{stream}'
+              <<: *build-job-settings
+      - multijob:
+          name: 'build all compass images'
+          condition: SUCCESSFUL
+          execution-type: PARALLEL
+          projects:
+            - name: 'compass-cobbler-build-amd64-{stream}'
+              <<: *build-job-settings
+            - name: 'compass-db-build-amd64-{stream}'
+              <<: *build-job-settings
+            - name: 'compass-deck-build-amd64-{stream}'
+              <<: *build-job-settings
+            - name: 'compass-tasks-k8s-build-amd64-{stream}'
+              <<: *build-job-settings
+            - name: 'compass-tasks-osa-build-amd64-{stream}'
+              <<: *build-job-settings
+
+    publishers:
+      - 'compass-amd64-recipients'
+
+- job-template:
+    name: 'compass-{image}-build-{arch_tag}-{stream}'
+    disabled: '{obj:disabled}'
+    parameters:
+      - compass-job-parameters:
+          project: '{project}'
+          branch: '{branch}'
+          slave_label: '{slave_label}'
+          arch_tag: '{arch_tag}'
+      - string:
+          name: DOCKER_REPO_NAME
+          default: "opnfv/compass-{image}"
+          description: "Dockerhub repo to be pushed to."
+      - string:
+          name: DOCKER_DIR
+          default: "compass-{image}"
+          description: "Directory containing files needed by the Dockerfile"
+    scm:
+      - git-scm
+    builders:
+      - shell:
+          !include-raw-escape: ./opnfv-docker.sh
+
+# parameter macro
+- parameter:
+    name: compass-job-parameters
+    parameters:
+      - project-parameter:
+          project: '{project}'
+          branch: '{branch}'
+      - label:
+          name: SLAVE_LABEL
+          default: '{slave_label}'
+          description: 'Slave label on Jenkins'
+      - string:
+          name: GIT_BASE
+          default: https://gerrit.opnfv.org/gerrit/$PROJECT
+          description: 'Git URL to use on this Jenkins Slave'
+      - string:
+          name: PUSH_IMAGE
+          default: "true"
+          description: "To enable/disable pushing the image to Dockerhub."
+      - string:
+          name: COMMIT_ID
+          default: ""
+          description: "commit id to make a snapshot docker image"
+      - string:
+          name: RELEASE_VERSION
+          default: ""
+          description: "Docker tag to be built, e.g. 5.0.0, opnfv-5.0.0, 5.0.RC1"
+      - string:
+          name: DOCKERFILE
+          default: "Dockerfile"
+          description: "Dockerfile to use for creating the image."
+      - string:
+          name: ARCH_TAG
+          default: ""
+          description: "If set, this value will be added to the docker image tag as a prefix"
+      - string:
+          name: PROJECT
+          default: "{project}"
+          description: "Project name used to enable job conditions"
+
+# publisher macros
+- publisher:
+    name: 'compass-amd64-recipients'
+    publishers:
+      - email:
+          recipients: >
+            chigang@huawei.com
+            huangxiangyui5@huawei.com
+            xueyifei@huawei.com
+            wutianwei1@huawei.com
diff --git a/jjb/releng/opnfv-docker.sh b/jjb/releng/opnfv-docker.sh
index c179b1d8e..ec7b3fda3 100644
--- a/jjb/releng/opnfv-docker.sh
+++ b/jjb/releng/opnfv-docker.sh
@@ -17,6 +17,36 @@ echo "Starting opnfv-docker for $DOCKER_REPO_NAME ..."
 echo "--------------------------------------------------------"
 echo
 
+function remove_containers_images()
+{
+    # Remove previous running containers if exist
+    if [[ -n "$(docker ps -a | grep $DOCKER_REPO_NAME)" ]]; then
+        echo "Removing existing $DOCKER_REPO_NAME containers..."
+        docker ps -a | grep $DOCKER_REPO_NAME | awk '{print $1}' | xargs docker rm -f
+        t=60
+        # Wait max 60 sec for containers to be removed
+        while [[ $t -gt 0 ]] && [[ -n "$(docker ps| grep $DOCKER_REPO_NAME)" ]]; do
+            sleep 1
+            let t=t-1
+        done
+    fi
+
+
+    # Remove existing images if exist
+    if [[ -n "$(docker images | grep $DOCKER_REPO_NAME)" ]]; then
+        echo "Docker images to remove:"
+        docker images | head -1 && docker images | grep $DOCKER_REPO_NAME
+        image_ids=($(docker images | grep $DOCKER_REPO_NAME | awk '{print $3}'))
+        for id in "${image_ids[@]}"; do
+            if [[ -n "$(docker images|grep $DOCKER_REPO_NAME|grep $id)" ]]; then
+                echo "Removing docker image $DOCKER_REPO_NAME:$id..."
+                docker rmi -f $id
+            fi
+        done
+    fi
+}
+
+
 count=30 # docker build jobs might take up to ~30 min
 while [[ -n `ps -ef| grep 'docker build' | grep $DOCKER_REPO_NAME | grep -v grep` ]]; do
     echo "Build or cleanup of $DOCKER_REPO_NAME in progress. Waiting..."
@@ -28,31 +58,8 @@ while [[ -n `ps -ef| grep 'docker build' | grep $DOCKER_REPO_NAME | grep -v grep
     fi
 done
 
-# Remove previous running containers if exist
-if [[ -n "$(docker ps -a | grep $DOCKER_REPO_NAME)" ]]; then
-    echo "Removing existing $DOCKER_REPO_NAME containers..."
-    docker ps -a | grep $DOCKER_REPO_NAME | awk '{print $1}' | xargs docker rm -f
-    t=60
-    # Wait max 60 sec for containers to be removed
-    while [[ $t -gt 0 ]] && [[ -n "$(docker ps| grep $DOCKER_REPO_NAME)" ]]; do
-        sleep 1
-        let t=t-1
-    done
-fi
-
-
-# Remove existing images if exist
-if [[ -n "$(docker images | grep $DOCKER_REPO_NAME)" ]]; then
-    echo "Docker images to remove:"
-    docker images | head -1 && docker images | grep $DOCKER_REPO_NAME
-    image_ids=($(docker images | grep $DOCKER_REPO_NAME | awk '{print $3}'))
-    for id in "${image_ids[@]}"; do
-        if [[ -n "$(docker images|grep $DOCKER_REPO_NAME|grep $id)" ]]; then
-            echo "Removing docker image $DOCKER_REPO_NAME:$id..."
-            docker rmi -f $id
-        fi
-    done
-fi
+# Remove the existing containers and images before building
+remove_containers_images
 
 cd "$WORKSPACE/$DOCKER_DIR" || exit 1
 HOST_ARCH="$(uname -m)"
@@ -117,3 +124,6 @@ if [[ "$PUSH_IMAGE" == "true" ]]; then
     echo
     docker push $DOCKER_REPO_NAME:$DOCKER_TAG
 fi
+
+# Remove the existing containers and images after building
+remove_containers_images