dovetail: offline support by making users to download docker images from artifacts

JIRA: DOVETAIL-433

1) bugfix of cleanup by using existed dovetail cleanup scripts, see logs
https://build.opnfv.org/ci/view/dovetail/job/dovetail-functest-artifacts-upload-master/3/console
2) opnfv/testapi:latest and mongo:3.5 images are needed for dovetail, added.

Change-Id: I647f7152b0122a5b792f6f18efbae0e537b81332
Signed-off-by: MatthewLi <matthew.lijun@huawei.com>

3 files changed, 15 insertions, 31 deletions

diff --git a/jjb/dovetail/dovetail-artifacts-upload.sh b/jjb/dovetail/dovetail-artifacts-upload.sh
index b23decad1..f1a9e7222 100755
--- a/jjb/dovetail/dovetail-artifacts-upload.sh
+++ b/jjb/dovetail/dovetail-artifacts-upload.sh
@@ -52,7 +52,7 @@ echo "signature Upload Complete!"
 
 upload () {
 # log info to console
-echo "Uploading to artifact. This could take some time..."
+echo "Uploading ${STORE_FILE_NAME} to artifact. This could take some time..."
 echo
 
 cd $WORKSPACE
diff --git a/jjb/dovetail/dovetail-artifacts-upload.yml b/jjb/dovetail/dovetail-artifacts-upload.yml
index 3d9af5ed7..0c8efbe0d 100644
--- a/jjb/dovetail/dovetail-artifacts-upload.yml
+++ b/jjb/dovetail/dovetail-artifacts-upload.yml
@@ -19,6 +19,8 @@
         - 'dovetail'
         - 'functest'
         - 'yardstick'
+        - 'testapi'
+        - 'mongo'
 
 #############################################
 # job template
@@ -55,7 +57,8 @@
 
     builders:
         - 'dovetail-builder-artifacts-upload'
-        - 'dovetail-workspace-cleanup'
+        - 'dovetail-upload-artifacts-cache-cleanup'
+        - 'dovetail-images-cleanup'
 
 ####################
 # parameter macros
@@ -94,7 +97,7 @@
             !include-raw: ./dovetail-artifacts-upload.sh
 
 - builder:
-    name: dovetail-workspace-cleanup
+    name: dovetail-upload-artifacts-cache-cleanup
     builders:
         - shell: |
             #!/bin/bash
@@ -104,27 +107,8 @@
 
             /bin/rm -rf $CACHE_DIR
 
-            # Remove previous running containers if exist
-            if [[ -n "$(docker ps -a | grep $DOCKER_REPO_NAME)" ]]; then
-                echo "Removing existing $DOCKER_REPO_NAME containers..."
-                docker ps -a | grep $DOCKER_REPO_NAME | awk '{print $1}' | xargs docker rm -f
-                t=60
-                # Wait max 60 sec for containers to be removed
-                while [[ $t -gt 0 ]] && [[ -n "$(docker ps| grep $DOCKER_REPO_NAME)" ]]; do
-                    sleep 1
-                    let t=t-1
-                done
-            fi
-
-            # Remove existing images if exist
-            if [[ -n "$(docker images | grep $DOCKER_REPO_NAME)" ]]; then
-                echo "Docker images to remove:"
-                docker images | head -1 && docker images | grep $DOCKER_REPO_NAME
-                image_tags=($(docker images | grep $DOCKER_REPO_NAME | awk '{print $2}'))
-                for tag in "${image_tags[@]}"; do
-                    if [[ -n "$(docker images|grep $DOCKER_REPO_NAME|grep $tag)" ]]; then
-                        echo "Removing docker image $DOCKER_REPO_NAME:$tag..."
-                        docker rmi -f $DOCKER_REPO_NAME:$tag
-                    fi
-                done
-            fi
+- builder:
+    name: dovetail-images-cleanup
+    builders:
+        - shell:
+            !include-raw: ./dovetail-cleanup.sh
diff --git a/jjb/dovetail/dovetail-cleanup.sh b/jjb/dovetail/dovetail-cleanup.sh
index 22b2ba2ce..0ee789a97 100755
--- a/jjb/dovetail/dovetail-cleanup.sh
+++ b/jjb/dovetail/dovetail-cleanup.sh
@@ -2,8 +2,8 @@
 
 [[ $CI_DEBUG == true ]] && redirect="/dev/stdout" || redirect="/dev/null"
 
-#clean up dependent project docker images, which has no containers and image tag None
-clean_images=(opnfv/functest opnfv/yardstick)
+# clean up dependent project docker images, which has no containers and image tag None
+clean_images=(opnfv/functest opnfv/yardstick opnfv/testapi mongo)
 for clean_image in "${clean_images[@]}"; do
     echo "Removing image $image_id, which has no containers and image tag is None"
     dangling_images=($(docker images -f "dangling=true" | grep ${clean_image} | awk '{print $3}'))
@@ -14,7 +14,7 @@ for clean_image in "${clean_images[@]}"; do
     fi
 done
 
-echo "Remove containers with image dovetail:<None>..."
+echo "Remove containers with image opnfv/dovetail:<None>..."
 dangling_images=($(docker images -f "dangling=true" | grep opnfv/dovetail | awk '{print $3}'))
 if [[ -n ${dangling_images} ]]; then
     for image_id in "${dangling_images[@]}"; do
@@ -24,7 +24,7 @@ if [[ -n ${dangling_images} ]]; then
     done
 fi
 
-echo "Cleaning up dovetail docker containers/images..."
+echo "Cleaning up dovetail docker containers..."
 if [[ ! -z $(docker ps -a | grep opnfv/dovetail) ]]; then
     echo "Removing existing opnfv/dovetail containers..."
     docker ps -a | grep opnfv/dovetail | awk '{print $1}' | xargs docker rm -f >${redirect}