Adding multijob builder and script for deploy of testapi

docker-update and generate-doc will run in opnfv-build defaults, and
docker-deploy will run in testresults.

Script will ensure this :-
Docker Images in testresults.opnfv.org
- N-1 docker image will be labeled as stable.
- N docker image will be latest.

Docker containers
- N-1 container will be stopped but not removed but will be removed in next iteration.

JIRA: FUNCTEST-689

Change-Id: I8c66ff716bae969b34fe698cdfa6140ef494ec49
Signed-off-by: rohitsakala <rohitsakala@gmail.com>

2 files changed, 189 insertions, 8 deletions

diff --git a/jjb/releng/testapi-automate.yml b/jjb/releng/testapi-automate.yml
index 47d217e51..4d450f35c 100644
--- a/jjb/releng/testapi-automate.yml
+++ b/jjb/releng/testapi-automate.yml
@@ -4,8 +4,18 @@
         - master:
             branch: '{stream}'
             gs-pathname: ''
+
+    phase:
+        - 'docker-update':
+            slave-label: 'opnfv-build'
+        - 'docker-deploy':
+            slave-label: 'testresults'
+        - 'generate-doc':
+            slave-label: 'opnfv-build'
+
     jobs:
         - 'testapi-automate-{stream}'
+        - 'testapi-automate-{phase}-{stream}'
         - 'testapi-verify-{stream}'
 
     project: 'releng'
@@ -15,6 +25,11 @@
 
     slave-label: 'testresults'
 
+    parameters:
+        - project-parameter:
+            project: '{project}'
+            branch: '{branch}'
+
     scm:
         - git-scm
 
@@ -80,10 +95,20 @@
                     healthy: 50
                     unhealthy: 40
                     failing: 30
+        - 'email-publisher'
 
 - job-template:
     name: 'testapi-automate-{stream}'
 
+    project-type: multijob
+
+    properties:
+        - throttle:
+            enabled: true
+            max-total: 1
+            max-per-node: 1
+            option: 'project'
+
     parameters:
         - project-parameter:
             project: '{project}'
@@ -97,6 +122,12 @@
     scm:
         - git-scm
 
+    wrappers:
+        - ssh-agent-wrapper
+        - timeout:
+            timeout: 360
+            fail: true
+
     triggers:
         - gerrit:
             server-name: 'gerrit.opnfv.org'
@@ -112,12 +143,57 @@
                     branch-pattern: '**/{branch}'
                 file-paths:
                   - compare-type: 'ANT'
-                    pattern: 'utils/**'
+                    pattern: 'utils/test/testapi/**'
 
     builders:
-        - docker-update
-        - testapi-doc-build
-        - upload-doc-artifact
+        - multijob:
+            name: docker-update
+            condition: SUCCESSFUL
+            projects:
+                - name: 'testapi-automate-docker-update-{stream}'
+                  current-parameters: true
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+        - multijob:
+            name: docker-deploy
+            condition: SUCCESSFUL
+            projects:
+                - name: 'testapi-automate-docker-deploy-{stream}'
+                  current-parameters: true
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+        - multijob:
+            name: generate-doc
+            condition: SUCCESSFUL
+            projects:
+                - name: 'testapi-automate-generate-doc-{stream}'
+                  current-parameters: true
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+
+    publishers:
+        - 'email-publisher'
+
+- job-template:
+    name: 'testapi-automate-{phase}-{stream}'
+
+    properties:
+        - throttle:
+            enabled: true
+            max-per-node: 1
+            option: 'project'
+
+    wrappers:
+        - ssh-agent-wrapper
+        - timeout:
+            timeout: 120
+            fail: true
+
+    scm:
+        - git-scm
+
+    builders:
+        - 'testapi-automate-{phase}-macro'
 
 ################################
 # job builders
@@ -129,25 +205,49 @@
             bash ./jjb/releng/testapi-backup-mongodb.sh
 
 - builder:
-    name: run-unit-tests
+    name: 'run-unit-tests'
     builders:
         - shell: |
             bash ./utils/test/testapi/run_test.sh
 
 - builder:
-    name: docker-update
+    name: 'testapi-automate-docker-update-macro'
     builders:
         - shell: |
             bash ./jjb/releng/testapi-docker-update.sh
 
 - builder:
-    name: testapi-doc-build
+    name: 'testapi-automate-generate-doc-macro'
+    builders:
+        - 'testapi-doc-build'
+        - 'upload-doc-artifact'
+
+- builder:
+    name: 'testapi-doc-build'
     builders:
         - shell: |
             bash ./utils/test/testapi/htmlize/doc-build.sh
 
 - builder:
-    name: upload-doc-artifact
+    name: 'upload-doc-artifact'
     builders:
         - shell: |
             bash ./utils/test/testapi/htmlize/push-doc-artifact.sh
+
+- builder:
+    name: 'testapi-automate-docker-deploy-macro'
+    builders:
+        - shell: |
+            bash ./jjb/releng/testapi-docker-deploy.sh
+
+################################
+# job publishers
+################################
+
+- publisher:
+    name: 'email-publisher'
+    publishers:
+        - email:
+            recipients: rohitsakala@gmail.com serena.feng.711@gmail.com
+            notify-every-unstable-build: false
+            send-to-individuals: true
\ No newline at end of file
diff --git a/jjb/releng/testapi-docker-deploy.sh b/jjb/releng/testapi-docker-deploy.sh
new file mode 100644
index 000000000..04d71f76e
--- /dev/null
+++ b/jjb/releng/testapi-docker-deploy.sh
@@ -0,0 +1,81 @@
+#!/bin/bash
+
+function check() {
+
+    # Verify hosted
+    sleep 5
+    cmd=`curl -s --head  --request GET http://testresults.opnfv.org/auto/swagger/spec | grep '200 OK' > /dev/null`
+    rc=$?
+    echo $rc
+
+    if [[ $rc == 0 ]]
+    then
+        return 0
+    else
+        return 1
+    fi
+
+}
+
+echo "Getting contianer Id of the currently running one"
+contId=$(sudo docker ps | grep "opnfv/testapi:latest" | awk '{print $1}')
+
+echo "Pulling the latest image"
+sudo docker pull opnfv/testapi:latest
+
+echo "Deleting old containers of opnfv/testapi:old"
+sudo docker ps -a | grep "opnfv/testapi" | grep "old" | awk '{print $1}' | xargs -r sudo docker rm -f
+
+echo "Deleting old images of opnfv/testapi:latest"
+sudo docker images | grep "opnfv/testapi" | grep "old" | awk '{print $3}' | xargs -r sudo docker rmi -f
+
+
+if [[ -z "$contId" ]]
+then
+    echo "No running testapi container"
+
+    echo "Removing stopped testapi containers in the previous iterations"
+    sudo docker ps -f status=exited | grep "opnfv_testapi" | awk '{print $1}' | xargs -r sudo docker rm -f
+else
+    echo $contId
+
+    echo "Get the image id of the currently running conatiner"
+    currImgId=$(sudo docker ps | grep "$contId" | awk '{print $2}')
+    echo $currImgId
+
+    if [[ -z "$currImgId" ]]
+    then
+        echo "No image id found for the container id"
+        exit 1
+    fi
+
+    echo "Changing current image tag to old"
+    sudo docker tag "$currImgId" opnfv/testapi:old
+
+    echo "Removing stopped testapi containers in the previous iteration"
+    sudo docker ps -f status=exited | grep "opnfv_testapi" | awk '{print $1}' | xargs -r sudo docker rm -f
+
+    echo "Renaming the running container name to opnfv_testapi as to identify it."
+    sudo docker rename $contId opnfv_testapi
+
+    echo "Stop the currently running container"
+    sudo docker stop $contId
+fi
+
+echo "Running a container with the new image"
+sudo docker run -dti -p "8711:8000" -e "mongodb_url=mongodb://172.17.0.1:27017" -e "swagger_url=http://testresults.opnfv.org/auto" opnfv/testapi:latest
+
+if check; then
+    echo "TestResults Hosted."
+else
+    echo "TestResults Hosting Failed"
+    if [[ $(sudo docker images | grep "opnfv/testapi" | grep "old" | awk '{print $3}') ]]; then
+        echo "Running old Image"
+        sudo docker run -dti -p "8711:8000" -e "mongodb_url=mongodb://172.17.0.1:27017" -e "swagger_url=http://testresults.opnfv.org/auto" opnfv/testapi:old
+        exit 1
+    fi
+fi
+
+# Echo Images and Containers
+sudo docker images
+sudo docker ps -a