Add merge job to Daisy4nfv CI

Change-Id: I42e828663af780ab596477741d1d34c985c86197
Signed-off-by: zeyu zhu <zhu.zeyu@zte.com.cn>

5 files changed, 449 insertions, 1 deletions

diff --git a/jjb/daisy4nfv/daisy4nfv-build.sh b/jjb/daisy4nfv/daisy4nfv-build.sh
index ec11db587..eb29fed72 100755
--- a/jjb/daisy4nfv/daisy4nfv-build.sh
+++ b/jjb/daisy4nfv/daisy4nfv-build.sh
@@ -4,11 +4,31 @@ echo "--------------------------------------------------------"
 echo "This is diasy4nfv build job!"
 echo "--------------------------------------------------------"
 
+# set OPNFV_ARTIFACT_VERSION
+if [[ "$JOB_NAME" =~ "merge" ]]; then
+    echo "Building Daisy4nfv ISO for a merged change"
+    export OPNFV_ARTIFACT_VERSION="gerrit-$GERRIT_CHANGE_NUMBER"
+else
+    export OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d_%H-%M-%S")
+fi
+
 # build output directory
 OUTPUT_DIR=$WORKSPACE/build_output
 mkdir -p $OUTPUT_DIR
 
 # start the build
 cd $WORKSPACE
-./ci/build.sh $OUTPUT_DIR
+./ci/build.sh $OUTPUT_DIR $OPNFV_ARTIFACT_VERSION
+
+# save information regarding artifact into file
+(
+    echo "OPNFV_ARTIFACT_VERSION=$OPNFV_ARTIFACT_VERSION"
+    echo "OPNFV_GIT_URL=$(git config --get remote.origin.url)"
+    echo "OPNFV_GIT_SHA1=$(git rev-parse HEAD)"
+    echo "OPNFV_ARTIFACT_URL=$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.bin"
+    echo "OPNFV_BUILD_URL=$BUILD_URL"
+) > $WORKSPACE/opnfv.properties
 
+echo
+echo "--------------------------------------------------------"
+echo "Done!"
diff --git a/jjb/daisy4nfv/daisy4nfv-download-artifact.sh b/jjb/daisy4nfv/daisy4nfv-download-artifact.sh
new file mode 100755
index 000000000..7ac76a5eb
--- /dev/null
+++ b/jjb/daisy4nfv/daisy4nfv-download-artifact.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+##############################################################################
+# Copyright (c) 2016 ZTE Coreporation and others.
+# hu.zhijiang@zte.com.cn
+# sun.jing22@zte.com.cn
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+set -o errexit
+set -o pipefail
+
+if [[ "$JOB_NAME" =~ "merge" ]]; then
+    echo "Downloading http://$GS_URL/opnfv-gerrit-$GERRIT_CHANGE_NUMBER.properties"
+    # get the properties file for the Daisy4nfv BIN built for a merged change
+    curl -s -o $WORKSPACE/latest.properties http://$GS_URL/opnfv-gerrit-$GERRIT_CHANGE_NUMBER.properties
+else
+    # get the latest.properties file in order to get info regarding latest artifact
+    echo "Downloading http://$GS_URL/latest.properties"
+    curl -s -o $WORKSPACE/latest.properties http://$GS_URL/latest.properties
+fi
+
+# check if we got the file
+[[ -f latest.properties ]] || exit 1
+
+# source the file so we get artifact metadata
+source latest.properties
+
+# echo the info about artifact that is used during the deployment
+OPNFV_ARTIFACT=${OPNFV_ARTIFACT_URL/*\/}
+echo "Using $OPNFV_ARTIFACT for deployment"
+
+# log info to console
+echo "Downloading the $INSTALLER_TYPE artifact using URL http://$OPNFV_ARTIFACT_URL"
+echo "This could take some time..."
+echo "--------------------------------------------------------"
+echo
+
+# download the file
+curl -s -o $WORKSPACE/opnfv.bin http://$OPNFV_ARTIFACT_URL > gsutil.bin.log 2>&1
+
+# list the file
+ls -al $WORKSPACE/opnfv.bin
+
+echo
+echo "--------------------------------------------------------"
+echo "Done!"
diff --git a/jjb/daisy4nfv/daisy4nfv-merge-jobs.yml b/jjb/daisy4nfv/daisy4nfv-merge-jobs.yml
new file mode 100644
index 000000000..f8c0258a6
--- /dev/null
+++ b/jjb/daisy4nfv/daisy4nfv-merge-jobs.yml
@@ -0,0 +1,270 @@
+- project:
+    name: 'daisy4nfv-merge-jobs'
+
+    project: 'daisy'
+#####################################
+# branch definitions
+#####################################
+    stream:
+        - master:
+            branch: '{stream}'
+            gs-pathname: ''
+            disabled: false
+#####################################
+# patch merge phases
+#####################################
+    phase:
+        - 'basic':
+            slave-label: 'opnfv-build-centos'
+        - 'build':
+            slave-label: 'opnfv-build-centos'
+        - 'deploy-virtual':
+            slave-label: 'opnfv-build-centos'
+        - 'smoke-test':
+            slave-label: 'opnfv-build-centos'
+        - 'promote':
+            slave-label: 'opnfv-build-centos'
+#####################################
+# jobs
+#####################################
+    jobs:
+        - 'daisy4nfv-merge-{stream}'
+        - 'daisy4nfv-merge-{phase}-{stream}'
+#####################################
+# job templates
+#####################################
+- job-template:
+    name: 'daisy4nfv-merge-{stream}'
+
+    project-type: multijob
+
+    disabled: false
+
+    concurrent: true
+
+    properties:
+        - throttle:
+            enabled: true
+            max-total: 4
+            option: 'project'
+
+    scm:
+        - gerrit-trigger-scm:
+            credentials-id: '{ssh-credentials}'
+            refspec: '$GERRIT_REFSPEC'
+            choosing-strategy: 'gerrit'
+
+    wrappers:
+        - ssh-agent-credentials:
+            users:
+                - '{ssh-credentials}'
+        - timeout:
+            timeout: 360
+            fail: true
+
+    triggers:
+        - gerrit:
+            trigger-on:
+                - change-merged-event
+                - comment-added-contains-event:
+                    comment-contains-value: 'remerge'
+            projects:
+              - project-compare-type: 'ANT'
+                project-pattern: '{project}'
+                branches:
+                  - branch-compare-type: 'ANT'
+                    branch-pattern: '**/{branch}'
+                forbidden-file-paths:
+                  - compare-type: ANT
+                    pattern: 'docs/**|.gitignore'
+            readable-message: true
+
+    parameters:
+        - project-parameter:
+            project: '{project}'
+        - gerrit-parameter:
+            branch: '{branch}'
+        - 'opnfv-build-defaults'
+        - 'daisy4nfv-merge-defaults':
+            gs-pathname: '{gs-pathname}'
+
+    builders:
+        - description-setter:
+            description: "Built on $NODE_NAME"
+        - multijob:
+            name: basic
+            condition: SUCCESSFUL
+            projects:
+                - name: 'daisy4nfv-merge-basic-{stream}'
+                  current-parameters: false
+                  predefined-parameters: |
+                    GERRIT_BRANCH=$GERRIT_BRANCH
+                    GERRIT_REFSPEC=$GERRIT_REFSPEC
+                    GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                    GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                  node-parameters: false
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+        - multijob:
+            name: build
+            condition: SUCCESSFUL
+            projects:
+                - name: 'daisy4nfv-merge-build-{stream}'
+                  current-parameters: false
+                  predefined-parameters: |
+                    GERRIT_BRANCH=$GERRIT_BRANCH
+                    GERRIT_REFSPEC=$GERRIT_REFSPEC
+                    GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                    GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                  node-parameters: false
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+        - multijob:
+            name: deploy-virtual
+            condition: SUCCESSFUL
+            projects:
+                - name: 'daisy4nfv-merge-deploy-virtual-{stream}'
+                  current-parameters: false
+                  predefined-parameters: |
+                    GERRIT_BRANCH=$GERRIT_BRANCH
+                    GERRIT_REFSPEC=$GERRIT_REFSPEC
+                    GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                    GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                  node-parameters: false
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+        - multijob:
+            name: smoke-test
+            condition: SUCCESSFUL
+            projects:
+                - name: 'daisy4nfv-merge-smoke-test-{stream}'
+                  current-parameters: false
+                  predefined-parameters: |
+                    GERRIT_BRANCH=$GERRIT_BRANCH
+                    GERRIT_REFSPEC=$GERRIT_REFSPEC
+                    GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                    GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                  node-parameters: false
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+        - multijob:
+            name: promote
+            condition: SUCCESSFUL
+            projects:
+                - name: 'daisy4nfv-merge-promote-{stream}'
+                  current-parameters: false
+                  predefined-parameters: |
+                    GERRIT_BRANCH=$GERRIT_BRANCH
+                    GERRIT_REFSPEC=$GERRIT_REFSPEC
+                    GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                    GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                  node-parameters: false
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+
+- job-template:
+    name: 'daisy4nfv-merge-{phase}-{stream}'
+
+    disabled: '{obj:disabled}'
+
+    concurrent: true
+
+    properties:
+        - throttle:
+            enabled: true
+            max-total: 6
+            option: 'project'
+        - build-blocker:
+            use-build-blocker: true
+            blocking-jobs:
+                - 'daisy4nfv-merge-deploy-.*'
+                - 'daisy4nfv-merge-test-.*'
+            block-level: 'NODE'
+
+    scm:
+        - gerrit-trigger-scm:
+            credentials-id: '{ssh-credentials}'
+            refspec: '$GERRIT_REFSPEC'
+            choosing-strategy: 'gerrit'
+
+    wrappers:
+        - ssh-agent-credentials:
+            users:
+                - '{ssh-credentials}'
+        - timeout:
+            timeout: 360
+            fail: true
+
+    parameters:
+        - project-parameter:
+            project: '{project}'
+        - gerrit-parameter:
+            branch: '{branch}'
+        - '{slave-label}-defaults'
+        - 'daisy4nfv-merge-defaults':
+            gs-pathname: '{gs-pathname}'
+
+    builders:
+        - description-setter:
+            description: "Built on $NODE_NAME"
+        - '{project}-merge-{phase}-macro'
+#####################################
+# builder macros
+#####################################
+- builder:
+    name: 'daisy-merge-basic-macro'
+    builders:
+        - shell:
+            !include-raw: ./daisy4nfv-basic.sh
+
+- builder:
+    name: 'daisy-merge-build-macro'
+    builders:
+        - shell:
+            !include-raw:
+                - ./daisy4nfv-build.sh
+                - ./daisy4nfv-upload-artifact.sh
+                - ./daisy4nfv-workspace-cleanup.sh
+
+- builder:
+    name: 'daisy-merge-deploy-virtual-macro'
+    builders:
+        - shell:
+            !include-raw:
+                - ./daisy4nfv-download-artifact.sh
+                - ./daisy4nfv-virtual-deploy.sh
+                - ./daisy4nfv-workspace-cleanup.sh
+
+- builder:
+    name: 'daisy-merge-smoke-test-macro'
+    builders:
+        - shell: |
+            #!/bin/bash
+
+            echo "Not activated!"
+
+- builder:
+    name: 'daisy-merge-promote-macro'
+    builders:
+        - shell: |
+            #!/bin/bash
+
+            echo "Not activated!"
+#####################################
+# parameter macros
+#####################################
+- parameter:
+    name: 'daisy4nfv-merge-defaults'
+    parameters:
+        - string:
+            name: BUILD_DIRECTORY
+            default: $WORKSPACE/build_output
+            description: "Directory where the build artifact will be located upon the completion of the build."
+        - string:
+            name: CACHE_DIRECTORY
+            default: $HOME/opnfv/cache/$INSTALLER_TYPE
+            description: "Directory where the cache to be used during the build is located."
+        - string:
+            name: GS_URL
+            default: artifacts.opnfv.org/$PROJECT{gs-pathname}
+            description: "URL to Google Storage."
diff --git a/jjb/daisy4nfv/daisy4nfv-upload-artifact.sh b/jjb/daisy4nfv/daisy4nfv-upload-artifact.sh
new file mode 100755
index 000000000..6b0aec54f
--- /dev/null
+++ b/jjb/daisy4nfv/daisy4nfv-upload-artifact.sh
@@ -0,0 +1,94 @@
+#!/bin/bash
+##############################################################################
+# Copyright (c) 2016 ZTE Coreporation and others.
+# hu.zhijiang@zte.com.cn
+# sun.jing22@zte.com.cn
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+set -o pipefail
+
+# check if we built something
+if [ -f $WORKSPACE/.noupload ]; then
+    echo "Nothing new to upload. Exiting."
+    /bin/rm -f $WORKSPACE/.noupload
+    exit 0
+fi
+
+# source the opnfv.properties to get ARTIFACT_VERSION
+source $WORKSPACE/opnfv.properties
+
+importkey () {
+# clone releng repository
+echo "Cloning releng repository..."
+[ -d releng ] && rm -rf releng
+git clone https://gerrit.opnfv.org/gerrit/releng $WORKSPACE/releng/ &> /dev/null
+#this is where we import the siging key
+if [ -f $WORKSPACE/releng/utils/gpg_import_key.sh ]; then
+  source $WORKSPACE/releng/utils/gpg_import_key.sh
+fi
+}
+
+signbin () {
+gpg2 -vvv --batch --yes --no-tty \
+  --default-key opnfv-helpdesk@rt.linuxfoundation.org  \
+  --passphrase besteffort \
+  --detach-sig $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.bin
+
+gsutil cp $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.bin.sig gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.bin.sig
+echo "BIN signature Upload Complete!"
+}
+
+uploadbin () {
+# log info to console
+echo "Uploading $INSTALLER_TYPE artifact. This could take some time..."
+echo
+
+cd $WORKSPACE
+# upload artifact and additional files to google storage
+gsutil cp $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.bin \
+    gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.bin > gsutil.bin.log 2>&1
+gsutil cp $WORKSPACE/opnfv.properties \
+    gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > gsutil.properties.log 2>&1
+if [[ ! "$JOB_NAME" =~ (verify|merge) ]]; then
+    gsutil cp $WORKSPACE/opnfv.properties \
+    gs://$GS_URL/latest.properties > gsutil.latest.log 2>&1
+elif [[ "$JOB_NAME" =~ "merge" ]]; then
+    echo "Uploaded Daisy4nfv BIN for a merged change"
+fi
+
+gsutil -m setmeta \
+    -h "Content-Type:text/html" \
+    -h "Cache-Control:private, max-age=0, no-transform" \
+    gs://$GS_URL/latest.properties \
+    gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > /dev/null 2>&1
+
+gsutil -m setmeta \
+    -h "Cache-Control:private, max-age=0, no-transform" \
+    gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.bin > /dev/null 2>&1
+
+# disabled errexit due to gsutil setmeta complaints
+#   BadRequestException: 400 Invalid argument
+# check if we uploaded the file successfully to see if things are fine
+gsutil ls gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.bin > /dev/null 2>&1
+if [[ $? -ne 0 ]]; then
+    echo "Problem while uploading artifact!"
+    echo "Check log $WORKSPACE/gsutil.bin.log on the machine where this build is done."
+    exit 1
+fi
+
+echo "Done!"
+echo
+echo "--------------------------------------------------------"
+echo
+echo "Artifact is available as http://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.bin"
+echo
+echo "--------------------------------------------------------"
+echo
+}
+
+importkey
+signbin
+uploadbin
diff --git a/jjb/daisy4nfv/daisy4nfv-workspace-cleanup.sh b/jjb/daisy4nfv/daisy4nfv-workspace-cleanup.sh
new file mode 100755
index 000000000..26f7e9a01
--- /dev/null
+++ b/jjb/daisy4nfv/daisy4nfv-workspace-cleanup.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+##############################################################################
+# Copyright (c) 2016 ZTE Coreporation and others.
+# hu.zhijiang@zte.com.cn
+# sun.jing22@zte.com.cn
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# delete the $WORKSPACE to open some space
+/bin/rm -rf $WORKSPACE