Configure jenkins jobs for fuel

- verify jobs are configured in order to run builds and virtual deployments
for each and every new patch.
- merge jobs are configured in order to run builds and virtual deployments
for merged changes.
- daily jobs are not configured and currently disabled.
- builder scripts are taken out from yml and converted to standalone scripts.

Change-Id: I2fc8a5072a48ecabfb9b0460e3717c7bf3623ff4
Signed-off-by: Fatih Degirmenci <fatih.degirmenci@ericsson.com>

7 files changed, 319 insertions, 44 deletions

diff --git a/jjb/fuel/fuel-build.sh b/jjb/fuel/fuel-build.sh
new file mode 100755
index 000000000..3a041c546
--- /dev/null
+++ b/jjb/fuel/fuel-build.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# log info to console
+echo "Starting the build of $INSTALLER. This could take some time..."
+echo "--------------------------------------------------------"
+echo
+
+# create the cache directory if it doesn't exist
+[[ -d $CACHE_DIRECTORY ]] || mkdir -p $CACHE_DIRECTORY
+
+# set OPNFV_ARTIFACT_VERSION
+export OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d_%H-%M-%S")
+
+# start the build
+cd $WORKSPACE/$INSTALLER/ci
+./build.sh -v $OPNFV_ARTIFACT_VERSION -c file://$CACHE_DIRECTORY $BUILD_DIRECTORY
+
+# list the build artifacts
+ls -al $BUILD_DIRECTORY
+
+# save information regarding artifact into file
+(
+    echo "OPNFV_ARTIFACT_VERSION=$OPNFV_ARTIFACT_VERSION"
+    echo "OPNFV_GIT_URL=$(git config --get remote.origin.url)"
+    echo "OPNFV_GIT_SHA1=$(git rev-parse HEAD)"
+    echo "OPNFV_ARTIFACT_URL=$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso"
+    echo "OPNFV_ARTIFACT_MD5SUM=$(md5sum $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.iso | cut -d' ' -f1)"
+    echo "OPNFV_BUILD_URL=$BUILD_URL"
+) > $WORKSPACE/opnfv.properties
+
+echo
+echo "--------------------------------------------------------"
+echo "Done!"
diff --git a/jjb/fuel/fuel-download-artifact.sh b/jjb/fuel/fuel-download-artifact.sh
new file mode 100755
index 000000000..6eb1ba463
--- /dev/null
+++ b/jjb/fuel/fuel-download-artifact.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# get the latest.properties file in order to get info regarding latest artifact
+curl -s -o $WORKSPACE/latest.properties http://$GS_URL/latest.properties
+
+# check if we got the file
+[[ -f latest.properties ]] || exit 1
+
+# source the file so we get OPNFV vars
+source latest.properties
+
+# log info to console
+echo "Downloading the $INSTALLER artifact using URL http://$OPNFV_ARTIFACT_URL"
+echo "This could take some time..."
+echo "--------------------------------------------------------"
+echo
+
+# download the file
+curl -s -o $WORKSPACE/opnfv.iso http://$OPNFV_ARTIFACT_URL > gsutil.iso.log 2>&1
+
+# list the file
+ls -al $WORKSPACE/opnfv.iso
+
+echo
+echo "--------------------------------------------------------"
+echo "Done!"
diff --git a/jjb/fuel/fuel-upload-artifact.sh b/jjb/fuel/fuel-upload-artifact.sh
new file mode 100755
index 000000000..3b700c649
--- /dev/null
+++ b/jjb/fuel/fuel-upload-artifact.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# log info to console
+echo "Uploading the $INSTALLER artifact. This could take some time..."
+echo "--------------------------------------------------------"
+echo
+
+# source the opnfv.properties to get ARTIFACT_VERSION
+source $WORKSPACE/opnfv.properties
+
+# upload artifact and additional files to google storage
+gsutil cp $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.iso gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso > gsutil.iso.log 2>&1
+gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > gsutil.properties.log 2>&1
+gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/latest.properties > gsutil.latest.log 2>&1
+
+echo
+echo "--------------------------------------------------------"
+echo "Done!"
+echo "Artifact is available as http://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso"
diff --git a/jjb/fuel/fuel-virtual-deploy.sh b/jjb/fuel/fuel-virtual-deploy.sh
new file mode 100755
index 000000000..4e0b2dc47
--- /dev/null
+++ b/jjb/fuel/fuel-virtual-deploy.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+echo "Placeholder for Fuel virtual deploy"
diff --git a/jjb/fuel/fuel-workspace-cleanup.sh b/jjb/fuel/fuel-workspace-cleanup.sh
new file mode 100755
index 000000000..4e1a3a727
--- /dev/null
+++ b/jjb/fuel/fuel-workspace-cleanup.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# delete the $WORKSPACE to open some space
+/bin/rm -rf $WORKSPACE
diff --git a/jjb/fuel/fuel.yml b/jjb/fuel/fuel.yml
index e108051ea..a2a0257ef 100644
--- a/jjb/fuel/fuel.yml
+++ b/jjb/fuel/fuel.yml
@@ -1,24 +1,43 @@
+########################
+# Job configuration for fuel
+########################
 - project:
+
     name: fuel
+
+    project: 'fuel'
+
+    installer: 'fuel'
+
     jobs:
-        - 'fuel-verify'
-        - 'fuel-merge'
+        - 'fuel-verify-build-{stream}'
+        - 'fuel-verify-virtual-deploy-{stream}'
+        - 'fuel-merge-build-{stream}'
+        - 'fuel-merge-virtual-deploy-{stream}'
         - 'fuel-daily-{stream}'
 
-    # stream:    branch with - in place of / (eg. stable-arno)
-    # branch:    branch (eg. stable/arno)
     stream:
         - master:
             branch: 'master'
+            gs-pathname: ''
 
-    project: 'fuel'
+########################
+# job templates
+########################
 
 - job-template:
-    name: 'fuel-verify'
+    name: 'fuel-verify-build-{stream}'
+
+    project-type: freestyle
 
     node: ericsson-build
 
-    project-type: freestyle
+    concurrent: true
+
+    properties:
+        - throttle:
+            enabled: true
+            max-total: 3
 
     logrotate:
         daysToKeep: 30
@@ -30,11 +49,10 @@
         - project-parameter:
             project: '{project}'
         - gerrit-parameter:
-            branch: 'master'
-        - string:
-            name: GIT_BASE
-            default: https://gerrit.opnfv.org/gerrit/$PROJECT
-            description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW."
+            branch: '{branch}'
+        - fuel-parameter:
+            installer: '{installer}'
+            gs-pathname: '{gs-pathname}'
 
     scm:
         - gerrit-trigger-scm:
@@ -60,26 +78,101 @@
                     comment-contains-value: 'reverify'
             projects:
               - project-compare-type: 'ANT'
-                project-pattern: 'fuel'
+                project-pattern: '{project}'
                 branches:
                   - branch-compare-type: 'ANT'
-                    branch-pattern: '**/master'
+                    branch-pattern: '**/{branch}'
 
     builders:
         - shell:
-            echo "Hello World"
+            !include-raw ./fuel-build.sh
+        - shell:
+            !include-raw ./fuel-upload-artifact.sh
+#        - shell:
+#            !include-raw ./fuel-workspace-cleanup.sh
 
 - job-template:
-    name: 'fuel-merge'
+    name: 'fuel-verify-virtual-deploy-{stream}'
+
+    project-type: freestyle
+
+    node: ericsson-build
+
+    concurrent: true
+
+    properties:
+        - throttle:
+            enabled: true
+            max-total: 2
+            max-per-node: 1
+
+    logrotate:
+        daysToKeep: 30
+        numToKeep: 10
+        artifactDaysToKeep: -1
+        artifactNumToKeep: -1
+
+    parameters:
+        - project-parameter:
+            project: '{project}'
+        - gerrit-parameter:
+            branch: '{branch}'
+        - fuel-parameter:
+            installer: '{installer}'
+            gs-pathname: '{gs-pathname}'
 
-    # builder-merge job to run JJB update
-    #
-    # This job's purpose is to update all the JJB
+    scm:
+        - gerrit-trigger-scm:
+            credentials-id: '{ssh-credentials}'
+            refspec: '$GERRIT_REFSPEC'
+            choosing-strategy: 'gerrit'
 
-    node: master
+    wrappers:
+        - ssh-agent-credentials:
+            user: '{ssh-credentials}'
+
+    triggers:
+        - gerrit:
+            trigger-on:
+                - patchset-created-event:
+                    exclude-drafts: 'false'
+                    exclude-trivial-rebase: 'false'
+                    exclude-no-code-change: 'false'
+                - draft-published-event
+                - comment-added-contains-event:
+                    comment-contains-value: 'recheck'
+                - comment-added-contains-event:
+                    comment-contains-value: 'reverify'
+            projects:
+              - project-compare-type: 'ANT'
+                project-pattern: '{project}'
+                branches:
+                  - branch-compare-type: 'ANT'
+                    branch-pattern: '**/{branch}'
+            dependency-jobs: 'fuel-verify-build-{stream}'
+
+    builders:
+        - shell:
+            !include-raw ./fuel-download-artifact.sh
+        - shell:
+            !include-raw ./fuel-virtual-deploy.sh
+#        - shell:
+#            !include-raw ./fuel-workspace-cleanup.sh
+
+- job-template:
+    name: 'fuel-merge-build-{stream}'
 
     project-type: freestyle
 
+    node: ericsson-build
+
+    concurrent: true
+
+    properties:
+        - throttle:
+            enabled: true
+            max-total: 2
+
     logrotate:
         daysToKeep: 30
         numToKeep: 40
@@ -90,13 +183,70 @@
         - project-parameter:
             project: '{project}'
         - gerrit-parameter:
-            branch: 'master'
-        - string:
-            name: GIT_BASE
-            default: https://gerrit.opnfv.org/gerrit/$PROJECT
-            description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW."
+            branch: '{branch}'
+        - fuel-parameter:
+            installer: '{installer}'
+            gs-pathname: ''
+    scm:
+        - gerrit-trigger-scm:
+            credentials-id: '{ssh-credentials}'
+            refspec: ''
+            choosing-strategy: 'default'
 
+    wrappers:
+        - ssh-agent-credentials:
+            user: '{ssh-credentials}'
+
+    triggers:
+        - gerrit:
+            trigger-on:
+                - change-merged-event
+                - comment-added-contains-event:
+                    comment-contains-value: 'remerge'
+            projects:
+              - project-compare-type: 'ANT'
+                project-pattern: '{project}'
+                branches:
+                    - branch-compare-type: 'ANT'
+                      branch-pattern: '**/{branch}'
+
+    builders:
+        - shell:
+            !include-raw ./fuel-build.sh
+        - shell:
+            !include-raw ./fuel-upload-artifact.sh
+#        - shell:
+#            !include-raw ./fuel-workspace-cleanup.sh
 
+- job-template:
+    name: 'fuel-merge-virtual-deploy-{stream}'
+
+    project-type: freestyle
+
+    node: ericsson-build
+
+    concurrent: true
+
+    properties:
+        - throttle:
+            enabled: true
+            max-total: 2
+            max-per-node: 1
+
+    logrotate:
+        daysToKeep: 30
+        numToKeep: 40
+        artifactDaysToKeep: -1
+        artifactNumToKeep: 5
+
+    parameters:
+        - project-parameter:
+            project: '{project}'
+        - gerrit-parameter:
+            branch: '{branch}'
+        - fuel-parameter:
+            installer: '{installer}'
+            gs-pathname: ''
     scm:
         - gerrit-trigger-scm:
             credentials-id: '{ssh-credentials}'
@@ -115,25 +265,24 @@
                     comment-contains-value: 'remerge'
             projects:
               - project-compare-type: 'ANT'
-                project-pattern: 'fuel'
+                project-pattern: '{project}'
                 branches:
                     - branch-compare-type: 'ANT'
-                      branch-pattern: '**/master'
+                      branch-pattern: '**/{branch}'
+            dependency-jobs: 'fuel-merge-build-{stream}'
 
     builders:
         - shell:
-            echo "Hello World"
-
+            !include-raw ./fuel-download-artifact.sh
+        - shell:
+            !include-raw ./fuel-virtual-deploy.sh
+#        - shell:
+#            !include-raw ./fuel-workspace-cleanup.sh
 
 - job-template:
     name: 'fuel-daily-{stream}'
 
-    # Job template for daily builders
-    #
-    # Required Variables:
-    #     stream:    branch with - in place of / (eg. stable)
-    #     branch:    branch (eg. stable)
-    node: master
+    node: ericsson-build
 
     disabled: true
 
@@ -165,3 +314,30 @@
     builders:
         - shell:
              echo "Hello World"
+
+########################
+# parameter macros
+########################
+- parameter:
+    name: fuel-parameter
+    parameters:
+        - string:
+            name: INSTALLER
+            default: '{installer}'
+            description: "Installer to use."
+        - string:
+            name: BUILD_DIRECTORY
+            default: $WORKSPACE/build_output
+            description: "Directory where the build artifact will be located upon the completion of the build."
+        - string:
+            name: CACHE_DIRECTORY
+            default: $HOME/opnfv/cache/$INSTALLER
+            description: "Directory where the cache to be used during the build is located."
+        - string:
+            name: GIT_BASE
+            default: https://gerrit.opnfv.org/gerrit/$PROJECT
+            description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW."
+        - string:
+            name: GS_URL
+            default: artifacts.opnfv.org/$PROJECT{gs-pathname}
+            description: "URL to Google Storage."
diff --git a/jjb/genesis/genesis-fuel.yml b/jjb/genesis/genesis-fuel.yml
index d2ef49199..dc203efb0 100644
--- a/jjb/genesis/genesis-fuel.yml
+++ b/jjb/genesis/genesis-fuel.yml
@@ -60,7 +60,7 @@
             project: '{project}'
         - gerrit-parameter:
             branch: '{branch}'
-        - fuel-parameter:
+        - genesis-fuel-parameter:
             installer: '{installer}'
             gs-pathname: '{gs-pathname}'
 
@@ -146,7 +146,7 @@
             project: '{project}'
         - gerrit-parameter:
             branch: '{branch}'
-        - fuel-parameter:
+        - genesis-fuel-parameter:
             installer: '{installer}'
             gs-pathname: ''
 
@@ -216,7 +216,7 @@
     parameters:
         - project-parameter:
             project: '{project}'
-        - fuel-parameter:
+        - genesis-fuel-parameter:
             installer: '{installer}'
             gs-pathname: '{gs-pathname}'
 
@@ -283,7 +283,7 @@
     parameters:
         - project-parameter:
             project: '{project}'
-        - fuel-parameter:
+        - genesis-fuel-parameter:
             installer: '{installer}'
             gs-pathname: '{gs-pathname}'
         - string:
@@ -328,7 +328,7 @@
     parameters:
         - project-parameter:
             project: '{project}'
-        - fuel-parameter:
+        - genesis-fuel-parameter:
             installer: '{installer}'
             gs-pathname: '{gs-pathname}'
 
@@ -359,7 +359,7 @@
     parameters:
         - project-parameter:
             project: '{project}'
-        - fuel-parameter:
+        - genesis-fuel-parameter:
             installer: '{installer}'
             gs-pathname: '{gs-pathname}'
         - string:
@@ -405,7 +405,7 @@
     parameters:
         - project-parameter:
             project: '{project}'
-        - fuel-parameter:
+        - genesis-fuel-parameter:
             installer: '{installer}'
             gs-pathname: '{gs-pathname}'
         - string:
@@ -440,7 +440,7 @@
     parameters:
         - project-parameter:
             project: '{project}'
-        - fuel-parameter:
+        - genesis-fuel-parameter:
             installer: '{installer}'
             gs-pathname: ''
 
@@ -463,7 +463,7 @@
 # parameter macros
 ########################
 - parameter:
-    name: fuel-parameter
+    name: genesis-fuel-parameter
     parameters:
         - string:
             name: INSTALLER