fuel: Enable all available scenarios for brahmaputra branch

This change
- Enables all available scenarios for stable/brahmaputra branch on LF POD2
and Ericsson POD1.
- Enables new node type, virtual, which is able to run available scenarios.
- Adjusts build blocker for multijob projects.
- Skips cloning of fuel repo for multijob projects.
- Enables os_odl-l2_ha auto trigger for LF POD2, Ericsson POD1, and Ericsson
POD2.
- Enables the use of an already downloaded ISO if the ISO was previously
downloaded to jumphost to speed up deployment.
- Adds a generic job for fuel for easier CI development/troubleshooting which
is kept disabled by default.
- Removes SDN_CONTROLLER parameter.

Change-Id: Icf26241216522b58db6908623a77405cdaac2ba2
Signed-off-by: Fatih Degirmenci <fatih.degirmenci@ericsson.com>

6 files changed, 214 insertions, 270 deletions

diff --git a/jjb/fuel/fuel-ci-jobs-master.yml b/jjb/fuel/fuel-ci-jobs-master.yml
deleted file mode 100644
index dffafe850..000000000
--- a/jjb/fuel/fuel-ci-jobs-master.yml
+++ /dev/null
@@ -1,154 +0,0 @@
-- project:
-
-    name: 'fuel-master'
-
-    project: 'fuel'
-
-    installer: '{project}'
-
-#--------------------------------
-# BRANCH ANCHORS
-#--------------------------------
-    master: &master
-        stream: master
-        branch: '{stream}'
-        gs-pathname: ''
-#--------------------------------
-# POD, INSTALLER, AND BRANCH MAPPING
-#--------------------------------
-    pod:
-        - ericsson-pod2:
-            <<: *master
-#--------------------------------
-
-# new scenario descriptions
-    scenario:
-        - 'os_ha'
-        - 'os_odl-l2_ha'
-        - 'os_odl-l3_ha'
-        - 'os_onos_ha'
-        - 'os_kvm_ha'
-        - 'os_ovs_ha'
-        - 'os_kvm_ovs_ha'
-
-    jobs:
-        - 'fuel-{scenario}-{pod}-daily-{stream}'
-        - 'fuel-deploy-{pod}-daily-{stream}-development'
-
-########################
-# job templates - new type of defining the scenario
-########################
-- job-template:
-    name: 'fuel-{scenario}-{pod}-daily-{stream}'
-
-    project-type: multijob
-
-    concurrent: false
-
-    properties:
-        - throttle:
-            enabled: true
-            max-total: 1
-            max-per-node: 1
-        - build-blocker:
-            use-build-blocker: true
-            blocking-jobs:
-                - 'fuel-os_.*'
-
-    wrappers:
-        - build-name:
-            name: '$BUILD_NUMBER - Scenario: $DEPLOY_SCENARIO'
-
-    parameters:
-        - project-parameter:
-            project: '{project}'
-        - '{pod}-defaults'
-        - '{installer}-defaults'
-        - string:
-            name: DEPLOY_SCENARIO
-            default: '{scenario}'
-        - fuel-ci-parameter:
-            gs-pathname: '{gs-pathname}'
-
-    scm:
-        - git-scm:
-            credentials-id: '{ssh-credentials}'
-            refspec: ''
-            branch: '{branch}'
-
-    builders:
-        - multijob:
-            name: deploy
-            condition: SUCCESSFUL
-            projects:
-                - name: 'fuel-deploy-{pod}-daily-{stream}'
-                  current-parameters: true
-                  git-revision: true
-                  kill-phase-on: FAILURE
-        - multijob:
-            name: functest
-            condition: COMPLETED
-            projects:
-                - name: 'functest-fuel-{pod}-daily-{stream}'
-                  current-parameters: true
-                  kill-phase-on: NEVER
-        - multijob:
-            name: yardstick
-            condition: COMPLETED
-            projects:
-                - name: 'yardstick-fuel-{pod}-daily-{stream}'
-                  current-parameters: true
-                  kill-phase-on: NEVER
-
-- job-template:
-    name: 'fuel-deploy-{pod}-daily-{stream}-development'
-
-    concurrent: false
-
-    properties:
-        - throttle:
-            enabled: true
-            max-total: 1
-            max-per-node: 1
-
-    parameters:
-        - project-parameter:
-            project: 'fuel'
-        - 'ericsson-pod2-defaults'
-        - 'fuel-defaults'
-        - fuel-ci-parameter-new:
-            gs-pathname: ''
-
-    scm:
-        - git-scm:
-            credentials-id: '{ssh-credentials}'
-            refspec: ''
-            branch: 'master'
-
-    wrappers:
-        - build-name:
-            name: '$BUILD_NUMBER - Scenario: $DEPLOY_SCENARIO'
-
-    builders:
-        - shell:
-            !include-raw-escape: ./fuel-download-artifact.sh
-        - shell:
-            !include-raw-escape: ./fuel-deploy-master.sh
-########################
-# parameter macros
-########################
-- parameter:
-    name: fuel-ci-parameter-new
-    parameters:
-        - string:
-            name: BUILD_DIRECTORY
-            default: $WORKSPACE/build_output
-            description: "Directory where the build artifact will be located upon the completion of the build."
-        - string:
-            name: CACHE_DIRECTORY
-            default: $HOME/opnfv/cache/$INSTALLER_TYPE
-            description: "Directory where the cache to be used during the build is located."
-        - string:
-            name: GS_URL
-            default: artifacts.opnfv.org/$PROJECT{gs-pathname}
-            description: "URL to Google Storage."
diff --git a/jjb/fuel/fuel-ci-jobs.yml b/jjb/fuel/fuel-ci-jobs.yml
index c90d930c1..6b28b565f 100644
--- a/jjb/fuel/fuel-ci-jobs.yml
+++ b/jjb/fuel/fuel-ci-jobs.yml
@@ -25,6 +25,8 @@
     pod:
         - opnfv-jump-2:
             <<: *brahmaputra
+        - virtual:
+            <<: *brahmaputra
 #--------------------------------
 #        master
 #--------------------------------
@@ -32,21 +34,36 @@
             <<: *master
         - ericsson-pod2:
             <<: *master
+        - virtual:
+            <<: *master
 #--------------------------------
 
-# please check the triggers before enabling any of the controllers!!!
-    sdn-controller:
-        - 'nosdn'
+# new scenario descriptions
+    scenario:
+        - 'os_ha':
+            auto-trigger-name: 'disabled-trigger'
+        - 'os_odl-l2_ha':
+            auto-trigger-name: '{scenario}-trigger'
+        - 'os_odl-l3_ha':
+            auto-trigger-name: 'disabled-trigger'
+        - 'os_onos_ha':
+            auto-trigger-name: 'disabled-trigger'
+        - 'os_kvm_ha':
+            auto-trigger-name: 'disabled-trigger'
+        - 'os_ovs_ha':
+            auto-trigger-name: 'disabled-trigger'
+        - 'os_kvm_ovs_ha':
+            auto-trigger-name: 'disabled-trigger'
 
     jobs:
-        - 'fuel-{sdn-controller}-{pod}-daily-{stream}'
+        - 'fuel-{scenario}-{pod}-daily-{stream}'
         - 'fuel-deploy-{pod}-daily-{stream}'
 
 ########################
 # job templates - old type of defining the scenario
 ########################
 - job-template:
-    name: 'fuel-{sdn-controller}-{pod}-daily-{stream}'
+    name: 'fuel-{scenario}-{pod}-daily-{stream}'
 
     project-type: multijob
 
@@ -60,14 +77,14 @@
         - build-blocker:
             use-build-blocker: true
             blocking-jobs:
-                - 'fuel-{sdn-controller}-{pod}-daily-{stream}'
+                - 'fuel-os_.*?-{pod}-daily-{stream}'
 
     wrappers:
         - build-name:
-            name: '$BUILD_NUMBER - SDN: $SDN_CONTROLLER Scenario: $DEPLOY_SCENARIO'
+            name: '$BUILD_NUMBER - Scenario: $DEPLOY_SCENARIO'
 
     triggers:
-        - 'fuel-{pod}-{sdn-controller}-trigger'
+        - '{auto-trigger-name}'
 
     parameters:
         - project-parameter:
@@ -75,20 +92,11 @@
         - '{pod}-defaults'
         - '{installer}-defaults'
         - string:
-            name: SDN_CONTROLLER
-            default: '{sdn-controller}'
-        - string:
             name: DEPLOY_SCENARIO
-            default: 'none'
+            default: '{scenario}'
         - fuel-ci-parameter:
             gs-pathname: '{gs-pathname}'
 
-    scm:
-        - git-scm:
-            credentials-id: '{ssh-credentials}'
-            refspec: ''
-            branch: '{branch}'
-
     builders:
         - multijob:
             name: deploy
@@ -96,7 +104,6 @@
             projects:
                 - name: 'fuel-deploy-{pod}-daily-{stream}'
                   current-parameters: true
-                  git-revision: true
                   kill-phase-on: FAILURE
         - multijob:
             name: functest
@@ -123,12 +130,6 @@
             enabled: true
             max-total: 1
             max-per-node: 1
-        - build-blocker:
-            use-build-blocker: true
-            blocking-jobs:
-                - 'fuel-deploy-{pod}-daily-{stream}.*'
-                - 'functest-fuel-{pod}-daily-{stream}'
-                - 'yardstick-fuel-{pod}-daily-{stream}'
 
     parameters:
         - project-parameter:
@@ -149,11 +150,15 @@
             name: '$BUILD_NUMBER - Scenario: $DEPLOY_SCENARIO'
 
     builders:
-        - 'builder-{stream}'
+        - shell:
+            !include-raw-escape: ./fuel-download-artifact.sh
+        - shell:
+            !include-raw-escape: ./fuel-deploy.sh
 
     publishers:
         - email:
             recipients: jonas.bjurel@ericsson.com stefan.k.berg@ericsson.com
+
 ########################
 # parameter macros
 ########################
@@ -172,37 +177,110 @@
             name: GS_URL
             default: artifacts.opnfv.org/$PROJECT{gs-pathname}
             description: "URL to Google Storage."
-########################
-# builder macros
-########################
-- builder:
-    name: 'builder-master'
-    builders:
-        - shell:
-            !include-raw: ./fuel-download-artifact.sh
-        - shell:
-            !include-raw: ./fuel-deploy-master.sh
-- builder:
-    name: 'builder-brahmaputra'
-    builders:
-        - shell:
-            !include-raw: ./fuel-download-artifact.sh
-        - shell:
-            !include-raw: ./fuel-deploy.sh
+- parameter:
+    name: 'virtual-defaults'
+    parameters:
+        - label:
+            name: SLAVE_LABEL
+            default: 'fuel-deploy-virtual'
+        - string:
+            name: GIT_BASE
+            default: https://gerrit.opnfv.org/gerrit/$PROJECT
+            description: 'Git URL to use on these Jenkins Slaves'
 ########################
 # trigger macros
 ########################
-# trigger for opnfv-jump-2 is set to run 1 hour ahead of others
-# to prevent doing unnecessary builds
-- trigger:
-    name: 'fuel-opnfv-jump-2-nosdn-trigger'
-    triggers:
-        - timed: '0 2 * * *'
+# os_odl-l2_ha trigger
 - trigger:
-    name: 'fuel-ericsson-pod1-nosdn-trigger'
+    name: 'os_odl-l2_ha-trigger'
     triggers:
         - timed: '0 2 * * *'
+
+# the rest of the scenarios disabled at the moment
 - trigger:
-    name: 'fuel-ericsson-pod2-nosdn-trigger'
+    name: 'disabled-trigger'
     triggers:
         - timed: ''
+
+# more granuler triggers based on the scenario and pod
+# commented out for the timebeing
+#- trigger:
+#    name: 'fuel-os_ha-opnfv-jump-2-trigger'
+#    triggers:
+#        - timed: ''
+#- trigger:
+#    name: 'fuel-os_odl-l3_ha-opnfv-jump-2-trigger'
+#    triggers:
+#        - timed: ''
+#- trigger:
+#    name: 'fuel-os_onos_ha-opnfv-jump-2-trigger'
+#    triggers:
+#        - timed: ''
+#- trigger:
+#    name: 'fuel-os_kvm_ha-opnfv-jump-2-trigger'
+#    triggers:
+#        - timed: ''
+#- trigger:
+#    name: 'fuel-os_ovs_ha-opnfv-jump-2-trigger'
+#    triggers:
+#        - timed: ''
+#- trigger:
+#    name: 'fuel-os_kvm_ovs_ha-opnfv-jump-2-trigger'
+#    triggers:
+#        - timed: ''
+#
+## ericsson-pod1 triggers - only os_odl-l2_ha trigger is enabled
+#- trigger:
+#    name: 'fuel-os_odl-l2_ha-ericsson-pod1-trigger'
+#    triggers:
+#        - timed: '0 2 * * *'
+#- trigger:
+#    name: 'fuel-os_ha-ericsson-pod1-trigger'
+#    triggers:
+#        - timed: ''
+#- trigger:
+#    name: 'fuel-os_odl-l3_ha-ericsson-pod1-trigger'
+#    triggers:
+#        - timed: ''
+#- trigger:
+#    name: 'fuel-os_onos_ha-ericsson-pod1-trigger'
+#    triggers:
+#        - timed: ''
+#- trigger:
+#    name: 'fuel-os_kvm_ha-ericsson-pod1-trigger'
+#    triggers:
+#        - timed: ''
+#- trigger:
+#    name: 'fuel-os_ovs_ha-ericsson-pod1-trigger'
+#    triggers:
+#        - timed: ''
+#- trigger:
+#    name: 'fuel-os_kvm_ovs_ha-ericsson-pod1-trigger'
+#    triggers:
+#        - timed: ''
+#
+## ericsson-pod2 triggers - only os_odl-l2_ha trigger is enabled
+#- trigger:
+#    name: 'fuel-os_odl-l2_ha-ericsson-pod2-trigger'
+#    triggers:
+#        - timed: '0 2 * * *'
+#- trigger:
+#    name: 'fuel-os_ha-ericsson-pod2-trigger'
+#    triggers:
+#        - timed: ''
+#- trigger:
+#    name: 'fuel-os_odl-l3_ha-ericsson-pod2-trigger'
+#    triggers:
+#        - timed: ''
+#- trigger:
+#    name: 'fuel-os_onos_ha-ericsson-pod2-trigger'
+#    triggers:
+#        - timed: ''
+#- trigger:
+#    name: 'fuel-os_kvm_ha-ericsson-pod2-trigger'
+#    triggers:
+#        - timed: ''
+#- trigger:
+#    name: 'fuel-os_ovs_ha-ericsson-pod2-trigger'
+#    triggers:
+#        - timed: ''
diff --git a/jjb/fuel/fuel-deploy-master.sh b/jjb/fuel/fuel-deploy-master.sh
deleted file mode 100755
index 4a7d1d9fb..000000000
--- a/jjb/fuel/fuel-deploy-master.sh
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/bin/bash
-set -o errexit
-set -o nounset
-set -o pipefail
-
-# source the file so we get OPNFV vars
-source latest.properties
-
-# echo the info about artifact that is used during the deployment
-echo "Using $OPNFV_ARTIFACT_URL for deployment"
-
-# checkout the commit that was used for building the downloaded artifact
-# to make sure the ISO and deployment mechanism uses same versions
-echo "Checking out $OPNFV_GIT_SHA1"
-git checkout $OPNFV_GIT_SHA1 --quiet
-
-# set deployment parameters
-BRIDGE=pxebr
-export TMPDIR=$HOME/tmpdir
-LAB_NAME=${NODE_NAME/-*}
-POD_NAME=${NODE_NAME/*-}
-
-# create TMPDIR if it doesn't exist
-mkdir -p $TMPDIR
-
-# change permissions down to TMPDIR
-chmod a+x $HOME
-chmod a+x $TMPDIR
-
-# clone the securedlab repo
-cd $WORKSPACE
-echo "Cloning securedlab repo"
-git clone ssh://jenkins-ericsson@gerrit.opnfv.org:29418/securedlab --quiet
-
-# construct the command
-DEPLOY_COMMAND="sudo $WORKSPACE/ci/deploy.sh -b file://$WORKSPACE/securedlab -l $LAB_NAME -p $POD_NAME -s $DEPLOY_SCENARIO -i file://$WORKSPACE/opnfv.iso -H -B $BRIDGE -S $TMPDIR"
-
-# log info to console
-echo "Deployment parameters"
-echo "--------------------------------------------------------"
-echo "Scenario: $DEPLOY_SCENARIO"
-echo "Lab: $LAB_NAME"
-echo "POD: $POD_NAME"
-echo "ISO: $(echo $OPNFV_ARTIFACT_URL | cut -d'/' -f3)"
-echo
-echo "Starting the deployment using $INSTALLER_TYPE. This could take some time..."
-echo "--------------------------------------------------------"
-echo
-
-# start the deployment
-echo "Issuing command"
-echo "$DEPLOY_COMMAND"
-echo
-
-$DEPLOY_COMMAND
-
-echo
-echo "--------------------------------------------------------"
-echo "Done!"
diff --git a/jjb/fuel/fuel-deploy.sh b/jjb/fuel/fuel-deploy.sh
index dc3d81a3a..6875a09e7 100755
--- a/jjb/fuel/fuel-deploy.sh
+++ b/jjb/fuel/fuel-deploy.sh
@@ -7,13 +7,28 @@ set -o pipefail
 source latest.properties
 
 # echo the info about artifact that is used during the deployment
-echo "Using $OPNFV_ARTIFACT_URL for deployment"
+echo "Using ${OPNFV_ARTIFACT_URL/*\/} for deployment"
 
 # checkout the commit that was used for building the downloaded artifact
 # to make sure the ISO and deployment mechanism uses same versions
 echo "Checking out $OPNFV_GIT_SHA1"
 git checkout $OPNFV_GIT_SHA1 --quiet
 
+# set deployment parameters
+BRIDGE=pxebr
+export TMPDIR=$HOME/tmpdir
+LAB_NAME=${NODE_NAME/-*}
+POD_NAME=${NODE_NAME/*-}
+
+if [[ "$NODE_NAME" =~ "virtual" ]]; then
+    POD_NAME="virtual_kvm"
+fi
+
+if [[ "$NODE_NAME" == "opnfv-jump-2" ]]; then
+    LAB_NAME="lf"
+    POD_NAME="pod2"
+fi
+
 # create TMPDIR if it doesn't exist
 export TMPDIR=$HOME/tmpdir
 mkdir -p $TMPDIR
@@ -22,19 +37,32 @@ mkdir -p $TMPDIR
 chmod a+x $HOME
 chmod a+x $TMPDIR
 
-# set BRIDGE
-BRIDGE=pxebr
+# clone the securedlab repo
+cd $WORKSPACE
+echo "Cloning securedlab repo"
+git clone ssh://jenkins-ericsson@gerrit.opnfv.org:29418/securedlab --quiet
+
+# construct the command
+DEPLOY_COMMAND="sudo $WORKSPACE/ci/deploy.sh -b file://$WORKSPACE/securedlab -l $LAB_NAME -p $POD_NAME -s $DEPLOY_SCENARIO -i file://$WORKSPACE/opnfv.iso -H -B $BRIDGE -S $TMPDIR"
 
 # log info to console
+echo "Deployment parameters"
+echo "--------------------------------------------------------"
+echo "Scenario: $DEPLOY_SCENARIO"
+echo "Lab: $LAB_NAME"
+echo "POD: $POD_NAME"
+echo "ISO: ${OPNFV_ARTIFACT_URL/*\/}"
+echo
 echo "Starting the deployment using $INSTALLER_TYPE. This could take some time..."
 echo "--------------------------------------------------------"
 echo
 
 # start the deployment
 echo "Issuing command"
-echo "sudo $WORKSPACE/ci/deploy.sh -iso $WORKSPACE/opnfv.iso -dea $POD_CONF_DIR/dea.yaml -dha $POD_CONF_DIR/dha.yaml -s $TMPDIR -b $BRIDGE -nh"
+echo "$DEPLOY_COMMAND"
+echo
 
-sudo $WORKSPACE/ci/deploy.sh -iso $WORKSPACE/opnfv.iso -dea $POD_CONF_DIR/dea.yaml -dha $POD_CONF_DIR/dha.yaml -s $TMPDIR -b $BRIDGE -nh
+$DEPLOY_COMMAND
 
 echo
 echo "--------------------------------------------------------"
diff --git a/jjb/fuel/fuel-download-artifact.sh b/jjb/fuel/fuel-download-artifact.sh
index 917bc97a0..655554127 100755
--- a/jjb/fuel/fuel-download-artifact.sh
+++ b/jjb/fuel/fuel-download-artifact.sh
@@ -16,9 +16,22 @@ fi
 # check if we got the file
 [[ -f latest.properties ]] || exit 1
 
-# source the file so we get OPNFV vars
+# source the file so we get artifact metadata
 source latest.properties
 
+# echo the info about artifact that is used during the deployment
+OPNFV_ARTIFACT=${OPNFV_ARTIFACT_URL/*\/}
+echo "Using $OPNFV_ARTIFACT for deployment"
+
+# check if we already have the ISO to avoid redownload
+ISO_STORE=$HOME/opnfv/iso_store/fuel
+if [[ -f "$ISO_STORE/$OPNFV_ARTIFACT" ]]; then
+    echo "ISO already exists. Skipping the download"
+    ln -s $ISO_STORE/$OPNFV_ARTIFACT $WORKSPACE/opnfv.iso
+    ls -al $WORKSPACE/opnfv.iso
+    exit 0
+fi
+
 # log info to console
 echo "Downloading the $INSTALLER_TYPE artifact using URL http://$OPNFV_ARTIFACT_URL"
 echo "This could take some time..."
diff --git a/jjb/fuel/fuel-project-jobs.yml b/jjb/fuel/fuel-project-jobs.yml
index a0064237b..40f41b883 100644
--- a/jjb/fuel/fuel-project-jobs.yml
+++ b/jjb/fuel/fuel-project-jobs.yml
@@ -21,6 +21,7 @@
         - 'fuel-verify-build-{stream}'
         - 'fuel-merge-build-{stream}'
         - 'fuel-merge-deploy-virtual-{stream}'
+        - 'fuel-deploy-generic-daily-{stream}'
 
 ########################
 # job templates
@@ -239,6 +240,43 @@
         - email:
             recipients: jonas.bjurel@ericsson.com stefan.k.berg@ericsson.com
 
+- job-template:
+    name: 'fuel-deploy-generic-daily-{stream}'
+
+    concurrent: false
+
+    disabled: true
+
+    properties:
+        - throttle:
+            enabled: true
+            max-total: 1
+            max-per-node: 1
+
+    parameters:
+        - project-parameter:
+            project: '{project}'
+        - 'ericsson-pod2-defaults'
+        - '{installer}-defaults'
+        - fuel-project-parameter:
+            gs-pathname: '{gs-pathname}'
+
+    scm:
+        - git-scm:
+            credentials-id: '{ssh-credentials}'
+            refspec: ''
+            branch: '{branch}'
+
+    wrappers:
+        - build-name:
+            name: '$BUILD_NUMBER - Scenario: $DEPLOY_SCENARIO'
+
+    builders:
+        - shell:
+            !include-raw-escape: ./fuel-download-artifact.sh
+        - shell:
+            !include-raw-escape: ./fuel-deploy.sh
+
 ########################
 # parameter macros
 ########################