Switch to project type MultiJob for Joid and structure jobs accordingly

Multijob project type has some advantages over the Freestyle project. [1]
- Jobs in same phase run in parallel.
- Jobs in another phase need to wait completion of jobs in earlier phase.
- Ability to create phases where different jobs can be run in parallel
  so a deploy phase can contain virtual and bare metal deployment jobs
  where both are running in parallel.
- Ability to determine what should be done when a job fails
- Ability to automatically abort other jobswhen a job is aborted. [2]
- Expose parameters from main job to jobs executed by it.
  (If a parameter is defined in sub job, it won't be overriden.
  That's why SDN_CONTROLLER and OPNFV_FEATURE parameters are not defined
  in sub jobs.)
- See full CI pipeline by just clicking the main job.
- Access the console logs of sub jobs from main job.

See example job from link [3].

Other important aspects of this change are
- One main job per installer, SDN controller and POD created. The purpose
  of the main job is to ease the scheduling and troubleshooting. The actual
  stuff is done by subjobs.
- Use same build, deploy, and test jobs for an installer and POD for running
  different combinations. The installer and test project scripts should do
  what needs to be done by determining the combination using SDN_CONTROLLER
  OPNFV_FEATURE env variables.
- Create and enable main installer jobs with supported SDN controllers by
  the specific installer which will automatically be triggered.
- The change requires us to increase no of executors per pod jumphost/slave
  by one since main and subjobs get executed on same slave due to passing
  parameters from main job to subjobs.
- Enabling additional OPNFV features can be done by using Build with Parameters
  manually and changing the OPNFV_FEATURE parameter before clicking the Build
  button.
- Use common naming scheme for jobs:
  Installer main jobs: [installer]-[controller]-[pod]-daily-[stream]
  Installer sub jobs: [installer]-[activity]-[pod]-daily-[stream]
  Test jobs: [test_project]-[installer]-[pod]-daily-[stream]

[1] Details of Multijob Plugin is available on below link.
https://wiki.jenkins-ci.org/display/JENKINS/Multijob+Plugin

[2] Available once the patch below gets merged.
https://review.openstack.org/#/c/257948/

[3] Example main job showing subjobs.
https://build.opnfv.org/ci/job/installer-controller-pod-daily-master/

Change-Id: Ie6ce2aaa7117583bc71e5153748a8e32f7d00351
Signed-off-by: Fatih Degirmenci <fatih.degirmenci@ericsson.com>

1 files changed, 92 insertions, 28 deletions

diff --git a/jjb/joid/joid.yml b/jjb/joid/joid.yml
index 28b019fcd..72fbcf850 100644
--- a/jjb/joid/joid.yml
+++ b/jjb/joid/joid.yml
@@ -5,39 +5,58 @@
 
     name: 'joid'
 
+    project: '{name}'
+
     installer: 'joid'
 
-    controller: 'odl'
+    sdn-controller:
+        - 'nosdn':
+            disabled: true
+        - 'odl':
+            disabled: false
+        - 'onos':
+            disabled: true
+        - 'opencontrail':
+            disabled: true
 
     pod:
         - 'intel-pod5'
         - 'orange-pod2'
 
+# only master branch is enabled at the moment to keep no of jobs sane
     stream:
         - master:
-            branch: 'master'
+            branch: '{stream}'
             gs-pathname: ''
+#        - brahmaputra:
+#            branch: 'stable/{stream}'
+#            gs-pathname: '{stream}'
 
     jobs:
-        - 'joid-{controller}-{pod}-daily-{stream}'
+        - 'joid-{sdn-controller}-{pod}-daily-{stream}'
         - 'joid-deploy-{pod}-daily-{stream}'
 
 ########################
 # job templates
 ########################
 - job-template:
-    name: 'joid-{controller}-{pod}-daily-{stream}'
+    name: 'joid-{sdn-controller}-{pod}-daily-{stream}'
 
-    disabled: false
+    project-type: multijob
+
+    disabled: '{obj:disabled}'
 
     concurrent: false
 
     parameters:
-        - project-parameter:
-            project: '{installer}'
-        - 'intel-us-build-1-defaults'
-        - '{installer}-defaults':
-            controller: '{controller}'
+        - '{pod}-defaults'
+        - '{installer}-defaults'
+        - string:
+            name: SDN_CONTROLLER
+            default: '{sdn-controller}'
+        - string:
+            name: OPNFV_FEATURE
+            default: 'none'
 
     scm:
         - git-scm:
@@ -46,12 +65,7 @@
             branch: '{branch}'
 
     builders:
-        - trigger-builds:
-          - project: 'joid-deploy-{pod}-daily-{stream}'
-            git-revision: true
-            block: true
-            predefined-parameters:
-                CONTROLLER={controller}
+        - '{installer}-multijob-{pod}-builder'
 
 - job-template:
     name: 'joid-deploy-{pod}-daily-{stream}'
@@ -62,14 +76,13 @@
 
     wrappers:
         - build-name:
-            name: '$BUILD_NUMBER: {installer} $CONTROLLER'
+            name: '$BUILD_NUMBER: $INSTALLER_TYPE $SDN_CONTROLLER'
 
     parameters:
         - project-parameter:
-            project: '{installer}'
+            project: '{project}'
         - '{pod}-defaults'
-        - '{installer}-defaults':
-            controller: '{controller}'
+        - '{installer}-defaults'
 
     scm:
         - git-scm:
@@ -81,6 +94,58 @@
         - '{pod}-builder'
 
 ########################
+# multijob builder macros
+########################
+# intel-pod5 is CI POD so full CI will run on it
+# deploy, functest, yardstick
+- builder:
+    name: joid-multijob-intel-pod5-builder
+    builders:
+        - multijob:
+            name: deploy
+            condition: SUCCESSFUL
+            projects:
+                - name: 'joid-deploy-{pod}-daily-{stream}'
+                  current-parameters: true
+                  git-revision: true
+                  kill-phase-on: FAILURE
+        - multijob:
+            name: functest
+            condition: COMPLETED
+            projects:
+                - name: 'functest-joid-{pod}-daily-{stream}'
+                  current-parameters: true
+                  kill-phase-on: NEVER
+# yardstick placeholder
+#        - multijob:
+#            name: yardstick
+#            condition: COMPLETED
+#            projects:
+#                - name: 'yardstick-joid-{pod}-daily-{stream}'
+#                  current-parameters: true
+#                  kill-phase-on: NEVER
+
+# orange-pod2 is NOT a CI POD so only the selected jobs will run
+# deploy and functest currently
+- builder:
+    name: joid-multijob-orange-pod2-builder
+    builders:
+        - multijob:
+            name: deploy
+            condition: SUCCESSFUL
+            projects:
+                - name: 'joid-deploy-{pod}-daily-{stream}'
+                  current-parameters: true
+                  git-revision: true
+                  kill-phase-on: FAILURE
+        - multijob:
+            name: functest
+            condition: COMPLETED
+            projects:
+                - name: 'functest-joid-{pod}-daily-{stream}'
+                  current-parameters: true
+                  kill-phase-on: NEVER
+########################
 # builder macros
 ########################
 - builder:
@@ -88,26 +153,25 @@
     builders:
         - shell: |
             #!/bin/bash
-            echo "Running $INSTALLER_TYPE with controller $CONTROLLER"
+            echo "Running $INSTALLER_TYPE with controller $SDN_CONTROLLER"
             echo "Please note that this is a quick try to see how joid deployment works"
             echo
             echo "Executing clean.sh"
             cd $WORKSPACE/ci
             ./clean.sh
-        - shell: |
-            cd $WORKSPACE/ci
+            echo
             echo "Executing 02-maasdeploy.sh intelpod5"
             ./02-maasdeploy.sh intelpod5
-        - shell: |
-            cd $WORKSPACE/ci
-            echo "Executing deploy.sh -o liberty -s $CONTROLLER -t ha -l intelpod5"
-            ./deploy.sh -o liberty -s $CONTROLLER -t ha -l intelpod5
+            echo
+            echo "Executing deploy.sh -o liberty -s $SDN_CONTROLLER -t ha -l intelpod5"
+            ./deploy.sh -o liberty -s $SDN_CONTROLLER -t ha -l intelpod5
+
 - builder:
     name: orange-pod2-builder
     builders:
         - shell: |
             #!/bin/bash
-            echo "Running $INSTALLER_TYPE with controller $CONTROLLER"
+            echo "Running $INSTALLER_TYPE with controller $SDN_CONTROLLER"
             echo "Please note that this is WIP generic builder"
             echo
             echo "Executing clean.sh"