Apex: Renables async builds and fixes blocking bug

Previously we had apex-verify-master running multiple instances on
virtual-slaves.  Apex-verify-master would kick off a build on our build
server, and then apex-verify-master would execute deploy multijob and
then functest multijob.  However we found a bug where jenkins build
blocker would see that a deploy finished on the virtual slave, and then
execute functest multijob as well as deploy multijob (for the next
verify job) at the same time.

This patch adds a parent job apex-virtual-{stream} which calls deploy
and functest multijob and will block correctly.  It also renables having
more than 1 apex-verify job running at a time on the virtual slaves.

Change-Id: Id15b2415407fc3318f333e3dfc59076d04db4ffb
Signed-off-by: Tim Rozet <trozet@redhat.com>

2 files changed, 170 insertions, 48 deletions

diff --git a/jjb/apex/apex.yml b/jjb/apex/apex.yml
index 12373f354..4162c9e70 100644
--- a/jjb/apex/apex.yml
+++ b/jjb/apex/apex.yml
@@ -6,6 +6,7 @@
         - 'apex-verify-gate-{stream}'
         - 'apex-verify-unit-tests-{stream}'
         - 'apex-runner-cperf-{stream}'
+        - 'apex-virtual-{stream}'
         - 'apex-deploy-{platform}-{stream}'
         - 'apex-daily-{stream}'
         - 'apex-csit-promote-daily-{stream}'
@@ -344,13 +345,8 @@
 
     properties:
         - logrotate-default
-        - build-blocker:
-            use-build-blocker: true
-            block-level: 'NODE'
-            blocking-jobs:
-                - 'apex-verify.*'
         - throttle:
-            max-per-node: 1
+            max-per-node: 3
             max-total: 10
             option: 'project'
 
@@ -392,7 +388,7 @@
             name: deploy-virtual
             condition: SUCCESSFUL
             projects:
-                - name: 'apex-deploy-virtual-{stream}'
+                - name: 'apex-virtual-{stream}'
                   current-parameters: false
                   predefined-parameters: |
                     ARTIFACT_VERSION=$ARTIFACT_VERSION
@@ -406,23 +402,6 @@
                   kill-phase-on: FAILURE
                   abort-all-job: true
                   git-revision: true
-        - multijob:
-            name: functest-smoke
-            condition: SUCCESSFUL
-            projects:
-              - name: 'functest-apex-virtual-suite-{stream}'
-                current-parameters: false
-                predefined-parameters: |
-                  DEPLOY_SCENARIO={verify-scenario}
-                  FUNCTEST_SUITE_NAME=healthcheck
-                  GERRIT_BRANCH=$GERRIT_BRANCH
-                  GERRIT_REFSPEC=$GERRIT_REFSPEC
-                  GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
-                  GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
-                node-parameters: true
-                kill-phase-on: NEVER
-                abort-all-job: true
-                git-revision: false
 
 # Verify Scenario Gate
 - job-template:
@@ -483,6 +462,7 @@
             block-level: 'NODE'
             blocking-jobs:
                 - 'apex-verify.*'
+                - 'apex-virtual.*'
         - throttle:
             max-per-node: 1
             max-total: 10
@@ -654,6 +634,87 @@
         - 'clean-workspace'
 
 
+# Virtual Deploy and Test
+- job-template:
+    name: 'apex-virtual-{stream}'
+
+    project-type: 'multijob'
+
+    disabled: false
+
+    scm:
+        - git-scm-gerrit
+
+    parameters:
+        - '{project}-defaults'
+        - '{project}-virtual-{stream}-defaults'
+        - project-parameter:
+            project: '{project}'
+            branch: '{branch}'
+        - apex-parameter:
+            gs-pathname: '{gs-pathname}'
+        - string:
+            name: DEPLOY_SCENARIO
+            default: '{verify-scenario}'
+            description: "Scenario to deploy with."
+        - string:
+            name: ARTIFACT_VERSION
+            default: dev
+            description: "Used for overriding the ARTIFACT_VERSION"
+    properties:
+        - logrotate-default
+        - build-blocker:
+            use-build-blocker: true
+            block-level: 'NODE'
+            blocking-jobs:
+                - 'apex-runner.*'
+                - 'apex-.*-promote.*'
+                - 'apex-run.*'
+                - 'apex-virtual-.*'
+                - 'apex-verify-gate-.*'
+        - throttle:
+            max-per-node: 1
+            max-total: 10
+            option: 'project'
+    builders:
+        - description-setter:
+            description: "Deployed on $NODE_NAME - Scenario: $DEPLOY_SCENARIO"
+        - multijob:
+            name: deploy-virtual
+            condition: SUCCESSFUL
+            projects:
+                - name: 'apex-deploy-virtual-{stream}'
+                  current-parameters: false
+                  predefined-parameters: |
+                    ARTIFACT_VERSION=$ARTIFACT_VERSION
+                    DEPLOY_SCENARIO=$DEPLOY_SCENARIO
+                    OPNFV_CLEAN=yes
+                    GERRIT_BRANCH=$GERRIT_BRANCH
+                    GERRIT_REFSPEC=$GERRIT_REFSPEC
+                    GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                    GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                  node-parameters: true
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+                  git-revision: true
+        - multijob:
+            name: functest-smoke
+            condition: SUCCESSFUL
+            projects:
+              - name: 'functest-apex-virtual-suite-{stream}'
+                current-parameters: false
+                predefined-parameters: |
+                  DEPLOY_SCENARIO=$DEPLOY_SCENARIO
+                  FUNCTEST_SUITE_NAME=healthcheck
+                  GERRIT_BRANCH=$GERRIT_BRANCH
+                  GERRIT_REFSPEC=$GERRIT_REFSPEC
+                  GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                  GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                node-parameters: true
+                kill-phase-on: NEVER
+                abort-all-job: true
+                git-revision: false
+
 # Baremetal Deploy and Test
 - job-template:
     name: 'apex-{scenario}-baremetal-{scenario_stream}'
diff --git a/jjb/apex/apex.yml.j2 b/jjb/apex/apex.yml.j2
index 736a43c91..a8fc70f0c 100644
--- a/jjb/apex/apex.yml.j2
+++ b/jjb/apex/apex.yml.j2
@@ -6,6 +6,7 @@
         - 'apex-verify-gate-{stream}'
         - 'apex-verify-unit-tests-{stream}'
         - 'apex-runner-cperf-{stream}'
+        - 'apex-virtual-{stream}'
         - 'apex-deploy-{platform}-{stream}'
         - 'apex-daily-{stream}'
         - 'apex-csit-promote-daily-{stream}'
@@ -214,13 +215,8 @@
 
     properties:
         - logrotate-default
-        - build-blocker:
-            use-build-blocker: true
-            block-level: 'NODE'
-            blocking-jobs:
-                - 'apex-verify.*'
         - throttle:
-            max-per-node: 1
+            max-per-node: 3
             max-total: 10
             option: 'project'
 
@@ -262,7 +258,7 @@
             name: deploy-virtual
             condition: SUCCESSFUL
             projects:
-                - name: 'apex-deploy-virtual-{stream}'
+                - name: 'apex-virtual-{stream}'
                   current-parameters: false
                   predefined-parameters: |
                     ARTIFACT_VERSION=$ARTIFACT_VERSION
@@ -276,23 +272,6 @@
                   kill-phase-on: FAILURE
                   abort-all-job: true
                   git-revision: true
-        - multijob:
-            name: functest-smoke
-            condition: SUCCESSFUL
-            projects:
-              - name: 'functest-apex-virtual-suite-{stream}'
-                current-parameters: false
-                predefined-parameters: |
-                  DEPLOY_SCENARIO={verify-scenario}
-                  FUNCTEST_SUITE_NAME=healthcheck
-                  GERRIT_BRANCH=$GERRIT_BRANCH
-                  GERRIT_REFSPEC=$GERRIT_REFSPEC
-                  GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
-                  GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
-                node-parameters: true
-                kill-phase-on: NEVER
-                abort-all-job: true
-                git-revision: false
 
 # Verify Scenario Gate
 - job-template:
@@ -353,6 +332,7 @@
             block-level: 'NODE'
             blocking-jobs:
                 - 'apex-verify.*'
+                - 'apex-virtual.*'
         - throttle:
             max-per-node: 1
             max-total: 10
@@ -524,6 +504,87 @@
         - 'clean-workspace'
 
 
+# Virtual Deploy and Test
+- job-template:
+    name: 'apex-virtual-{stream}'
+
+    project-type: 'multijob'
+
+    disabled: false
+
+    scm:
+        - git-scm-gerrit
+
+    parameters:
+        - '{project}-defaults'
+        - '{project}-virtual-{stream}-defaults'
+        - project-parameter:
+            project: '{project}'
+            branch: '{branch}'
+        - apex-parameter:
+            gs-pathname: '{gs-pathname}'
+        - string:
+            name: DEPLOY_SCENARIO
+            default: '{verify-scenario}'
+            description: "Scenario to deploy with."
+        - string:
+            name: ARTIFACT_VERSION
+            default: dev
+            description: "Used for overriding the ARTIFACT_VERSION"
+    properties:
+        - logrotate-default
+        - build-blocker:
+            use-build-blocker: true
+            block-level: 'NODE'
+            blocking-jobs:
+                - 'apex-runner.*'
+                - 'apex-.*-promote.*'
+                - 'apex-run.*'
+                - 'apex-virtual-.*'
+                - 'apex-verify-gate-.*'
+        - throttle:
+            max-per-node: 1
+            max-total: 10
+            option: 'project'
+    builders:
+        - description-setter:
+            description: "Deployed on $NODE_NAME - Scenario: $DEPLOY_SCENARIO"
+        - multijob:
+            name: deploy-virtual
+            condition: SUCCESSFUL
+            projects:
+                - name: 'apex-deploy-virtual-{stream}'
+                  current-parameters: false
+                  predefined-parameters: |
+                    ARTIFACT_VERSION=$ARTIFACT_VERSION
+                    DEPLOY_SCENARIO=$DEPLOY_SCENARIO
+                    OPNFV_CLEAN=yes
+                    GERRIT_BRANCH=$GERRIT_BRANCH
+                    GERRIT_REFSPEC=$GERRIT_REFSPEC
+                    GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                    GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                  node-parameters: true
+                  kill-phase-on: FAILURE
+                  abort-all-job: true
+                  git-revision: true
+        - multijob:
+            name: functest-smoke
+            condition: SUCCESSFUL
+            projects:
+              - name: 'functest-apex-virtual-suite-{stream}'
+                current-parameters: false
+                predefined-parameters: |
+                  DEPLOY_SCENARIO=$DEPLOY_SCENARIO
+                  FUNCTEST_SUITE_NAME=healthcheck
+                  GERRIT_BRANCH=$GERRIT_BRANCH
+                  GERRIT_REFSPEC=$GERRIT_REFSPEC
+                  GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                  GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                node-parameters: true
+                kill-phase-on: NEVER
+                abort-all-job: true
+                git-revision: false
+
 # Baremetal Deploy and Test
 - job-template:
     name: 'apex-{scenario}-baremetal-{scenario_stream}'