Moves lab-reconfig to execute on master

The problem is the lab reconfig includes a reboot of nodes in a POD and
since Jenkins slave is on one of those nodes, it will sever the jenkins
connection and fail the job.  Therefore we need to run the lab-reconfig
from master node, then execute the deploy on the slave when it is back
up.

JIRA: OCTO-109

Change-Id: I10724b95685f6dba2cd463e01ff7618de292c851
Signed-off-by: Tim Rozet <trozet@redhat.com>

1 files changed, 74 insertions, 2 deletions

diff --git a/jjb/genesis/genesis-foreman.yml b/jjb/genesis/genesis-foreman.yml
index dc22ab24b..e9a522438 100644
--- a/jjb/genesis/genesis-foreman.yml
+++ b/jjb/genesis/genesis-foreman.yml
@@ -15,6 +15,7 @@
         - 'genesis-foreman-daily-{stream}'
         - 'genesis-foreman-build-{stream}'
         - 'genesis-foreman-deploy-{stream}'
+        - 'genesis-foreman-deploy-runner-{stream}'
 
     # stream:    branch with - in place of / (eg. stable-arno)
     # branch:    branch (eg. stable/arno)
@@ -191,7 +192,7 @@
             git-revision: true
             block: true
         - trigger-builds:
-          - project: 'genesis-foreman-deploy-{stream}'
+          - project: 'genesis-foreman-deploy-runner-{stream}'
             git-revision: true
             block: true
         - trigger-builds:
@@ -264,9 +265,47 @@
             branch: '{branch}'
 
     builders:
-        - 'foreman-lab-reconfig'
         - 'foreman-deploy'
 
+- job-template:
+    name: 'genesis-foreman-deploy-runner-{stream}'
+
+    project-type: freestyle
+
+    disabled: false
+
+    node: master
+
+    logrotate:
+        daysToKeep: 30
+        numToKeep: 10
+        artifactDaysToKeep: -1
+        artifactNumToKeep: -1
+
+    parameters:
+        - project-parameter:
+            project: '{project}'
+        - foreman-parameter:
+            installer: '{installer}'
+            gs-pathname: '{gs-pathname}'
+        - string:
+            name: GIT_BASE
+            default: ssh://gerrit.opnfv.org:29418/$PROJECT
+            description: "Used for overriding the GIT URL coming from parameters macro."
+
+    scm:
+        - git-scm:
+            credentials-id: '{ssh-credentials}'
+            refspec: ''
+            branch: '{branch}'
+
+    builders:
+        - 'foreman-lab-reconfig'
+        - trigger-builds:
+          - project: 'genesis-foreman-deploy-{stream}'
+            git-revision: true
+            block: true
+
 ########################
 # parameter macros
 ########################
@@ -335,8 +374,41 @@
             source $WORKSPACE/releng/utils/lab-reconfiguration/venv/bin/activate
             # set nounset back again
             set -o nounset
+
+            # verify we are in venv
+            if [[ ! $(which python | grep venv) ]]; then
+              echo "Unable to activate venv...Exiting"
+              exit 1
+            fi
+
             python $WORKSPACE/releng/utils/lab-reconfiguration/reconfigUcsNet.py -i $ucs_host -u $ucs_user -p $ucs_password -f $WORKSPACE/releng/utils/lab-reconfiguration/foreman.yaml
 
+            # while undergoing reboot
+            sleep 30
+
+            # check to see if slave is back up
+            ping_counter=0
+            ping_flag=0
+            while [ "$ping_counter" -lt 20 ]; do
+              if [[ $(ping -c 5 172.30.10.72) ]]; then
+                ping_flag=1
+                break
+              fi
+              ((ping_counter++))
+              sleep 10
+            done
+
+            if [ "$ping_flag" -eq 1 ]; then
+              echo "Slave is pingable, now wait 60 seconds for services to start"
+              sleep 60
+            else
+              echo "Slave did not come back up after reboot: please check opnfv-jump-2"
+              exit 1
+            fi
+
+            set +o nounset
+            deactivate
+
             echo
             echo "--------------------------------------------------------"
             echo "Done!"