Apex: Enable overcloud log collection

With this commit:
https://gerrit.opnfv.org/gerrit/#/c/47875/

We can now gather logs from after at the end of a deploy/test job and
upload them to artifacts.  This patch enables that behavior on verify
and daily jobs.

Change-Id: If7af4cf1363e4bbcee78d98850af7c0e20250f3f
Signed-off-by: Tim Rozet <trozet@redhat.com>

4 files changed, 164 insertions, 2 deletions

diff --git a/jjb/apex/apex-fetch-logs.sh b/jjb/apex/apex-fetch-logs.sh
new file mode 100755
index 000000000..f25f456b1
--- /dev/null
+++ b/jjb/apex/apex-fetch-logs.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# log info to console
+echo "Fetching logs from overcloud. This could take some time..."
+echo "--------------------------------------------------------"
+echo
+
+if sudo opnfv-pyutil --fetch-logs; then
+  LOG_LOCATION=$(cat apex_util.log | grep 'Log retrieval complete' | grep -Eo '/tmp/.+$')
+  if [ -z "$LOG_LOCATION" ]; then
+      echo "WARNING: Unable to determine log location.  Logs will not be uploaded"
+      exit 0
+  else
+    UPLOAD_LOCATION="${GS_URL}/logs/${JOB_NAME}/${BUILD_NUMBER}/"
+    gsutil cp -r ${LOG_LOCATION} gs://${UPLOAD_LOCATION} > gsutil.latest_logs.log
+    echo "Logs available at https://${UPLOAD_LOCATION}/$(basename $LOG_LOCATION)"
+  fi
+else
+  echo "WARNING: Log retrieval failed.  No logs will be uploaded"
+  exit 0
+fi
diff --git a/jjb/apex/apex-verify-jobs.yml b/jjb/apex/apex-verify-jobs.yml
index 3a693c87e..689f68f44 100644
--- a/jjb/apex/apex-verify-jobs.yml
+++ b/jjb/apex/apex-verify-jobs.yml
@@ -358,6 +358,20 @@
               kill-phase-on: NEVER
               abort-all-job: true
               git-revision: false
+      - multijob:
+          name: apex-fetch-logs
+          condition: ALWAYS
+          projects:
+            - name: 'apex-fetch-logs-{stream}'
+              current-parameters: true
+              predefined-parameters: |
+                GERRIT_BRANCH=$GERRIT_BRANCH
+                GERRIT_REFSPEC=$GERRIT_REFSPEC
+                GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+              node-parameters: true
+              kill-phase-on: NEVER
+              abort-all-job: true
+              git-revision: false
 
 ########################
 # builder macros
diff --git a/jjb/apex/apex.yml b/jjb/apex/apex.yml
index a4c68600f..ed7ba7544 100644
--- a/jjb/apex/apex.yml
+++ b/jjb/apex/apex.yml
@@ -234,6 +234,35 @@
       - 'pike'
       - 'queens'
       - 'master'
+
+
+# Fetch Logs Job
+- job-template:
+    name: 'apex-fetch-logs-{stream}'
+
+    concurrent: true
+
+    disabled: false
+    scm:
+      - git-scm-gerrit
+    parameters:
+      - '{project}-{platform}-{stream}-defaults'
+      - project-parameter:
+          project: '{project}'
+          branch: '{branch}'
+      - apex-parameter:
+          gs-pathname: '{gs-pathname}'
+    # yamllint enable rule:line-length
+    properties:
+      - logrotate-default
+      - throttle:
+          max-per-node: 1
+          max-total: 10
+          option: 'project'
+
+    builders:
+      - 'apex-fetch-logs'
+
 - job-template:
     name: 'apex-runner-cperf-{stream}'
 
@@ -442,6 +471,21 @@
               kill-phase-on: NEVER
               abort-all-job: true
               git-revision: false
+      - multijob:
+          name: apex-fetch-logs
+          condition: ALWAYS
+          projects:
+            - name: 'apex-fetch-logs-{stream}'
+              current-parameters: false
+              predefined-parameters: |
+                GERRIT_BRANCH=$GERRIT_BRANCH
+                GERRIT_REFSPEC=$GERRIT_REFSPEC
+                GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+              node-parameters: true
+              kill-phase-on: NEVER
+              abort-all-job: true
+              git-revision: false
 
 # Baremetal Deploy and Test
 - job-template:
@@ -512,7 +556,19 @@
               kill-phase-on: NEVER
               abort-all-job: true
               git-revision: false
-
+      - multijob:
+          name: apex-fetch-logs
+          condition: ALWAYS
+          projects:
+            - name: 'apex-fetch-logs'
+              current-parameters: true
+              predefined-parameters: |
+                GERRIT_BRANCH=$GERRIT_BRANCH
+                GERRIT_REFSPEC=
+              node-parameters: true
+              kill-phase-on: NEVER
+              abort-all-job: true
+              git-revision: false
     publishers:
       - groovy-postbuild:
           script:
@@ -1826,6 +1882,12 @@
       - shell:
           !include-raw: ./apex-deploy.sh
 
+- builder:
+    name: 'apex-fetch-logs'
+    builders:
+      - shell:
+          !include-raw: ./apex-fetch-logs.sh
+
 #######################
 # trigger macros
 ########################
diff --git a/jjb/apex/apex.yml.j2 b/jjb/apex/apex.yml.j2
index 25b9cd9d7..b0fc0d451 100644
--- a/jjb/apex/apex.yml.j2
+++ b/jjb/apex/apex.yml.j2
@@ -76,6 +76,35 @@
       - 'pike'
       - 'queens'
       - 'master'
+
+
+# Fetch Logs Job
+- job-template:
+    name: 'apex-fetch-logs-{stream}'
+
+    concurrent: true
+
+    disabled: false
+    scm:
+      - git-scm-gerrit
+    parameters:
+      - '{project}-{platform}-{stream}-defaults'
+      - project-parameter:
+          project: '{project}'
+          branch: '{branch}'
+      - apex-parameter:
+          gs-pathname: '{gs-pathname}'
+    # yamllint enable rule:line-length
+    properties:
+      - logrotate-default
+      - throttle:
+          max-per-node: 1
+          max-total: 10
+          option: 'project'
+
+    builders:
+      - 'apex-fetch-logs'
+
 - job-template:
     name: 'apex-runner-cperf-{stream}'
 
@@ -284,6 +313,21 @@
               kill-phase-on: NEVER
               abort-all-job: true
               git-revision: false
+      - multijob:
+          name: apex-fetch-logs
+          condition: ALWAYS
+          projects:
+            - name: 'apex-fetch-logs-{stream}'
+              current-parameters: false
+              predefined-parameters: |
+                GERRIT_BRANCH=$GERRIT_BRANCH
+                GERRIT_REFSPEC=$GERRIT_REFSPEC
+                GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+              node-parameters: true
+              kill-phase-on: NEVER
+              abort-all-job: true
+              git-revision: false
 
 # Baremetal Deploy and Test
 - job-template:
@@ -354,7 +398,19 @@
               kill-phase-on: NEVER
               abort-all-job: true
               git-revision: false
-
+      - multijob:
+          name: apex-fetch-logs
+          condition: ALWAYS
+          projects:
+            - name: 'apex-fetch-logs'
+              current-parameters: true
+              predefined-parameters: |
+                GERRIT_BRANCH=$GERRIT_BRANCH
+                GERRIT_REFSPEC=
+              node-parameters: true
+              kill-phase-on: NEVER
+              abort-all-job: true
+              git-revision: false
     publishers:
       - groovy-postbuild:
           script:
@@ -997,6 +1053,12 @@
       - shell:
           !include-raw: ./apex-deploy.sh
 
+- builder:
+    name: 'apex-fetch-logs'
+    builders:
+      - shell:
+          !include-raw: ./apex-fetch-logs.sh
+
 #######################
 # trigger macros
 ########################