[fuel] verify: Add baremetal testing on request

Duplicate fuel-verify jobs based on cluster type (virtual, baremetal)
and allow (re)checking changes using a specific scenario:
- ALL changes will trigger a virtual POD CI cycle (build, deploy,
  smoke) on submit, rebase etc., just like they used to;
- change comments containing a substring formatted like:
  * `recheck: os-odl-nofeature-ha`
  * `reverify: os-odl-nofeature-noha`
  * `recheck`
  will trigger only virtual or baremetal deployments of the required
  scenario (default os-nosdn-nofeature-noha) on:
  * virtual   PODs if scenario is NOHA;
  * baremetal PODs if scenario is HA;
- limit arm64 verify job running on arm-virtual2 to the installation
  of infrastructure VMs and skip cluster setup to prevent POD time
  starvation;

Limitations:
- HA scenarios can only be verified on manual request via a change
  comment (e.g. 'recheck: os-odl-ovs-ha');

JIRA: FUEL-390

Change-Id: I6368cfd045180d6a7202b8d817b17a4b123ba2af
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>

3 files changed, 116 insertions, 31 deletions

diff --git a/jjb/fuel/fuel-deploy.sh b/jjb/fuel/fuel-deploy.sh
index 9d71f0833..1a5d6a448 100755
--- a/jjb/fuel/fuel-deploy.sh
+++ b/jjb/fuel/fuel-deploy.sh
@@ -40,17 +40,20 @@ echo "Using configuration for ${LAB_NAME}"
 
 # create TMPDIR if it doesn't exist, change permissions
 mkdir -p "${TMPDIR}"
-chmod a+x "${HOME}" "${TMPDIR}"
+sudo chmod a+x "${HOME}" "${TMPDIR}"
 
 cd "${WORKSPACE}" || exit 1
 
 # log file name
 FUEL_LOG_FILENAME="${JOB_NAME}_${BUILD_NUMBER}.log.tar.gz"
 
+# Limited scope for vPOD verify jobs running on armband-virtual
+[[ ! "${JOB_NAME}" =~ verify-deploy-virtual-arm64 ]] || EXTRA_ARGS='-e'
+
 # construct the command
 DEPLOY_COMMAND="${SUDO} ${WORKSPACE}/ci/deploy.sh \
     -l ${LAB_NAME} -p ${POD_NAME} -s ${DEPLOY_SCENARIO} \
-    -S ${TMPDIR} \
+    -S ${TMPDIR} ${EXTRA_ARGS:-} \
     -L ${WORKSPACE}/${FUEL_LOG_FILENAME}"
 
 # log info to console
diff --git a/jjb/fuel/fuel-set-scenario.sh b/jjb/fuel/fuel-set-scenario.sh
new file mode 100755
index 000000000..4c8ed7334
--- /dev/null
+++ b/jjb/fuel/fuel-set-scenario.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+# SPDX-license-identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2018 SUSE, Mirantis Inc., Enea Software AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+set -o pipefail
+set -x
+
+#----------------------------------------------------------------------
+# This script is used by CI and executed by Jenkins jobs.
+# You are not supposed to use this script manually if you don't know
+# what you are doing.
+#----------------------------------------------------------------------
+
+# This function allows developers to specify the impacted scenario by
+# requesting a RE-check via a gerrit change comment under a specific format.
+#
+# Patterns to be searched in change comment:
+#   recheck: <scenario-name>
+#   reverify: <scenario-name>
+# Examples:
+#   recheck: os-odl-ovs-noha
+#   reverify: os-nosdn-nofeature-ha
+
+function set_scenario() {
+    # process gerrit event comment text (if present)
+    DEPLOY_SCENARIO=$(echo "${GERRIT_EVENT_COMMENT_TEXT}" | \
+                      grep -Po '(?!:(recheck|reverify):\s*)([-\w]+ha)')
+    if [ -z "${DEPLOY_SCENARIO}" ]; then
+        if [[ "$JOB_NAME" =~ baremetal ]]; then
+            DEPLOY_SCENARIO='os-nosdn-nofeature-ha'
+        else
+            DEPLOY_SCENARIO='os-nosdn-nofeature-noha'
+        fi
+    fi
+    # save the scenario names into java properties file
+    # so they can be injected to downstream jobs via envInject
+    echo "Recording the scenario '${DEPLOY_SCENARIO}' for downstream jobs"
+    echo "DEPLOY_SCENARIO=${DEPLOY_SCENARIO}" > "$WORK_DIRECTORY/scenario.properties"
+}
+
+# ensure GERRIT vars are set
+[ -n "${GERRIT_CHANGE_NUMBER}" ] || exit 1
+GERRIT_EVENT_COMMENT_TEXT="${GERRIT_EVENT_COMMENT_TEXT:-''}"
+
+# this directory is where the temporary properties file will be stored
+WORK_DIRECTORY=/tmp/$GERRIT_CHANGE_NUMBER
+/bin/rm -rf "$WORK_DIRECTORY" && mkdir -p "$WORK_DIRECTORY"
+
+set_scenario
diff --git a/jjb/fuel/fuel-verify-jobs.yaml b/jjb/fuel/fuel-verify-jobs.yaml
index 2a8fe3b5d..51f4afec4 100644
--- a/jjb/fuel/fuel-verify-jobs.yaml
+++ b/jjb/fuel/fuel-verify-jobs.yaml
@@ -22,28 +22,34 @@
     #####################################
     arch_tag:
       - 'amd64':
-          slave-label: 'fuel-virtual'
-          functest-suite-label: 'fuel-virtual'
+          slave-label: 'fuel'
+          functest-suite-label: 'fuel'
       - 'arm64':
-          slave-label: 'armband-virtual'
-          functest-suite-label: 'fuel-armband-virtual'
+          slave-label: 'armband'
+          functest-suite-label: 'fuel-armband'
+    #####################################
+    # cluster types
+    #####################################
+    type:
+      - 'virtual'
+      - 'baremetal'
     #####################################
     # patch verification phases
     #####################################
     phase:
       - 'docker-build'
-      - 'deploy-virtual'
+      - 'deploy'
     #####################################
     # jobs
     #####################################
     jobs:
-      - 'fuel-verify-{arch_tag}-{stream}'
-      - 'fuel-verify-{phase}-{arch_tag}-{stream}'
+      - 'fuel-verify-{type}-{arch_tag}-{stream}'
+      - 'fuel-verify-{phase}-{type}-{arch_tag}-{stream}'
 #####################################
 # job templates
 #####################################
 - job-template:
-    name: 'fuel-verify-{arch_tag}-{stream}'
+    name: 'fuel-verify-{type}-{arch_tag}-{stream}'
 
     project-type: multijob
 
@@ -60,7 +66,7 @@
       - build-blocker:
           use-build-blocker: true
           blocking-jobs:
-            - 'fuel-os-.*?-virtual-daily-.*'
+            - 'fuel-os-.*?-daily-.*'
             - 'fuel-verify-.*'
           block-level: 'NODE'
 
@@ -74,7 +80,7 @@
           fail: true
 
     triggers:
-      - 'fuel-verify-{arch_tag}-trigger':
+      - 'fuel-verify-{type}-{arch_tag}-trigger':
           project: '{project}'
           branch: '{branch}'
 
@@ -82,22 +88,22 @@
       - project-parameter:
           project: '{project}'
           branch: '{branch}'
-      - '{slave-label}-defaults':
+      - '{slave-label}-{type}-defaults':
           installer: '{installer}'
       - '{installer}-defaults':
           gs-pathname: '{gs-pathname}'
-      - string:
-          name: DEPLOY_SCENARIO
-          default: 'os-nosdn-nofeature-ha'
 
     builders:
+      - 'fuel-verify-set-scenario-macro'
+      - inject:
+          properties-file: "/tmp/$GERRIT_CHANGE_NUMBER/scenario.properties"
       - description-setter:
-          description: "Built on $NODE_NAME"
+          description: "Scenario: $DEPLOY_SCENARIO | Node: $NODE_NAME"
       - multijob:
           name: docker-build
           condition: SUCCESSFUL
           projects:
-            - name: 'fuel-verify-docker-build-{arch_tag}-{stream}'
+            - name: 'fuel-verify-docker-build-{type}-{arch_tag}-{stream}'
               current-parameters: false
               predefined-parameters: |
                 BRANCH=$BRANCH
@@ -109,13 +115,14 @@
               abort-all-job: true
 
       - multijob:
-          name: deploy-virtual
+          name: deploy
           condition: SUCCESSFUL
           projects:
-            - name: 'fuel-verify-deploy-virtual-{arch_tag}-{stream}'
+            - name: 'fuel-verify-deploy-{type}-{arch_tag}-{stream}'
               current-parameters: false
               predefined-parameters: |
                 MCP_DOCKER_TAG={arch_tag}-verify
+                DEPLOY_SCENARIO=$DEPLOY_SCENARIO
                 BRANCH=$BRANCH
                 GERRIT_REFSPEC=$GERRIT_REFSPEC
                 GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
@@ -129,7 +136,7 @@
           condition: SUCCESSFUL
           projects:
             # Use Functest job definition from jjb/functest/functest-daily-jobs
-            - name: 'functest-{functest-suite-label}-suite-{stream}'
+            - name: 'functest-{functest-suite-label}-{type}-suite-{stream}'
               current-parameters: false
               predefined-parameters: |
                 FUNCTEST_MODE=tier
@@ -144,7 +151,7 @@
               abort-all-job: true
 
 - job-template:
-    name: 'fuel-verify-{phase}-{arch_tag}-{stream}'
+    name: 'fuel-verify-{phase}-{type}-{arch_tag}-{stream}'
 
     disabled: '{obj:disabled}'
 
@@ -177,7 +184,7 @@
       - project-parameter:
           project: '{project}'
           branch: '{branch}'
-      - '{slave-label}-defaults':
+      - '{slave-label}-{type}-defaults':
           installer: '{installer}'
       - '{installer}-defaults':
           gs-pathname: '{gs-pathname}'
@@ -190,9 +197,9 @@
 # trigger macros
 ########################
 - trigger:
-    name: 'fuel-verify-amd64-trigger'
+    name: 'fuel-verify-virtual-amd64-trigger'
     triggers:
-      - gerrit: &fuel_verify_amd64_trigger
+      - gerrit: &fuel_verify_virtual_amd64_trigger
           server-name: 'gerrit.opnfv.org'
           trigger-on:
             - patchset-created-event:
@@ -201,9 +208,7 @@
                 exclude-no-code-change: 'false'
             - draft-published-event
             - comment-added-contains-event:
-                comment-contains-value: 'recheck'
-            - comment-added-contains-event:
-                comment-contains-value: 'reverify'
+                comment-contains-value: '(recheck|reverify)(\s|:\s*[-\w]+-noha)'
           projects:
             - project-compare-type: 'ANT'
               project-pattern: '{project}'
@@ -221,10 +226,28 @@
                   pattern: 'docs/**'
           readable-message: true
 - trigger:
-    name: 'fuel-verify-arm64-trigger'
+    name: 'fuel-verify-virtual-arm64-trigger'
+    triggers:
+      - gerrit:
+          <<: *fuel_verify_virtual_amd64_trigger
+          skip-vote:
+            successful: true
+            failed: true
+            unstable: true
+            notbuilt: true
+- trigger:
+    name: 'fuel-verify-baremetal-amd64-trigger'
+    triggers:
+      - gerrit: &fuel_verify_baremetal_amd64_trigger
+          <<: *fuel_verify_virtual_amd64_trigger
+          trigger-on:
+            - comment-added-contains-event:
+                comment-contains-value: '(recheck|reverify):\s*[-\w]+-ha'
+- trigger:
+    name: 'fuel-verify-baremetal-arm64-trigger'
     triggers:
       - gerrit:
-          <<: *fuel_verify_amd64_trigger
+          <<: *fuel_verify_baremetal_amd64_trigger
           skip-vote:
             successful: true
             failed: true
@@ -234,7 +257,12 @@
 # builder macros
 #####################################
 - builder:
-    name: 'fuel-verify-deploy-virtual-macro'
+    name: 'fuel-verify-set-scenario-macro'
+    builders:
+      - shell:
+          !include-raw: ./fuel-set-scenario.sh
+- builder:
+    name: 'fuel-verify-deploy-macro'
     builders:
       - shell:
           !include-raw: ./fuel-deploy.sh