bifrost: Refactor bifrost jobs

This change
- Creates job templates to verify opnfv/bifrost patches
- Introduces type in order to prepare verification on baremetal
- Fixes git cloning so the job shows changes for the corresponding project
- Gets rid of unnecessary slave parameters from slave-params.yml
- Fixes build blocking
- Updates the bifrost-verify.sh script and includes in shell so we don't have
to clone releng repo separately
- Updates destroy-env.sh to create /httpboot and /tftpboot with permissions 755
- Removes daily infra-bifrost job and scripts  as we don't use them at the moment

Change-Id: Icb5a543628d3321bfe4e549ab92c67443ac578d8
Signed-off-by: Fatih Degirmenci <fatih.degirmenci@ericsson.com>

9 files changed, 195 insertions, 350 deletions

diff --git a/jjb/infra/bifrost-verify-jobs.yml b/jjb/infra/bifrost-verify-jobs.yml
new file mode 100644
index 000000000..b117b32ae
--- /dev/null
+++ b/jjb/infra/bifrost-verify-jobs.yml
@@ -0,0 +1,176 @@
+- project:
+    name: 'openstack-bifrost-verify'
+#--------------------------------
+# branches
+#--------------------------------
+    stream:
+        - master:
+            branch: '{stream}'
+#--------------------------------
+# projects
+#--------------------------------
+    project:
+        - 'openstack':
+            project-repo: 'https://git.openstack.org/openstack/bifrost'
+            clone-location: '/opt/bifrost'
+        - 'opnfv':
+            project-repo: 'https://gerrit.opnfv.org/gerrit/releng'
+            clone-location: '/opt/releng'
+#--------------------------------
+# distros
+#--------------------------------
+    distro:
+        - 'trusty':
+            disabled: false
+            dib-os-release: 'trusty'
+            dib-os-element: 'ubuntu-minimal'
+            dib-os-packages: 'openssh-server,vlan,vim,less,bridge-utils,language-pack-en,iputils-ping,rsyslog,curl'
+        - 'centos7':
+            disabled: false
+            dib-os-release: '7'
+            dib-os-element: 'centos-minimal'
+            dib-os-packages: 'openssh-server,vim,less,bridge-utils,iputils,rsyslog,curl'
+        - 'suse':
+            disabled: true
+            dib-os-release: 'suse'
+            dib-os-element: 'suse'
+            dib-os-packages: ''
+#--------------------------------
+# type
+#--------------------------------
+    type:
+        - virtual
+#--------------------------------
+# jobs
+#--------------------------------
+    jobs:
+        - '{project}-bifrost-verify-{distro}-{type}-{stream}'
+#--------------------------------
+# job templates
+#--------------------------------
+- job-template:
+    name: '{project}-bifrost-verify-{distro}-{type}-{stream}'
+
+    disabled: '{obj:disabled}'
+
+    concurrent: false
+
+    properties:
+        - build-blocker:
+            use-build-blocker: true
+            blocking-jobs:
+                - '.*-bifrost-verify.*-{type}'
+            block-level: 'NODE'
+
+    parameters:
+        - string:
+            name: PROJECT
+            default: '{project}'
+        - string:
+            name: PROJECT_REPO
+            default: '{project-repo}'
+        - string:
+            name: CLONE_LOCATION
+            default: '{clone-location}'
+        - string:
+            name: DISTRO
+            default: '{distro}'
+        - string:
+            name: DIB_OS_RELEASE
+            default: '{dib-os-release}'
+        - string:
+            name: DIB_OS_ELEMENT
+            default: '{dib-os-element}'
+        - string:
+            name: DIB_OS_PACKAGES
+            default: '{dib-os-packages}'
+        - string:
+            name: CLEAN_DIB_IMAGES
+            default: 'true'
+        - label:
+            name: SLAVE_LABEL
+            default: 'infra-{type}-{distro}'
+
+    scm:
+        - git:
+            url: '$PROJECT_REPO'
+            refspec: '$GERRIT_REFSPEC'
+            branches:
+                - 'origin/$GERRIT_BRANCH'
+            skip-tag: true
+            choosing-strategy: 'gerrit'
+            timeout: 10
+            wipe-workspace: true
+
+    triggers:
+        - '{project}-gerrit-trigger':
+            branch: '{branch}'
+
+    builders:
+        - description-setter:
+            description: "Built on $NODE_NAME"
+        - shell:
+            !include-raw-escape: ./bifrost-verify.sh
+
+    publishers:
+        - email:
+            recipients: fatih.degirmenci@ericsson.com yroblamo@redhat.com mchandras@suse.de jack.morgan@intel.com zhang.jun3g@zte.com.cn
+#--------------------------------
+# trigger macros
+#--------------------------------
+- trigger:
+    name: 'openstack-gerrit-trigger'
+    triggers:
+        - gerrit:
+            server-name: 'review.openstack.org'
+            silent-start: true
+            skip-vote:
+                successful: true
+                failed: true
+                unstable: true
+                notbuilt: true
+            escape-quotes: true
+            trigger-on:
+                - patchset-created-event:
+                    exclude-drafts: 'false'
+                    exclude-trivial-rebase: 'false'
+                    exclude-no-code-change: 'false'
+                - comment-added-contains-event:
+                    comment-contains-value: 'recheck'
+            projects:
+              - project-compare-type: 'PLAIN'
+                project-pattern: 'openstack/bifrost'
+                branches:
+                  - branch-compare-type: 'ANT'
+                    branch-pattern: '**/{branch}'
+                forbidden-file-paths:
+                  - compare-type: ANT
+                    pattern: 'doc/**'
+                  - compare-type: ANT
+                    pattern: 'releasenotes/**'
+            readable-message: true
+- trigger:
+    name: 'opnfv-gerrit-trigger'
+    triggers:
+        - gerrit:
+            server-name: 'gerrit.opnfv.org'
+            trigger-on:
+                - patchset-created-event:
+                    exclude-drafts: 'false'
+                    exclude-trivial-rebase: 'false'
+                    exclude-no-code-change: 'false'
+                - draft-published-event
+                - comment-added-contains-event:
+                    comment-contains-value: 'recheck'
+                - comment-added-contains-event:
+                    comment-contains-value: 'reverify'
+            projects:
+              - project-compare-type: 'ANT'
+                project-pattern: 'releng'
+                branches:
+                  - branch-compare-type: 'ANT'
+                    branch-pattern: '**/{branch}'
+                file-paths:
+                  - compare-type: ANT
+                    pattern: 'prototypes/bifrost/**'
+            readable-message: true
diff --git a/jjb/infra/openstack-bifrost-verify.sh b/jjb/infra/bifrost-verify.sh
index c17cb8861..759b50b8e 100755
--- a/jjb/infra/openstack-bifrost-verify.sh
+++ b/jjb/infra/bifrost-verify.sh
@@ -17,43 +17,38 @@ function fix_ownership() {
     if [ -z "${JOB_URL+x}" ]; then
         echo "Not running as part of Jenkins. Handle the logs manually."
     else
-        chown -R jenkins:jenkins $WORKSPACE
+        sudo chown -R jenkins:jenkins $WORKSPACE
     fi
 }
 
 # check distro to see if we support it
-# we will have centos and suse supported in future
-case "$DISTRO" in
-    trusty)
-        #start the test
-        echo "Starting provisioning of 3 VMs"
-        ;;
-    *)
-        echo "Distro $DISTRO is not supported!"
-        exit 1
-esac
+if [[ ! "$DISTRO" =~ (trusty|centos7|suse) ]]; then
+    echo "Distro $DISTRO is not supported!"
+    exit 1
+fi
 
 # remove previously cloned repos
-/bin/rm -rf /opt/bifrost /opt/puppet-infracloud /opt/stack /opt/releng
+sudo /bin/rm -rf /opt/bifrost /opt/puppet-infracloud /opt/stack /opt/releng
 
-# clone upstream bifrost repo and checkout the patch to verify
-git clone https://git.openstack.org/openstack/bifrost /opt/bifrost
-cd /opt/bifrost
-git fetch https://git.openstack.org/openstack/bifrost $GERRIT_REFSPEC && git checkout FETCH_HEAD
+# clone all the repos first and checkout the patch afterwards
+sudo git clone https://git.openstack.org/openstack/bifrost /opt/bifrost
+sudo git clone https://git.openstack.org/openstack-infra/puppet-infracloud /opt/puppet-infracloud
+sudo git clone https://gerrit.opnfv.org/gerrit/releng /opt/releng
 
-# clone puppet-infracloud
-git clone https://git.openstack.org/openstack-infra/puppet-infracloud /opt/puppet-infracloud
+# checkout the patch
+cd $CLONE_LOCATION
+sudo git fetch $PROJECT_REPO $GERRIT_REFSPEC && sudo git checkout FETCH_HEAD
 
 # combine opnfv and upstream scripts/playbooks
-cp -R $WORKSPACE/prototypes/bifrost/* /opt/bifrost/
+sudo /bin/cp -rf /opt/releng/prototypes/bifrost/* /opt/bifrost/
 
 # cleanup remnants of previous deployment
 cd /opt/bifrost
-./scripts/destroy-env.sh
+sudo -E ./scripts/destroy-env.sh
 
 # provision 3 VMs; jumphost, controller, and compute
 cd /opt/bifrost
-./scripts/test-bifrost-deployment.sh
+sudo -E ./scripts/test-bifrost-deployment.sh
 
 # list the provisioned VMs
 cd /opt/bifrost
diff --git a/jjb/infra/infra-daily-jobs.yml b/jjb/infra/infra-daily-jobs.yml
deleted file mode 100644
index d779d56d5..000000000
--- a/jjb/infra/infra-daily-jobs.yml
+++ /dev/null
@@ -1,166 +0,0 @@
-- project:
-    name: 'infra-daily-jobs'
-
-    project: 'releng'
-
-    installer: 'infra'
-#--------------------------------
-# BRANCH ANCHORS
-#--------------------------------
-    master: &master
-        stream: master
-        branch: '{stream}'
-        gs-pathname: ''
-#--------------------------------
-#        CI Slaves
-#--------------------------------
-    pod:
-        - virtual:
-            slave-label: infra-virtual-trusty
-            <<: *master
-#--------------------------------
-#       phases
-#--------------------------------
-    phase:
-        - 'provision'
-        - 'deploy'
-        - 'smoketest'
-#--------------------------------
-#       scenarios
-#--------------------------------
-    scenario:
-        - 'os-nosdn-nofeature-noha'
-#--------------------------------
-#       jobs
-#--------------------------------
-    jobs:
-        - 'infra-{scenario}-{pod}-daily-{stream}'
-        - 'infra-{phase}-{pod}-daily-{stream}'
-########################
-# job templates
-########################
-- job-template:
-    name: 'infra-{scenario}-{pod}-daily-{stream}'
-
-    concurrent: false
-
-    properties:
-        - build-blocker:
-            use-build-blocker: true
-            blocking-jobs:
-                - 'infra-os-.*?-{pod}-daily-{stream}'
-            block-level: 'NODE'
-
-    wrappers:
-        - build-name:
-            name: '$BUILD_NUMBER - Scenario: $DEPLOY_SCENARIO'
-
-    triggers:
-        - timed: 'H */3 * * *'
-
-    parameters:
-        - project-parameter:
-            project: '{project}'
-        - '{installer}-defaults'
-        - '{slave-label}-defaults'
-        - string:
-            name: DEPLOY_SCENARIO
-            default: '{scenario}'
-
-    builders:
-        - description-setter:
-            description: "Built on $NODE_NAME"
-        - trigger-builds:
-            - project: 'infra-provision-{pod}-daily-{stream}'
-              current-parameters: false
-              predefined-parameters:
-                DEPLOY_SCENARIO={scenario}
-              same-node: true
-              block: true
-        - trigger-builds:
-            - project: 'infra-deploy-{pod}-daily-{stream}'
-              current-parameters: false
-              predefined-parameters:
-                DEPLOY_SCENARIO={scenario}
-              same-node: true
-              block: true
-        - trigger-builds:
-            - project: 'infra-smoketest-{pod}-daily-{stream}'
-              current-parameters: false
-              predefined-parameters:
-                DEPLOY_SCENARIO={scenario}
-              block: true
-              same-node: true
-              block-thresholds:
-                build-step-failure-threshold: 'never'
-                failure-threshold: 'never'
-                unstable-threshold: 'FAILURE'
-
-    publishers:
-        - email:
-            recipients: fatih.degirmenci@ericsson.com yroblamo@redhat.com mchandras@suse.de jack.morgan@intel.com zhang.jun3g@zte.com.cn
-
-- job-template:
-    name: 'infra-{phase}-{pod}-daily-{stream}'
-
-    concurrent: false
-
-    properties:
-        - build-blocker:
-            use-build-blocker: true
-            blocking-jobs:
-                - 'infra-provision-{pod}-daily-{stream}'
-                - 'infra-deploy-{pod}-daily-{stream}'
-                - 'infra-smoketest-{pod}-daily-{stream}'
-            block-level: 'NODE'
-
-    parameters:
-        - project-parameter:
-            project: '{project}'
-        - '{installer}-defaults'
-        - '{slave-label}-defaults'
-        - string:
-            name: DEPLOY_SCENARIO
-            default: 'os-nosdn-nofeature-noha'
-        - string:
-            name: CLEAN_DIB_IMAGES
-            default: 'false'
-
-    scm:
-        - git-scm:
-            credentials-id: '{ssh-credentials}'
-            refspec: ''
-            branch: '{branch}'
-
-    wrappers:
-        - build-name:
-            name: '$BUILD_NUMBER - Scenario: $DEPLOY_SCENARIO'
-
-    builders:
-        - description-setter:
-            description: "Built on $NODE_NAME"
-        - 'infra-{phase}-daily-builder'
-#####################################
-# builder macros
-#####################################
-- builder:
-    name: 'infra-provision-daily-builder'
-    builders:
-        - shell: |
-            #!/bin/bash
-
-            sudo $WORKSPACE/jjb/infra/infra-provision.sh
-- builder:
-    name: 'infra-deploy-daily-builder'
-    builders:
-        - shell: |
-            #!/bin/bash
-
-            sudo $WORKSPACE/jjb/infra/infra-deploy.sh
-- builder:
-    name: 'infra-smoketest-daily-builder'
-    builders:
-        - shell: |
-            #!/bin/bash
-
-            sudo $WORKSPACE/jjb/infra/infra-smoketest.sh
diff --git a/jjb/infra/infra-deploy.sh b/jjb/infra/infra-deploy.sh
deleted file mode 100755
index 35ef9a1d0..000000000
--- a/jjb/infra/infra-deploy.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/bash
-
-echo "Not activated!"
diff --git a/jjb/infra/infra-provision.sh b/jjb/infra/infra-provision.sh
deleted file mode 100755
index 45ed3b928..000000000
--- a/jjb/infra/infra-provision.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/bash
-set -xe
-
-if [[ $(whoami) != "root" ]]; then
-    echo "Error: This script must be run as root!"
-    exit 1
-fi
-
-# remove previously cloned repos
-/bin/rm -rf /opt/bifrost /opt/puppet-infracloud /opt/releng
-
-# clone upstream repos
-git clone https://git.openstack.org/openstack/bifrost /opt/bifrost
-git clone https://git.openstack.org/openstack-infra/puppet-infracloud /opt/puppet-infracloud
-
-# clone opnfv releng repo
-git clone https://gerrit.opnfv.org/gerrit/releng /opt/releng
-
-# combine opnfv and upstream scripts/playbooks
-cp -R /opt/releng/prototypes/bifrost/* /opt/bifrost/
-
-# cleanup remnants of previous deployment
-cd /opt/bifrost
-./scripts/destroy-env.sh
-
-# provision 3 VMs; jumphost, controller, and compute
-cd /opt/bifrost
-./scripts/test-bifrost-deployment.sh
-
-# list the provisioned VMs
-cd /opt/bifrost
-source env-vars
-ironic node-list
-virsh list
diff --git a/jjb/infra/infra-smoketest.sh b/jjb/infra/infra-smoketest.sh
deleted file mode 100755
index 35ef9a1d0..000000000
--- a/jjb/infra/infra-smoketest.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/bash
-
-echo "Not activated!"
diff --git a/jjb/infra/openstack-bifrost-verify-jobs.yml b/jjb/infra/openstack-bifrost-verify-jobs.yml
deleted file mode 100644
index 8afe47cd1..000000000
--- a/jjb/infra/openstack-bifrost-verify-jobs.yml
+++ /dev/null
@@ -1,111 +0,0 @@
-- project:
-    name: 'openstack-bifrost-verify'
-
-    project: 'releng'
-#--------------------------------
-# branches
-#--------------------------------
-    stream:
-        - master:
-            branch: '{stream}'
-#--------------------------------
-# distros
-# jobs for centos7 and suse can be enabled once the support is there
-#--------------------------------
-    distro:
-        - 'trusty':
-            slave-label: infra-virtual-trusty
-            disabled: false
-        - 'centos7':
-            slave-label: infra-virtual-trusty
-            disabled: true
-        - 'suse':
-            slave-label: infra-virtual-trusty
-            disabled: true
-#--------------------------------
-# jobs
-#--------------------------------
-    jobs:
-        - 'openstack-bifrost-verify-{distro}-{stream}'
-#--------------------------------
-# job templates
-#--------------------------------
-- job-template:
-    name: 'openstack-bifrost-verify-{distro}-{stream}'
-
-    concurrent: false
-
-    disabled: '{obj:disabled}'
-
-    properties:
-        - build-blocker:
-            use-build-blocker: true
-            blocking-jobs:
-                - 'infra-os-.*?-daily-.*'
-            block-level: 'NODE'
-
-    parameters:
-        - project-parameter:
-            project: '{project}'
-        - string:
-            name: DISTRO
-            default: '{distro}'
-        - string:
-            name: CLEAN_DIB_IMAGES
-            default: 'true'
-        - '{slave-label}-defaults'
-
-    scm:
-        - git-scm:
-            credentials-id: '{ssh-credentials}'
-            refspec: ''
-            branch: '{branch}'
-
-    triggers:
-        - gerrit:
-            server-name: 'review.openstack.org'
-            silent-start: true
-            skip-vote:
-                successful: true
-                failed: true
-                unstable: true
-                notbuilt: true
-            escape-quotes: true
-            trigger-on:
-                - patchset-created-event:
-                    exclude-drafts: 'false'
-                    exclude-trivial-rebase: 'false'
-                    exclude-no-code-change: 'false'
-                - comment-added-contains-event:
-                    comment-contains-value: 'recheck'
-            projects:
-              - project-compare-type: 'PLAIN'
-                project-pattern: 'openstack/bifrost'
-                branches:
-                  - branch-compare-type: 'ANT'
-                    branch-pattern: '**/master'
-                forbidden-file-paths:
-                  - compare-type: ANT
-                    pattern: 'doc/**'
-                  - compare-type: ANT
-                    pattern: 'releasenotes/**'
-            readable-message: true
-
-    builders:
-        - description-setter:
-            description: "Built on $NODE_NAME"
-        - 'openstack-bifrost-verify-builder'
-
-    publishers:
-        - email:
-            recipients: fatih.degirmenci@ericsson.com yroblamo@redhat.com mchandras@suse.de jack.morgan@intel.com zhang.jun3g@zte.com.cn
-#####################################
-# builder macros
-#####################################
-- builder:
-    name: 'openstack-bifrost-verify-builder'
-    builders:
-        - shell: |
-            #!/bin/bash
-
-            sudo -E $WORKSPACE/jjb/infra/openstack-bifrost-verify.sh
diff --git a/jjb/opnfv/slave-params.yml b/jjb/opnfv/slave-params.yml
index ff24e9e6d..6cbaba4a5 100644
--- a/jjb/opnfv/slave-params.yml
+++ b/jjb/opnfv/slave-params.yml
@@ -239,16 +239,6 @@
             name: GIT_BASE
             default: https://gerrit.opnfv.org/gerrit/$PROJECT
             description: 'Git URL to use on this Jenkins Slave'
-- parameter:
-    name: 'infra-virtual-trusty-defaults'
-    parameters:
-        - label:
-            name: SLAVE_LABEL
-            default: 'infra-virtual-trusty'
-        - string:
-            name: GIT_BASE
-            default: https://gerrit.opnfv.org/gerrit/$PROJECT
-            description: 'Git URL to use on this Jenkins Slave'
 #####################################################
 # Parameters for build slaves
 #####################################################
diff --git a/prototypes/bifrost/scripts/destroy-env.sh b/prototypes/bifrost/scripts/destroy-env.sh
index 72ade5b14..674645777 100755
--- a/prototypes/bifrost/scripts/destroy-env.sh
+++ b/prototypes/bifrost/scripts/destroy-env.sh
@@ -37,8 +37,9 @@ rm -rf /var/log/libvirt/baremetal_logs/*.log
 CLEAN_DIB_IMAGES=${CLEAN_DIB_IMAGES:-false}
 
 if [ $CLEAN_DIB_IMAGES = "true" ]; then
-    rm -rf /httpboot
-    rm -rf /tftpboot
+    rm -rf /httpboot /tftpboot
+    mkdir /httpboot /tftpboot
+    chmod -R 755 /httpboot /tftpboot
 fi
 
 # remove VM disk images