12 files changed, 262 insertions, 62 deletions

diff --git a/jjb/functest/functest-daily-jobs.yml b/jjb/functest/functest-daily-jobs.yml
index a3268d3e5..e8d14321f 100644
--- a/jjb/functest/functest-daily-jobs.yml
+++ b/jjb/functest/functest-daily-jobs.yml
@@ -205,7 +205,7 @@
         - 'suite':
             job-timeout: 60
         - 'daily':
-            job-timeout: 180
+            job-timeout: 240
 
     jobs:
         - 'functest-{installer}-{pod}-{testsuite}-{stream}'
diff --git a/jjb/joid/joid-daily-jobs.yml b/jjb/joid/joid-daily-jobs.yml
index 8c69c8310..ec6c868c9 100644
--- a/jjb/joid/joid-daily-jobs.yml
+++ b/jjb/joid/joid-daily-jobs.yml
@@ -163,17 +163,17 @@
         #   i.e. one tempest smoke ipv6, two vping from functest)
         # 4.not used for release criteria or compliance,
         #   only to debug the dovetail tool bugs with joid
-        - trigger-builds:
-            - project: 'dovetail-joid-{pod}-debug-{stream}'
-              current-parameters: false
-              predefined-parameters:
-                DEPLOY_SCENARIO={scenario}
-              block: true
-              same-node: true
-              block-thresholds:
-                build-step-failure-threshold: 'never'
-                failure-threshold: 'never'
-                unstable-threshold: 'FAILURE'
+        #- trigger-builds:
+        #    - project: 'dovetail-joid-{pod}-debug-{stream}'
+        #      current-parameters: false
+        #      predefined-parameters:
+        #        DEPLOY_SCENARIO={scenario}
+        #      block: true
+        #      same-node: true
+        #      block-thresholds:
+        #        build-step-failure-threshold: 'never'
+        #        failure-threshold: 'never'
+        #        unstable-threshold: 'FAILURE'
 
 - job-template:
     name: 'joid-deploy-{pod}-daily-{stream}'
diff --git a/jjb/kvmfornfv/kvmfornfv.yml b/jjb/kvmfornfv/kvmfornfv.yml
index a782ee0fa..78ec32b85 100644
--- a/jjb/kvmfornfv/kvmfornfv.yml
+++ b/jjb/kvmfornfv/kvmfornfv.yml
@@ -79,6 +79,10 @@
                 branches:
                   - branch-compare-type: 'ANT'
                     branch-pattern: '**/{branch}'
+                disable-strict-forbidden-file-verification: 'true'
+                forbidden-file-paths:
+                  - compare-type: ANT
+                    pattern: 'docs/**|.gitignore'
 
     builders:
         - description-setter:
diff --git a/jjb/opnfvdocs/docs-post-rtd.sh b/jjb/opnfvdocs/docs-post-rtd.sh
deleted file mode 100644
index e3dc9b5f0..000000000
--- a/jjb/opnfvdocs/docs-post-rtd.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-if [ $GERRIT_BRANCH == "master" ]; then
-    RTD_BUILD_VERSION=latest
-else
-    RTD_BUILD_VERSION=${{GERRIT_BRANCH/\//-}}
-fi
-curl -X POST --data "version_slug=$RTD_BUILD_VERSION" https://readthedocs.org/build/opnfvdocsdemo
diff --git a/jjb/opnfvdocs/opnfvdocs.yml b/jjb/opnfvdocs/opnfvdocs.yml
index 0553cd436..fc825ff64 100644
--- a/jjb/opnfvdocs/opnfvdocs.yml
+++ b/jjb/opnfvdocs/opnfvdocs.yml
@@ -8,8 +8,8 @@
     project: '{name}'
 
     jobs:
-        - 'opnfvdocs-verify-{stream}'
-        - 'opnfvdocs-merge-{stream}'
+        - 'opnfvdocs-verify-shellcheck-{stream}'
+        - 'opnfvdocs-merge-shellcheck-{stream}'
         - 'opnfvdocs-daily-{stream}'
 
     stream:
@@ -27,7 +27,7 @@
 ########################
 
 - job-template:
-    name: 'opnfvdocs-verify-{stream}'
+    name: 'opnfvdocs-verify-shellcheck-{stream}'
 
     disabled: '{obj:disabled}'
 
@@ -72,7 +72,7 @@
         - check-bash-syntax
 
 - job-template:
-    name: 'opnfvdocs-merge-{stream}'
+    name: 'opnfvdocs-merge-shellcheck-{stream}'
 
     disabled: '{obj:disabled}'
 
diff --git a/jjb/qtip/helpers/validate-deploy.sh b/jjb/qtip/helpers/validate-deploy.sh
index a73e33c2c..aa07986ce 100644
--- a/jjb/qtip/helpers/validate-deploy.sh
+++ b/jjb/qtip/helpers/validate-deploy.sh
@@ -10,7 +10,9 @@ set -e
 
 envs="INSTALLER_TYPE=${INSTALLER_TYPE} -e INSTALLER_IP=${INSTALLER_IP}
 -e NODE_NAME=${NODE_NAME} -e CI_DEBUG=${CI_DEBUG}"
+ramfs=/tmp/qtip/ramfs
 dir_imgstore="${HOME}/imgstore"
+ramfs_volume="$ramfs:/mnt/ramfs"
 
 echo "--------------------------------------------------------"
 echo "POD: $NODE_NAME"
@@ -21,7 +23,24 @@ echo "--------------------------------------------------------"
 echo "Qtip: Pulling docker image: opnfv/qtip:${DOCKER_TAG}"
 docker pull opnfv/qtip:$DOCKER_TAG
 
-cmd=" docker run -id -e $envs opnfv/qtip:${DOCKER_TAG} /bin/bash"
+# use ramfs to fix docker socket connection issue with overlay mode in centos
+if [ ! -d $ramfs ]; then
+    mkdir -p $ramfs
+fi
+
+if [ ! -z $(df $ramfs | tail -n -1 | grep $ramfs) ]; then
+    sudo mount -t tmpfs -o size=32M tmpfs $ramfs
+fi
+
+# enable contro path in docker
+echo <<EOF > /tmp/ansible.cfg
+[defaults]
+callback_whitelist = profile_tasks
+[ssh_connection]
+control_path=/mnt/ramfs/ansible-ssh-%%h-%%p-%%r
+EOF
+
+cmd=" docker run -id -e $envs -v ${ramfs_volume} opnfv/qtip:${DOCKER_TAG} /bin/bash"
 echo "Qtip: Running docker command: ${cmd}"
 ${cmd}
 
@@ -32,6 +51,7 @@ if [ $(docker ps | grep 'opnfv/qtip' | wc -l) == 0 ]; then
 else
     echo "The container ID is: ${container_id}"
     QTIP_REPO=/home/opnfv/repos/qtip
+    docker cp /tmp/ansible.cfg ${container_id}:/home/opnfv/.ansible.cfg
 # TODO(zhihui_wu): use qtip cli to execute benchmark test in the future
     docker exec -t ${container_id} bash -c "cd ${QTIP_REPO}/qtip/runner/ &&
     python runner.py -d /home/opnfv/qtip/results/ -b all"
diff --git a/jjb/xci/bifrost-cleanup-job.yml b/jjb/xci/bifrost-cleanup-job.yml
index 571e275da..4e9e2a8d4 100644
--- a/jjb/xci/bifrost-cleanup-job.yml
+++ b/jjb/xci/bifrost-cleanup-job.yml
@@ -88,11 +88,8 @@
             trigger-on:
                 # We only run this when the change is merged or
                 # abandoned since we don't need the logs anymore
-                - patchset-uploaded-event: 'false'
-                - change-merged-event: 'true'
-                - change-abandoned-event: 'true'
-                - change-restored-event: 'false'
-                - draft-published-event: 'false'
+                - change-merged-event
+                - change-abandoned-event
             # This is an OPNFV maintenance job. We don't want to provide
             # feedback on Gerrit
             silent: true
@@ -119,11 +116,8 @@
             trigger-on:
                 # We only run this when the change is merged or
                 # abandoned since we don't need the logs anymore
-                - patchset-uploaded-event: 'false'
-                - change-merged-event: 'true'
-                - change-abandoned-event: 'true'
-                - change-restored-event: 'false'
-                - draft-published-event: 'false'
+                - change-merged-event
+                - change-abandoned-event
             # This is an OPNFV maintenance job. We don't want to provide
             # feedback on Gerrit
             silent: true
diff --git a/jjb/xci/bifrost-verify-jobs.yml b/jjb/xci/bifrost-verify-jobs.yml
index 33032bc7b..6aea1c4aa 100644
--- a/jjb/xci/bifrost-verify-jobs.yml
+++ b/jjb/xci/bifrost-verify-jobs.yml
@@ -145,6 +145,9 @@
         - bifrost-set-name
         - bifrost-build
 
+    wrappers:
+        - bifrost-fix-perms-workspace
+
     publishers:
         - email:
             recipients: fatih.degirmenci@ericsson.com yroblamo@redhat.com mchandras@suse.de jack.morgan@intel.com julienjut@gmail.com
@@ -208,6 +211,17 @@
             readable-message: true
 
 #---------------------------
+# wrapper macros
+#---------------------------
+- wrapper:
+    name: bifrost-fix-perms-workspace
+    wrappers:
+        - pre-scm-buildstep:
+          - shell: |
+                #!/bin/bash
+                sudo chown -R $USER $WORKSPACE || exit 1
+
+#---------------------------
 # builder macros
 #---------------------------
 - builder:
diff --git a/jjb/xci/bifrost-verify.sh b/jjb/xci/bifrost-verify.sh
index e0c50907a..762466825 100755
--- a/jjb/xci/bifrost-verify.sh
+++ b/jjb/xci/bifrost-verify.sh
@@ -115,7 +115,7 @@ sudo /bin/cp -rf /opt/releng/prototypes/bifrost/* /opt/bifrost/
 cd /opt/bifrost
 sudo -E ./scripts/destroy-env.sh
 
-# provision 3 VMs; jumphost, controller, and compute
+# provision 3 VMs; xcimaster, controller, and compute
 cd /opt/bifrost
 sudo -E ./scripts/test-bifrost-deployment.sh
 
diff --git a/jjb/xci/xci-daily-jobs.yml b/jjb/xci/xci-daily-jobs.yml
index c685aa974..f42f86263 100644
--- a/jjb/xci/xci-daily-jobs.yml
+++ b/jjb/xci/xci-daily-jobs.yml
@@ -3,15 +3,22 @@
 #--------------------------------
 # BRANCH ANCHORS
 #--------------------------------
+# the versions stated here default to branches which then later
+# on used for checking out the branches, pulling in head of the branch.
+# but they also allow us to state sha1 so instead of checking out the
+# branches, we can check out sha1 if we want to use locked/specific
+# sha1 or manually enter sha1.
     master: &master
         stream: master
-        openstack-branch: '{stream}'
-        opnfv-branch: 'master'
+        openstack-osa-version: '{stream}'
+        openstack-bifrost-version: '{stream}'
+        opnfv-releng-version: 'master'
         gs-pathname: ''
     ocata: &ocata
         stream: ocata
-        openstack-branch: 'stable/{stream}'
-        opnfv-branch: 'master'
+        openstack-osa-version: 'stable/{stream}'
+        openstack-bifrost-version: 'stable/{stream}'
+        opnfv-releng-version: 'master'
         gs-pathname: '/{stream}'
 #--------------------------------
 #       scenarios
@@ -78,15 +85,27 @@
     concurrent: false
 
     properties:
-        - logrotate-default
         - build-blocker:
             use-build-blocker: true
             blocking-jobs:
-                - 'xci-os-.*?-{pod}-daily-.*'
+                - '^xci-os.*'
             block-level: 'NODE'
+        - logrotate-default
 
     parameters:
         - string:
+            name: OPENSTACK_OSA_VERSION
+            default: '{openstack-osa-version}'
+        - string:
+            name: OPENSTACK_BIFROST_VERSION
+            default: '{openstack-osa-version}'
+        - string:
+            name: OPNFV_RELENG_VERSION
+            default: '{opnfv-releng-version}'
+        - string:
+            name: USE_PROMOTED_VERSIONS
+            default: 'true'
+        - string:
             name: DEPLOY_SCENARIO
             default: '{scenario}'
         - label:
@@ -96,28 +115,43 @@
     triggers:
         - '{auto-trigger-name}'
 
+    wrappers:
+        - xci-fix-perms-workspace
+
     builders:
         - description-setter:
             description: "Built on $NODE_NAME"
         - trigger-builds:
             - project: 'xci-provision-{pod}-{distro}-daily-{stream}'
               current-parameters: false
-              predefined-parameters:
-                DEPLOY_SCENARIO={scenario}
+              predefined-parameters: |
+                OPENSTACK_OSA_VERSION=$OPENSTACK_OSA_VERSION
+                OPENSTACK_BIFROST_VERSION=$OPENSTACK_BIFROST_VERSION
+                OPNFV_RELENG_VERSION=$OPNFV_RELENG_VERSION
+                USE_PROMOTED_VERSIONS=$USE_PROMOTED_VERSIONS
+                DEPLOY_SCENARIO=$DEPLOY_SCENARIO
               same-node: true
               block: true
         - trigger-builds:
             - project: 'xci-deploy-{pod}-{distro}-daily-{stream}'
               current-parameters: false
-              predefined-parameters:
-                DEPLOY_SCENARIO={scenario}
+              predefined-parameters: |
+                OPENSTACK_OSA_VERSION=$OPENSTACK_OSA_VERSION
+                OPENSTACK_BIFROST_VERSION=$OPENSTACK_BIFROST_VERSION
+                OPNFV_RELENG_VERSION=$OPNFV_RELENG_VERSION
+                USE_PROMOTED_VERSIONS=$USE_PROMOTED_VERSIONS
+                DEPLOY_SCENARIO=$DEPLOY_SCENARIO
               same-node: true
               block: true
         - trigger-builds:
             - project: 'xci-functest-{pod}-{distro}-daily-{stream}'
               current-parameters: false
-              predefined-parameters:
-                DEPLOY_SCENARIO={scenario}
+              predefined-parameters: |
+                OPENSTACK_OSA_VERSION=$OPENSTACK_OSA_VERSION
+                OPENSTACK_BIFROST_VERSION=$OPENSTACK_BIFROST_VERSION
+                OPNFV_RELENG_VERSION=$OPNFV_RELENG_VERSION
+                USE_PROMOTED_VERSIONS=$USE_PROMOTED_VERSIONS
+                DEPLOY_SCENARIO=$DEPLOY_SCENARIO
               same-node: true
               block: true
               block-thresholds:
@@ -137,22 +171,25 @@
     concurrent: false
 
     properties:
-        - logrotate-default
         - build-blocker:
             use-build-blocker: true
             blocking-jobs:
-                - 'xci-provision-.*?-{pod}-daily-.*'
-                - 'xci-deploy-.*?-{pod}-daily-.*'
-                - 'xci-functest-.*?-{pod}-daily-.*'
+                - '^xci-provision.*'
+                - '^xci-deploy.*'
+                - '^xci-functest.*'
             block-level: 'NODE'
+        - logrotate-default
 
     parameters:
         - string:
-            name: OPENSTACK_BRANCH
-            default: '{openstack-branch}'
+            name: OPENSTACK_OSA_VERSION
+            default: '{openstack-osa-version}'
+        - string:
+            name: OPENSTACK_BIFROST_VERSION
+            default: '{openstack-osa-version}'
         - string:
-            name: OPNFV_BRANCH
-            default: '{opnfv-branch}'
+            name: OPNFV_RELENG_VERSION
+            default: '{opnfv-releng-version}'
         - string:
             name: USE_PROMOTED_VERSIONS
             default: 'true'
@@ -177,11 +214,29 @@
         - string:
             name: CLEAN_DIB_IMAGES
             default: 'true'
+        - label:
+            name: SLAVE_LABEL
+            default: '{slave-label}'
+
+    wrappers:
+        - xci-fix-perms-workspace
 
     builders:
         - description-setter:
             description: "Built on $NODE_NAME - Scenario: $DEPLOY_SCENARIO"
         - 'xci-{phase}-builder'
+
+#---------------------------
+# wrapper macros
+#---------------------------
+- wrapper:
+    name: xci-fix-perms-workspace
+    wrappers:
+        - pre-scm-buildstep:
+          - shell: |
+                #!/bin/bash
+                sudo chown -R $USER $WORKSPACE || exit 1
+
 #---------------------------
 # builder macros
 #---------------------------
diff --git a/jjb/xci/xci-deploy.sh b/jjb/xci/xci-deploy.sh
index b97b24e47..87f9ec8db 100755
--- a/jjb/xci/xci-deploy.sh
+++ b/jjb/xci/xci-deploy.sh
@@ -1,3 +1,93 @@
 #!/bin/bash
+# SPDX-license-identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2016 Ericsson AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+set -o errexit
+set -o nounset
+set -o pipefail
 
-echo "Deployment via openstack-ansible"
+trap cleanup_and_upload EXIT
+
+function fix_ownership() {
+    if [ -z "${JOB_URL+x}" ]; then
+        echo "Not running as part of Jenkins. Handle the logs manually."
+    else
+        # Make sure cache exists
+        [[ ! -d ${HOME}/.cache ]] && mkdir ${HOME}/.cache
+
+        sudo chown -R jenkins:jenkins $WORKSPACE
+        sudo chown -R jenkins:jenkins ${HOME}/.cache
+    fi
+}
+
+function cleanup_and_upload() {
+    original_exit=$?
+    fix_ownership
+    exit $original_exit
+}
+
+# check distro to see if we support it
+if [[ ! "$DISTRO" =~ (xenial|centos7|suse) ]]; then
+    echo "Distro $DISTRO is not supported!"
+    exit 1
+fi
+
+# remove previously cloned repos
+sudo /bin/rm -rf /opt/openstack-ansible /opt/stack /opt/releng /opt/functest
+
+# Fix up permissions
+fix_ownership
+
+# openstack-ansible enables strict host key checking by default
+export ANSIBLE_HOST_KEY_CHECKING=False
+
+# ensure the versions to checkout are set
+export OPENSTACK_OSA_VERSION=${OPENSTACK_OSA_VERSION:-master}
+export OPNFV_RELENG_VERSION=${OPNFV_RELENG_VERSION:-master}
+
+# log some info
+echo -e "\n"
+echo "***********************************************************************"
+echo "*                                                                     *"
+echo "*                         Deploy OpenStack                            *"
+echo "*                                                                     *"
+echo "                 openstack-ansible version: $OPENSTACK_OSA_VERSION"
+echo "                       releng version: $OPNFV_RELENG_VERSION"
+echo "*                                                                     *"
+echo "***********************************************************************"
+echo -e "\n"
+# clone releng repo
+sudo git clone --quiet https://gerrit.opnfv.org/gerrit/releng /opt/releng
+cd /opt/releng && sudo git checkout --quiet $OPNFV_RELENG_VERSION
+echo "xci: using openstack-ansible commit"
+git show --oneline -s --pretty=format:'%h - %s (%cr) <%an>'
+
+# display the nodes
+echo "xci: OpenStack nodes"
+cd /opt/bifrost
+source env-vars
+ironic node-list
+
+# this script will be reused for promoting openstack-ansible versions and using
+# promoted openstack-ansible versions as part of xci daily.
+USE_PROMOTED_VERSIONS=${USE_PROMOTED_VERSIONS:-false}
+if [ $USE_PROMOTED_VERSIONS = "true" ]; then
+    echo "TBD: Will use the promoted versions of openstack/opnfv projects"
+fi
+
+cd /opt/releng/prototypes/openstack-ansible/scripts
+sudo -E ./osa-deploy.sh
+
+# log some info
+echo -e "\n"
+echo "***********************************************************************"
+echo "*                                                                     *"
+echo "*                OpenStack deployment is completed!                   *"
+echo "*                                                                     *"
+echo "***********************************************************************"
+echo -e "\n"
diff --git a/jjb/xci/xci-provision.sh b/jjb/xci/xci-provision.sh
index 4308c7ef8..a1ba78bf0 100755
--- a/jjb/xci/xci-provision.sh
+++ b/jjb/xci/xci-provision.sh
@@ -43,11 +43,32 @@ sudo /bin/rm -rf /opt/bifrost /opt/openstack-ansible /opt/stack /opt/releng /opt
 # Fix up permissions
 fix_ownership
 
-# clone all the repos first and checkout the patch afterwards
-OPENSTACK_BRANCH=${OPENSTACK_BRANCH:-master}
-OPNFV_BRANCH=${OPNFV_BRANCH:-master}
-sudo git clone -b $OPENSTACK_BRANCH https://git.openstack.org/openstack/bifrost /opt/bifrost
-sudo git clone -b $OPNFV_BRANCH https://gerrit.opnfv.org/gerrit/releng /opt/releng
+# ensure the versions to checkout are set
+export OPENSTACK_BIFROST_VERSION=${OPENSTACK_BIFROST_VERSION:-master}
+export OPNFV_RELENG_VERSION=${OPNFV_RELENG_VERSION:-master}
+
+# log some info
+echo -e "\n"
+echo "***********************************************************************"
+echo "*                                                                     *"
+echo "*                      Provision OpenStack Nodes                      *"
+echo "*                                                                     *"
+echo "                       bifrost version: $OPENSTACK_BIFROST_VERSION"
+echo "                       releng version: $OPNFV_RELENG_VERSION"
+echo "*                                                                     *"
+echo "***********************************************************************"
+echo -e "\n"
+
+# clone the repos and checkout the versions
+sudo git clone --quiet https://git.openstack.org/openstack/bifrost /opt/bifrost
+cd /opt/bifrost && sudo git checkout --quiet $OPENSTACK_BIFROST_VERSION
+echo "xci: using bifrost commit"
+git show --oneline -s --pretty=format:'%h - %s (%cr) <%an>'
+
+sudo git clone --quiet https://gerrit.opnfv.org/gerrit/releng /opt/releng
+cd /opt/releng && sudo git checkout --quiet $OPNFV_RELENG_VERSION
+echo "xci: using releng commit"
+git show --oneline -s --pretty=format:'%h - %s (%cr) <%an>'
 
 # this script will be reused for promoting bifrost versions and using
 # promoted bifrost versions as part of xci daily.
@@ -63,7 +84,7 @@ sudo /bin/cp -rf /opt/releng/prototypes/bifrost/* /opt/bifrost/
 cd /opt/bifrost
 sudo -E ./scripts/destroy-env.sh
 
-# provision 6 VMs; jumphost, controller00, controller01, controller02, compute00, and compute01
+# provision 6 VMs; xcimaster, controller00, controller01, controller02, compute00, and compute01
 cd /opt/bifrost
 sudo -E ./scripts/osa-bifrost-deployment.sh
 
@@ -72,3 +93,12 @@ cd /opt/bifrost
 source env-vars
 ironic node-list
 virsh list
+
+# log some info
+echo -e "\n"
+echo "***********************************************************************"
+echo "*                                                                     *"
+echo "*                 OpenStack nodes are provisioned!                    *"
+echo "*                                                                     *"
+echo "***********************************************************************"
+echo -e "\n"