apex: Take builders out of jjb

Having builders embedded in jjb makes maintenance harder than what
it could be and error prone so the builders have been taken out.

Apart from this, the upcoming change proposals will split yml files
in 2, aligning with the rest and jobs in these files can reuse same
builders when they are converted in shell scripts.

This is the first change in series, trying to align job structure
with the rest in order to ease the effort required to troubleshoot
the deployments during release verification. The changes are
- take builders out of jjb (this patch)
- align the main CI jobs with the rest; 1 parent job per scenario/branch
  which controls triggering and execution of 3 downstream jobs;
  deploy, yardstick, functest
- split yml file in 2; ci jobs ending in apex-ci-jobs.yml (daily) and
  project jobs ending in apex-project-jobs.yml. (verify, merge, build)

Change-Id: I01f8bf351f0cbafbee1f41ec24ad5c2c6f21316f
Signed-off-by: Fatih Degirmenci <fatih.degirmenci@ericsson.com>

7 files changed, 366 insertions, 360 deletions

diff --git a/jjb/apex/apex-build.sh b/jjb/apex/apex-build.sh
new file mode 100755
index 000000000..ca1821abf
--- /dev/null
+++ b/jjb/apex/apex-build.sh
@@ -0,0 +1,59 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+# log info to console
+echo "Starting the build of Apex using OpenStack Master packages. This will take some time..."
+echo "---------------------------------------------------------------------------------------"
+echo
+# create the cache directory if it doesn't exist
+[[ -d $CACHE_DIRECTORY ]] || mkdir -p $CACHE_DIRECTORY
+# set OPNFV_ARTIFACT_VERSION
+if echo $BUILD_TAG | grep "apex-verify" 1> /dev/null; then
+  if echo $GERRIT_BRANCH | grep "brahmaputra" 1> /dev/null; then
+    export OPNFV_ARTIFACT_VERSION=brahmaputra-dev${BUILD_NUMBER}
+    export BUILD_ARGS="-v $OPNFV_ARTIFACT_VERSION -c file://$CACHE_DIRECTORY $BUILD_DIRECTORY"
+  else
+    export OPNFV_ARTIFACT_VERSION=dev${BUILD_NUMBER}
+    export BUILD_ARGS="-r $OPNFV_ARTIFACT_VERSION -c file://$CACHE_DIRECTORY"
+  fi
+elif [ "$ARTIFACT_VERSION" == "daily" ]; then
+  if echo $GERRIT_BRANCH | grep "brahmaputra" 1> /dev/null; then
+    export OPNFV_ARTIFACT_VERSION=brahmaputra-$(date -u +"%Y-%m-%d")
+    export BUILD_ARGS="-v $OPNFV_ARTIFACT_VERSION -c file://$CACHE_DIRECTORY $BUILD_DIRECTORY"
+  else
+    export OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d")
+    export BUILD_ARGS="-r $OPNFV_ARTIFACT_VERSION -c file://$CACHE_DIRECTORY --iso"
+  fi
+else
+  export OPNFV_ARTIFACT_VERSION=${ARTIFACT_VERSION}
+  if echo $GERRIT_BRANCH | grep "brahmaputra" 1> /dev/null; then
+    export BUILD_ARGS="-v $OPNFV_ARTIFACT_VERSION -c file://$CACHE_DIRECTORY $BUILD_DIRECTORY"
+  else
+    export BUILD_ARGS="-r $OPNFV_ARTIFACT_VERSION -c file://$CACHE_DIRECTORY --iso"
+  fi
+fi
+# clean for stable but doesn't matter for master
+if echo $GERRIT_BRANCH | grep "brahmaputra" 1> /dev/null; then
+  sudo opnfv-clean
+fi
+# start the build
+cd $WORKSPACE/ci
+./build.sh $BUILD_ARGS
+RPM_VERSION=$(grep Version: $BUILD_DIRECTORY/opnfv-apex.spec | awk '{ print $2 }')-$(echo $OPNFV_ARTIFACT_VERSION | tr -d '_-')
+# list the contents of BUILD_OUTPUT directory
+ls -al $BUILD_DIRECTORY
+# save information regarding artifact into file
+(
+    echo "OPNFV_ARTIFACT_VERSION=$OPNFV_ARTIFACT_VERSION"
+    echo "OPNFV_GIT_URL=$(git config --get remote.origin.url)"
+    echo "OPNFV_GIT_SHA1=$(git rev-parse HEAD)"
+    echo "OPNFV_ARTIFACT_URL=$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso"
+    echo "OPNFV_ARTIFACT_MD5SUM=$(md5sum $BUILD_DIRECTORY/release/OPNFV-CentOS-7-x86_64-$OPNFV_ARTIFACT_VERSION.iso | cut -d' ' -f1)"
+    echo "OPNFV_SRPM_URL=$GS_URL/opnfv-apex-$RPM_VERSION.src.rpm"
+    echo "OPNFV_RPM_URL=$GS_URL/opnfv-apex-$RPM_VERSION.noarch.rpm"
+    echo "OPNFV_RPM_MD5SUM=$(md5sum $BUILD_DIRECTORY/noarch/opnfv-apex-$RPM_VERSION.noarch.rpm | cut -d' ' -f1)"
+    echo "OPNFV_BUILD_URL=$BUILD_URL"
+) > $WORKSPACE/opnfv.properties
+echo "--------------------------------------------------------"
+echo "Done!"
diff --git a/jjb/apex/apex-deploy-baremetal.sh b/jjb/apex/apex-deploy-baremetal.sh
new file mode 100755
index 000000000..efb6561d7
--- /dev/null
+++ b/jjb/apex/apex-deploy-baremetal.sh
@@ -0,0 +1,84 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# log info to console
+echo "Starting the Apex baremetal deployment."
+echo "--------------------------------------------------------"
+echo
+
+if [[ ! "$ARTIFACT_NAME" == "latest" ]]; then
+    # if artifact name is passed the pull a
+    # specific artifact from artifacts.opnfv.org
+    RPM_INSTALL_PATH=$GS_URL/$ARTIFACT_NAME
+else
+    if [[ $BUILD_DIRECTORY == *apex-build* ]]; then
+      BUILD_DIRECTORY=$WORKSPACE/../$BUILD_DIRECTORY
+      echo "BUILD DIRECTORY modified to $BUILD_DIRECTORY"
+    fi
+    if [[ -f ${BUILD_DIRECTORY}/../opnfv.properties ]]; then
+        # if opnfv.properties exists then use the
+        # local build. Source the file so we get local OPNFV vars
+        source ${BUILD_DIRECTORY}/../opnfv.properties
+        RPM_INSTALL_PATH=${BUILD_DIRECTORY}/$(basename $OPNFV_RPM_URL)
+    else
+        # no opnfv.properties means use the latest from artifacts.opnfv.org
+        # get the latest.properties to get the link to the latest artifact
+        curl -s -o $WORKSPACE/opnfv.properties http://$GS_URL/latest.properties
+        [[ -f opnfv.properties ]] || exit 1
+        # source the file so we get OPNFV vars
+        source opnfv.properties
+        RPM_INSTALL_PATH=$OPNFV_RPM_URL
+    fi
+fi
+
+if [ ! -e "$RPM_INSTALL_PATH" ]; then
+   RPM_INSTALL_PATH=http://${OPNFV_RPM_URL}
+fi
+
+RPM_LIST=$RPM_INSTALL_PATH
+for pkg in common undercloud; do
+    RPM_LIST+=" ${RPM_INSTALL_PATH/opnfv-apex/opnfv-apex-${pkg}}"
+done
+
+# update / install the new rpm
+if rpm -q opnfv-apex > /dev/null; then
+   if [ $(basename $OPNFV_RPM_URL) == $(rpm -q opnfv-apex).rpm ]; then
+     echo "RPM is already installed"
+   elif sudo yum update -y $RPM_LIST | grep "does not update installed package"; then
+       if ! sudo yum downgrade -y $RPM_LIST; then
+         sudo yum remove -y opnfv-undercloud opnfv-common
+         sudo yum downgrade -y $RPM_INSTALL_PATH
+       fi
+   fi
+else
+   sudo yum install -y $RPM_LIST;
+fi
+
+# cleanup environment before we start
+sudo opnfv-clean
+# initiate baremetal deployment
+if [ -e /etc/opnfv-apex/network_settings.yaml ]; then
+  if [ -n "$DEPLOY_SCENARIO" ]; then
+    echo "Deploy Scenario set to ${DEPLOY_SCENARIO}"
+    if [ -e /etc/opnfv-apex/${DEPLOY_SCENARIO}.yaml ]; then
+      sudo opnfv-deploy -i  /root/inventory/pod_settings.yaml \
+      -d /etc/opnfv-apex/${DEPLOY_SCENARIO}.yaml \
+      -n /root/network/network_settings.yaml --debug
+    else
+      echo "File does not exist /etc/opnfv-apex/${DEPLOY_SCENARIO}.yaml"
+      exit 1
+    fi
+  else
+    echo "Deploy scenario not set!"
+    exit 1
+  fi
+else
+  echo "File /etc/opnfv-apex/network_settings.yaml does not exist!"
+  exit 1
+fi
+
+echo
+echo "--------------------------------------------------------"
+echo "Done!"
diff --git a/jjb/apex/apex-deploy-virtual.sh b/jjb/apex/apex-deploy-virtual.sh
new file mode 100755
index 000000000..c7fb17e8c
--- /dev/null
+++ b/jjb/apex/apex-deploy-virtual.sh
@@ -0,0 +1,148 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# log info to console
+echo "Starting the Apex virtual deployment."
+echo "--------------------------------------------------------"
+echo
+
+if [[ $BUILD_DIRECTORY == *verify-master* ]]; then
+    cd $WORKSPACE/../${BUILD_DIRECTORY/build_output/}
+    WORKSPACE=$(pwd)
+    echo "WORKSPACE modified to $WORKSPACE"
+    cd $WORKSPACE/ci
+elif [[ ! "$ARTIFACT_NAME" == "latest" ]]; then
+    # if artifact name is passed the pull a
+    # specific artifact from artifacts.opnfv.org
+    RPM_INSTALL_PATH=$GS_URL
+    RPM_LIST=$RPM_INSTALL_PATH/$ARTIFACT_NAME
+else
+    if [[ $BUILD_DIRECTORY == *verify* ]]; then
+      BUILD_DIRECTORY=$WORKSPACE/../$BUILD_DIRECTORY
+      echo "BUILD DIRECTORY modified to $BUILD_DIRECTORY"
+    elif [[ $BUILD_DIRECTORY == *apex-build* ]]; then
+      BUILD_DIRECTORY=$WORKSPACE/../$BUILD_DIRECTORY
+      echo "BUILD DIRECTORY modified to $BUILD_DIRECTORY"
+    fi
+
+    if [[ -f ${BUILD_DIRECTORY}/../opnfv.properties ]]; then
+        # if opnfv.properties exists then use the
+        # local build. Source the file so we get local OPNFV vars
+        source ${BUILD_DIRECTORY}/../opnfv.properties
+        RPM_INSTALL_PATH=${BUILD_DIRECTORY}/noarch
+        RPM_LIST=$RPM_INSTALL_PATH/$(basename $OPNFV_RPM_URL)
+    else
+        if [[ $BUILD_DIRECTORY == *verify* ]]; then
+          echo "BUILD_DIRECTORY is from a verify job, so will not use latest from URL"
+          echo "Check that the slave has opnfv.properties in $BUILD_DIRECTORY"
+          exit 1
+        elif [[ $BUILD_DIRECTORY == *apex-build* ]]; then
+          echo "BUILD_DIRECTORY is from a daily job, so will not use latest from URL"
+          echo "Check that the slave has opnfv.properties in $BUILD_DIRECTORY"
+          exit 1
+        fi
+        # no opnfv.properties means use the latest from artifacts.opnfv.org
+        # get the latest.properties to get the link to the latest artifact
+        curl -s -o $WORKSPACE/opnfv.properties http://$GS_URL/latest.properties
+        [[ -f opnfv.properties ]] || exit 1
+        # source the file so we get OPNFV vars
+        source opnfv.properties
+        RPM_INSTALL_PATH=$(echo $OPNFV_RPM_URL | sed 's/'"$(basename $OPNFV_RPM_URL)"'//')
+        RPM_LIST=$RPM_INSTALL_PATH/$(basename $OPNFV_RPM_URL)
+    fi
+fi
+
+if [ -z "$DEPLOY_SCENARIO" ]; then
+  echo "Deploy scenario not set!"
+  exit 1
+fi
+
+# use local build for verify
+if [[ $BUILD_DIRECTORY == *verify-master* ]]; then
+    if [ ! -e "${WORKSPACE}/build/lib" ]; then ln -s ${WORKSPACE}/lib ${WORKSPACE}/build/lib; fi
+    DEPLOY_CMD="CONFIG=${WORKSPACE}/build RESOURCES=${WORKSPACE}/build/images/ ./deploy.sh -c ${WORKSPACE}/build -r ${WORKSPACE}/build/images/"
+    DEPLOY_FILE="${WORKSPACE}/config/deploy/${DEPLOY_SCENARIO}.yaml"
+    NETWORK_FILE="${WORKSPACE}/config/network/network_settings.yaml"
+    # Make sure python34 is installed
+    if ! rpm -q python34 > /dev/null; then
+        sudo yum install -y epel-release
+        if ! sudo yum install -y python34; then
+            echo "Failed to install python34"
+            exit 1
+        fi
+    fi
+    if ! rpm -q python34-PyYAML > /dev/null; then
+        sudo yum install -y epel-release
+        if ! sudo yum install -y python34-PyYAML; then
+            echo "Failed to install python34-PyYAML"
+            exit 1
+        fi
+    fi
+    if ! rpm -q python34-setuptools > /dev/null; then
+        if ! sudo yum install -y python34-setuptools; then
+            echo "Failed to install python34-setuptools"
+            exit 1
+        fi
+    fi
+    if [ -z ${PYTHONPATH:-} ]; then
+        export PYTHONPATH=${WORKSPACE}/lib/python
+    else
+        export PYTHONPATH=$PYTHONPATH:${WORKSPACE}/lib/python
+    fi
+else
+    VERSION_EXTENSION=$(echo $(basename $RPM_LIST) | sed 's/opnfv-apex-//')
+    for pkg in common undercloud opendaylight-sfc onos; do
+        RPM_LIST+=" ${RPM_INSTALL_PATH}/opnfv-apex-${pkg}-${VERSION_EXTENSION}"
+    done
+
+    # update / install the new rpm
+    if rpm -q opnfv-apex > /dev/null; then
+       INSTALLED_RPMS=$(rpm -qa | grep apex)
+       for x in $INSTALLED_RPMS; do
+         INSTALLED_RPM_VER=$(echo $x | sed 's/opnfv-apex-//').rpm
+         # Does each RPM's version match the version required for deployment
+         if [ "$INSTALLED_RPM_VER" == "$VERSION_EXTENSION" ]; then
+           echo "RPM $x is already installed"
+         else
+           echo "RPM $x does not match $VERSION_EXTENSION"
+           echo "Will upgrade/downgrade RPMs..."
+           # Try to upgrade/downgrade RPMS
+           if sudo yum update -y $RPM_LIST | grep "does not update installed package"; then
+             if ! sudo yum downgrade -y $RPM_LIST; then
+               sudo yum remove -y opnfv-apex-undercloud opnfv-apex-common opnfv-apex-opendaylight-sfc opnfv-apex-onos
+               sudo yum downgrade -y $RPM_INSTALL_PATH
+             fi
+           fi
+           break
+         fi
+       done
+    else
+       sudo yum install -y $RPM_LIST;
+    fi
+    DEPLOY_CMD=opnfv-deploy
+    DEPLOY_FILE="/etc/opnfv-apex/${DEPLOY_SCENARIO}.yaml"
+    NETWORK_FILE="/etc/opnfv-apex/network_settings.yaml"
+    export RESOURCES="/var/opt/opnfv/images"
+    export CONFIG="/var/opt/opnfv"
+fi
+
+if [ "$OPNFV_CLEAN" == 'yes' ]; then
+    if [[ $BUILD_DIRECTORY == *verify-master* ]]; then
+        sudo CONFIG=${WORKSPACE}/build ./clean.sh
+    else
+        sudo opnfv-clean
+    fi
+fi
+# initiate virtual deployment
+echo "Deploy Scenario set to ${DEPLOY_SCENARIO}"
+if [ -e $DEPLOY_FILE ]; then
+  sudo $DEPLOY_CMD -v -d ${DEPLOY_FILE} -n $NETWORK_FILE --debug
+else
+  echo "File does not exist /etc/opnfv-apex/${DEPLOY_SCENARIO}.yaml"
+  exit 1
+fi
+echo
+echo "--------------------------------------------------------"
+echo "Done!"
diff --git a/jjb/apex/apex-gs-cleanup.sh b/jjb/apex/apex-gs-cleanup.sh
new file mode 100755
index 000000000..1629aa85e
--- /dev/null
+++ b/jjb/apex/apex-gs-cleanup.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# log info to console
+echo "Cleaning Google Storage"
+echo "-----------------------"
+echo
+
+thirty_days_ago=$(date -d "30 days ago" +"%Y%m%d")
+
+for i in $(gsutil ls gs://$GS_URL/*201?*); do
+    filedate=$(date -d "$(echo $i | grep -Eo 201[0-9]-?[0-9][0-9]-?[0-9][0-9])" +"%Y%m%d")
+    if [ $filedate -lt $thirty_days_ago ]; then
+      # gsutil indicates what it is removing so no need for output here
+      gsutil rm $i
+    fi
+done
diff --git a/jjb/apex/apex-upload-artifact.sh b/jjb/apex/apex-upload-artifact.sh
new file mode 100755
index 000000000..d148258c0
--- /dev/null
+++ b/jjb/apex/apex-upload-artifact.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# log info to console
+echo "Uploading the Apex artifact. This could take some time..."
+echo "--------------------------------------------------------"
+echo
+
+# source the opnfv.properties to get ARTIFACT_VERSION
+source $WORKSPACE/opnfv.properties
+
+# upload artifact and additional files to google storage
+gsutil cp $BUILD_DIRECTORY/release/OPNFV-CentOS-7-x86_64-$OPNFV_ARTIFACT_VERSION.iso gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso > gsutil.iso.log 2>&1
+RPM_INSTALL_PATH=$BUILD_DIRECTORY/noarch
+RPM_LIST=$RPM_INSTALL_PATH/$(basename $OPNFV_RPM_URL)
+VERSION_EXTENSION=$(echo $(basename $OPNFV_RPM_URL) | sed 's/opnfv-apex-//')
+for pkg in common undercloud opendaylight-sfc onos; do
+    RPM_LIST+=" ${RPM_INSTALL_PATH}/opnfv-apex-${pkg}-${VERSION_EXTENSION}"
+done
+SRPM_INSTALL_PATH=$BUILD_DIRECTORY
+SRPM_LIST=$SRPM_INSTALL_PATH/$(basename $OPNFV_SRPM_URL)
+for pkg in common undercloud opendaylight-sfc onos; do
+    SRPM_LIST+=" ${SRPM_INSTALL_PATH}/opnfv-apex-${pkg}-${VERSION_EXTENSION}"
+done
+for artifact in $RPM_LIST $SRPM_LIST; do
+  gsutil cp $artifact gs://$GS_URL/$(basename $artifact) > gsutil.iso.log 2>&1
+done
+gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > gsutil.properties.log 2>&1
+gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/latest.properties > gsutil.latest.log 2>&1
+
+echo
+echo "--------------------------------------------------------"
+echo "Done!"
+echo "ISO Artifact is available as http://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso"
+echo "RPM Artifact is available as http://$GS_URL/$(basename $OPNFV_RPM_URL)"
diff --git a/jjb/apex/apex-workspace-cleanup.sh b/jjb/apex/apex-workspace-cleanup.sh
new file mode 100755
index 000000000..d2f71a562
--- /dev/null
+++ b/jjb/apex/apex-workspace-cleanup.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# delete everything that is in $WORKSPACE
+sudo /bin/rm -rf $WORKSPACE
diff --git a/jjb/apex/apex.yml b/jjb/apex/apex.yml
index 115e75bf8..ed06113d5 100644
--- a/jjb/apex/apex.yml
+++ b/jjb/apex/apex.yml
@@ -732,386 +732,38 @@
 - builder:
     name: 'apex-build'
     builders:
-        - shell: |
-            #!/bin/bash
-            set -o errexit
-            set -o nounset
-            set -o pipefail
-            # log info to console
-            echo "Starting the build of Apex using OpenStack Master packages. This will take some time..."
-            echo "---------------------------------------------------------------------------------------"
-            echo
-            # create the cache directory if it doesn't exist
-            [[ -d $CACHE_DIRECTORY ]] || mkdir -p $CACHE_DIRECTORY
-            # set OPNFV_ARTIFACT_VERSION
-            if echo $BUILD_TAG | grep "apex-verify" 1> /dev/null; then
-              if echo $GERRIT_BRANCH | grep "brahmaputra" 1> /dev/null; then
-                export OPNFV_ARTIFACT_VERSION=brahmaputra-dev${BUILD_NUMBER}
-                export BUILD_ARGS="-v $OPNFV_ARTIFACT_VERSION -c file://$CACHE_DIRECTORY $BUILD_DIRECTORY"
-              else
-                export OPNFV_ARTIFACT_VERSION=dev${BUILD_NUMBER}
-                export BUILD_ARGS="-r $OPNFV_ARTIFACT_VERSION -c file://$CACHE_DIRECTORY"
-              fi
-            elif [ "$ARTIFACT_VERSION" == "daily" ]; then
-              if echo $GERRIT_BRANCH | grep "brahmaputra" 1> /dev/null; then
-                export OPNFV_ARTIFACT_VERSION=brahmaputra-$(date -u +"%Y-%m-%d")
-                export BUILD_ARGS="-v $OPNFV_ARTIFACT_VERSION -c file://$CACHE_DIRECTORY $BUILD_DIRECTORY"
-              else
-                export OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d")
-                export BUILD_ARGS="-r $OPNFV_ARTIFACT_VERSION -c file://$CACHE_DIRECTORY --iso"
-              fi
-            else
-              export OPNFV_ARTIFACT_VERSION=${ARTIFACT_VERSION}
-              if echo $GERRIT_BRANCH | grep "brahmaputra" 1> /dev/null; then
-                export BUILD_ARGS="-v $OPNFV_ARTIFACT_VERSION -c file://$CACHE_DIRECTORY $BUILD_DIRECTORY"
-              else
-                export BUILD_ARGS="-r $OPNFV_ARTIFACT_VERSION -c file://$CACHE_DIRECTORY --iso"
-              fi
-            fi
-            # clean for stable but doesn't matter for master
-            if echo $GERRIT_BRANCH | grep "brahmaputra" 1> /dev/null; then
-              sudo opnfv-clean
-            fi
-            # start the build
-            cd $WORKSPACE/ci
-            ./build.sh $BUILD_ARGS
-            RPM_VERSION=$(grep Version: $BUILD_DIRECTORY/opnfv-apex.spec | awk '{ print $2 }')-$(echo $OPNFV_ARTIFACT_VERSION | tr -d '_-')
-            # list the contents of BUILD_OUTPUT directory
-            ls -al $BUILD_DIRECTORY
-            # save information regarding artifact into file
-            (
-                echo "OPNFV_ARTIFACT_VERSION=$OPNFV_ARTIFACT_VERSION"
-                echo "OPNFV_GIT_URL=$(git config --get remote.origin.url)"
-                echo "OPNFV_GIT_SHA1=$(git rev-parse HEAD)"
-                echo "OPNFV_ARTIFACT_URL=$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso"
-                echo "OPNFV_ARTIFACT_MD5SUM=$(md5sum $BUILD_DIRECTORY/release/OPNFV-CentOS-7-x86_64-$OPNFV_ARTIFACT_VERSION.iso | cut -d' ' -f1)"
-                echo "OPNFV_SRPM_URL=$GS_URL/opnfv-apex-$RPM_VERSION.src.rpm"
-                echo "OPNFV_RPM_URL=$GS_URL/opnfv-apex-$RPM_VERSION.noarch.rpm"
-                echo "OPNFV_RPM_MD5SUM=$(md5sum $BUILD_DIRECTORY/noarch/opnfv-apex-$RPM_VERSION.noarch.rpm | cut -d' ' -f1)"
-                echo "OPNFV_BUILD_URL=$BUILD_URL"
-            ) > $WORKSPACE/opnfv.properties
-            echo "--------------------------------------------------------"
-            echo "Done!"
+        - shell:
+            !include-raw: ./apex-build.sh
 
 - builder:
     name: 'apex-workspace-cleanup'
     builders:
-        - shell: |
-            #!/bin/bash
-            set -o errexit
-            set -o nounset
-            set -o pipefail
-
-            # delete everything that is in $WORKSPACE
-            sudo /bin/rm -rf $WORKSPACE
+        - shell:
+            !include-raw: ./apex-workspace-cleanup.sh
 
 - builder:
     name: 'apex-upload-artifact'
     builders:
-        - shell: |
-            #!/bin/bash
-            set -o errexit
-            set -o nounset
-            set -o pipefail
-
-            # log info to console
-            echo "Uploading the Apex artifact. This could take some time..."
-            echo "--------------------------------------------------------"
-            echo
-
-            # source the opnfv.properties to get ARTIFACT_VERSION
-            source $WORKSPACE/opnfv.properties
-
-            # upload artifact and additional files to google storage
-            gsutil cp $BUILD_DIRECTORY/release/OPNFV-CentOS-7-x86_64-$OPNFV_ARTIFACT_VERSION.iso gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso > gsutil.iso.log 2>&1
-            RPM_INSTALL_PATH=$BUILD_DIRECTORY/noarch
-            RPM_LIST=$RPM_INSTALL_PATH/$(basename $OPNFV_RPM_URL)
-            VERSION_EXTENSION=$(echo $(basename $OPNFV_RPM_URL) | sed 's/opnfv-apex-//')
-            for pkg in common undercloud opendaylight-sfc onos; do
-                RPM_LIST+=" ${RPM_INSTALL_PATH}/opnfv-apex-${pkg}-${VERSION_EXTENSION}"
-            done
-            SRPM_INSTALL_PATH=$BUILD_DIRECTORY
-            SRPM_LIST=$SRPM_INSTALL_PATH/$(basename $OPNFV_SRPM_URL)
-            for pkg in common undercloud opendaylight-sfc onos; do
-                SRPM_LIST+=" ${SRPM_INSTALL_PATH}/opnfv-apex-${pkg}-${VERSION_EXTENSION}"
-            done
-            for artifact in $RPM_LIST $SRPM_LIST; do
-              gsutil cp $artifact gs://$GS_URL/$(basename $artifact) > gsutil.iso.log 2>&1
-            done
-            gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > gsutil.properties.log 2>&1
-            gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/latest.properties > gsutil.latest.log 2>&1
-
-            echo
-            echo "--------------------------------------------------------"
-            echo "Done!"
-            echo "ISO Artifact is available as http://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso"
-            echo "RPM Artifact is available as http://$GS_URL/$(basename $OPNFV_RPM_URL)"
+        - shell:
+            !include-raw: ./apex-upload-artifact.sh
 
 - builder:
     name: 'apex-gs-cleanup'
     builders:
-        - shell: |
-            #!/bin/bash
-            set -o errexit
-            set -o nounset
-            set -o pipefail
-
-            # log info to console
-            echo "Cleaning Google Storage"
-            echo "-----------------------"
-            echo
-
-            thirty_days_ago=$(date -d "30 days ago" +"%Y%m%d")
-
-            for i in $(gsutil ls gs://$GS_URL/*201?*); do
-                filedate=$(date -d "$(echo $i | grep -Eo 201[0-9]-?[0-9][0-9]-?[0-9][0-9])" +"%Y%m%d")
-                if [ $filedate -lt $thirty_days_ago ]; then
-                  # gsutil indicates what it is removing so no need for output here
-                  gsutil rm $i
-                fi
-            done
+        - shell:
+            !include-raw: ./apex-gs-cleanup.sh
 
 - builder:
     name: 'apex-deploy-virtual'
     builders:
-        - shell: |
-            #!/bin/bash
-            set -o errexit
-            set -o nounset
-            set -o pipefail
-
-            # log info to console
-            echo "Starting the Apex virtual deployment."
-            echo "--------------------------------------------------------"
-            echo
-
-            if [[ $BUILD_DIRECTORY == *verify-master* ]]; then
-                cd $WORKSPACE/../${BUILD_DIRECTORY/build_output/}
-                WORKSPACE=$(pwd)
-                echo "WORKSPACE modified to $WORKSPACE"
-                cd $WORKSPACE/ci
-            elif [[ ! "$ARTIFACT_NAME" == "latest" ]]; then
-                # if artifact name is passed the pull a
-                # specific artifact from artifacts.opnfv.org
-                RPM_INSTALL_PATH=$GS_URL
-                RPM_LIST=$RPM_INSTALL_PATH/$ARTIFACT_NAME
-            else
-                if [[ $BUILD_DIRECTORY == *verify* ]]; then
-                  BUILD_DIRECTORY=$WORKSPACE/../$BUILD_DIRECTORY
-                  echo "BUILD DIRECTORY modified to $BUILD_DIRECTORY"
-                elif [[ $BUILD_DIRECTORY == *apex-build* ]]; then
-                  BUILD_DIRECTORY=$WORKSPACE/../$BUILD_DIRECTORY
-                  echo "BUILD DIRECTORY modified to $BUILD_DIRECTORY"
-                fi
-
-                if [[ -f ${BUILD_DIRECTORY}/../opnfv.properties ]]; then
-                    # if opnfv.properties exists then use the
-                    # local build. Source the file so we get local OPNFV vars
-                    source ${BUILD_DIRECTORY}/../opnfv.properties
-                    RPM_INSTALL_PATH=${BUILD_DIRECTORY}/noarch
-                    RPM_LIST=$RPM_INSTALL_PATH/$(basename $OPNFV_RPM_URL)
-                else
-                    if [[ $BUILD_DIRECTORY == *verify* ]]; then
-                      echo "BUILD_DIRECTORY is from a verify job, so will not use latest from URL"
-                      echo "Check that the slave has opnfv.properties in $BUILD_DIRECTORY"
-                      exit 1
-                    elif [[ $BUILD_DIRECTORY == *apex-build* ]]; then
-                      echo "BUILD_DIRECTORY is from a daily job, so will not use latest from URL"
-                      echo "Check that the slave has opnfv.properties in $BUILD_DIRECTORY"
-                      exit 1
-                    fi
-                    # no opnfv.properties means use the latest from artifacts.opnfv.org
-                    # get the latest.properties to get the link to the latest artifact
-                    curl -s -o $WORKSPACE/opnfv.properties http://$GS_URL/latest.properties
-                    [[ -f opnfv.properties ]] || exit 1
-                    # source the file so we get OPNFV vars
-                    source opnfv.properties
-                    RPM_INSTALL_PATH=$(echo $OPNFV_RPM_URL | sed 's/'"$(basename $OPNFV_RPM_URL)"'//')
-                    RPM_LIST=$RPM_INSTALL_PATH/$(basename $OPNFV_RPM_URL)
-                fi
-            fi
-
-            if [ -z "$DEPLOY_SCENARIO" ]; then
-              echo "Deploy scenario not set!"
-              exit 1
-            fi
-
-            # use local build for verify
-            if [[ $BUILD_DIRECTORY == *verify-master* ]]; then
-                if [ ! -e "${WORKSPACE}/build/lib" ]; then ln -s ${WORKSPACE}/lib ${WORKSPACE}/build/lib; fi
-                DEPLOY_CMD="CONFIG=${WORKSPACE}/build RESOURCES=${WORKSPACE}/build/images/ ./deploy.sh -c ${WORKSPACE}/build -r ${WORKSPACE}/build/images/"
-                DEPLOY_FILE="${WORKSPACE}/config/deploy/${DEPLOY_SCENARIO}.yaml"
-                NETWORK_FILE="${WORKSPACE}/config/network/network_settings.yaml"
-                # Make sure python34 is installed
-                if ! rpm -q python34 > /dev/null; then
-                    sudo yum install -y epel-release
-                    if ! sudo yum install -y python34; then
-                        echo "Failed to install python34"
-                        exit 1
-                    fi
-                fi
-                if ! rpm -q python34-PyYAML > /dev/null; then
-                    sudo yum install -y epel-release
-                    if ! sudo yum install -y python34-PyYAML; then
-                        echo "Failed to install python34-PyYAML"
-                        exit 1
-                    fi
-                fi
-                if ! rpm -q python34-setuptools > /dev/null; then
-                    if ! sudo yum install -y python34-setuptools; then
-                        echo "Failed to install python34-setuptools"
-                        exit 1
-                    fi
-                fi
-                if [ -z ${PYTHONPATH:-} ]; then
-                    export PYTHONPATH=${WORKSPACE}/lib/python
-                else
-                    export PYTHONPATH=$PYTHONPATH:${WORKSPACE}/lib/python
-                fi
-            else
-                VERSION_EXTENSION=$(echo $(basename $RPM_LIST) | sed 's/opnfv-apex-//')
-                for pkg in common undercloud opendaylight-sfc onos; do
-                    RPM_LIST+=" ${RPM_INSTALL_PATH}/opnfv-apex-${pkg}-${VERSION_EXTENSION}"
-                done
-
-                # update / install the new rpm
-                if rpm -q opnfv-apex > /dev/null; then
-                   INSTALLED_RPMS=$(rpm -qa | grep apex)
-                   for x in $INSTALLED_RPMS; do
-                     INSTALLED_RPM_VER=$(echo $x | sed 's/opnfv-apex-//').rpm
-                     # Does each RPM's version match the version required for deployment
-                     if [ "$INSTALLED_RPM_VER" == "$VERSION_EXTENSION" ]; then
-                       echo "RPM $x is already installed"
-                     else
-                       echo "RPM $x does not match $VERSION_EXTENSION"
-                       echo "Will upgrade/downgrade RPMs..."
-                       # Try to upgrade/downgrade RPMS
-                       if sudo yum update -y $RPM_LIST | grep "does not update installed package"; then
-                         if ! sudo yum downgrade -y $RPM_LIST; then
-                           sudo yum remove -y opnfv-apex-undercloud opnfv-apex-common opnfv-apex-opendaylight-sfc opnfv-apex-onos
-                           sudo yum downgrade -y $RPM_INSTALL_PATH
-                         fi
-                       fi
-                       break
-                     fi
-                   done
-                else
-                   sudo yum install -y $RPM_LIST;
-                fi
-                DEPLOY_CMD=opnfv-deploy
-                DEPLOY_FILE="/etc/opnfv-apex/${DEPLOY_SCENARIO}.yaml"
-                NETWORK_FILE="/etc/opnfv-apex/network_settings.yaml"
-                export RESOURCES="/var/opt/opnfv/images"
-                export CONFIG="/var/opt/opnfv"
-            fi
-
-            if [ "$OPNFV_CLEAN" == 'yes' ]; then
-                if [[ $BUILD_DIRECTORY == *verify-master* ]]; then
-                    sudo CONFIG=${WORKSPACE}/build ./clean.sh
-                else
-                    sudo opnfv-clean
-                fi
-            fi
-            # initiate virtual deployment
-            echo "Deploy Scenario set to ${DEPLOY_SCENARIO}"
-            if [ -e $DEPLOY_FILE ]; then
-              sudo $DEPLOY_CMD -v -d ${DEPLOY_FILE} -n $NETWORK_FILE --debug
-            else
-              echo "File does not exist /etc/opnfv-apex/${DEPLOY_SCENARIO}.yaml"
-              exit 1
-            fi
-            echo
-            echo "--------------------------------------------------------"
-            echo "Done!"
+        - shell:
+            !include-raw: ./apex-deploy-virtual.sh
 
 - builder:
     name: 'apex-deploy-baremetal'
     builders:
-        - shell: |
-            #!/bin/bash
-            set -o errexit
-            set -o nounset
-            set -o pipefail
-
-            # log info to console
-            echo "Starting the Apex baremetal deployment."
-            echo "--------------------------------------------------------"
-            echo
-
-            if [[ ! "$ARTIFACT_NAME" == "latest" ]]; then
-                # if artifact name is passed the pull a
-                # specific artifact from artifacts.opnfv.org
-                RPM_INSTALL_PATH=$GS_URL/$ARTIFACT_NAME
-            else
-                if [[ $BUILD_DIRECTORY == *apex-build* ]]; then
-                  BUILD_DIRECTORY=$WORKSPACE/../$BUILD_DIRECTORY
-                  echo "BUILD DIRECTORY modified to $BUILD_DIRECTORY"
-                fi
-                if [[ -f ${BUILD_DIRECTORY}/../opnfv.properties ]]; then
-                    # if opnfv.properties exists then use the
-                    # local build. Source the file so we get local OPNFV vars
-                    source ${BUILD_DIRECTORY}/../opnfv.properties
-                    RPM_INSTALL_PATH=${BUILD_DIRECTORY}/$(basename $OPNFV_RPM_URL)
-                else
-                    # no opnfv.properties means use the latest from artifacts.opnfv.org
-                    # get the latest.properties to get the link to the latest artifact
-                    curl -s -o $WORKSPACE/opnfv.properties http://$GS_URL/latest.properties
-                    [[ -f opnfv.properties ]] || exit 1
-                    # source the file so we get OPNFV vars
-                    source opnfv.properties
-                    RPM_INSTALL_PATH=$OPNFV_RPM_URL
-                fi
-            fi
-
-            if [ ! -e "$RPM_INSTALL_PATH" ]; then
-               RPM_INSTALL_PATH=http://${OPNFV_RPM_URL}
-            fi
-
-            RPM_LIST=$RPM_INSTALL_PATH
-            for pkg in common undercloud; do
-                RPM_LIST+=" ${RPM_INSTALL_PATH/opnfv-apex/opnfv-apex-${pkg}}"
-            done
-
-            # update / install the new rpm
-            if rpm -q opnfv-apex > /dev/null; then
-               if [ $(basename $OPNFV_RPM_URL) == $(rpm -q opnfv-apex).rpm ]; then
-                 echo "RPM is already installed"
-               elif sudo yum update -y $RPM_LIST | grep "does not update installed package"; then
-                   if ! sudo yum downgrade -y $RPM_LIST; then
-                     sudo yum remove -y opnfv-undercloud opnfv-common
-                     sudo yum downgrade -y $RPM_INSTALL_PATH
-                   fi
-               fi
-            else
-               sudo yum install -y $RPM_LIST;
-            fi
-
-            # cleanup environment before we start
-            sudo opnfv-clean
-            # initiate baremetal deployment
-            if [ -e /etc/opnfv-apex/network_settings.yaml ]; then
-              if [ -n "$DEPLOY_SCENARIO" ]; then
-                echo "Deploy Scenario set to ${DEPLOY_SCENARIO}"
-                if [ -e /etc/opnfv-apex/${DEPLOY_SCENARIO}.yaml ]; then
-                  sudo opnfv-deploy -i  /root/inventory/pod_settings.yaml \
-                  -d /etc/opnfv-apex/${DEPLOY_SCENARIO}.yaml \
-                  -n /root/network/network_settings.yaml --debug
-                else
-                  echo "File does not exist /etc/opnfv-apex/${DEPLOY_SCENARIO}.yaml"
-                  exit 1
-                fi
-              else
-                echo "Deploy scenario not set!"
-                exit 1
-              fi
-            else
-              echo "File /etc/opnfv-apex/network_settings.yaml does not exist!"
-              exit 1
-            fi
-
-            echo
-            echo "--------------------------------------------------------"
-            echo "Done!"
+        - shell:
+            !include-raw: ./apex-deploy-baremetal.sh
 
 #######################
 # trigger macros