Apex: Update for master upstream deployments

Change-Id: I92635178ddc354dd7773cd372a34d46f5ee0bf2a
Signed-off-by: Tim Rozet <trozet@redhat.com>

5 files changed, 99 insertions, 57 deletions

diff --git a/jjb/apex/apex-build.sh b/jjb/apex/apex-build.sh
index cf5999832..aabd20e30 100755
--- a/jjb/apex/apex-build.sh
+++ b/jjb/apex/apex-build.sh
@@ -18,10 +18,18 @@ elif echo $BUILD_TAG | grep "csit" 1> /dev/null; then
   export BUILD_ARGS="-r $OPNFV_ARTIFACT_VERSION -c $CACHE_DIRECTORY"
 elif [ "$ARTIFACT_VERSION" == "daily" ]; then
   export OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d")
-  export BUILD_ARGS="-r $OPNFV_ARTIFACT_VERSION -c $CACHE_DIRECTORY --iso"
+  if [ "$BRANCH" == 'master' ]; then
+    export BUILD_ARGS="-r $OPNFV_ARTIFACT_VERSION -c $CACHE_DIRECTORY"
+  else
+    export BUILD_ARGS="-r $OPNFV_ARTIFACT_VERSION -c $CACHE_DIRECTORY --iso"
+  fi
 else
   export OPNFV_ARTIFACT_VERSION=${ARTIFACT_VERSION}
-  export BUILD_ARGS="-r $OPNFV_ARTIFACT_VERSION -c $CACHE_DIRECTORY --iso"
+  if [ "$BRANCH" == 'master' ]; then
+    export BUILD_ARGS="-r $OPNFV_ARTIFACT_VERSION -c $CACHE_DIRECTORY"
+  else
+    export BUILD_ARGS="-r $OPNFV_ARTIFACT_VERSION -c $CACHE_DIRECTORY --iso"
+  fi
 fi
 
 # Temporary hack until we fix apex build script
@@ -46,7 +54,7 @@ echo "Cache Directory Contents:"
 echo "-------------------------"
 ls -al $CACHE_DIRECTORY
 
-if [[ "$BUILD_ARGS" =~ '--iso' ]]; then
+if [[ "$BUILD_ARGS" =~ '--iso' && "$BRANCH" != 'master' ]]; then
   mkdir -p /tmp/apex-iso/
   rm -f /tmp/apex-iso/*.iso
   cp -f $BUILD_DIRECTORY/../.build/release/OPNFV-CentOS-7-x86_64-$OPNFV_ARTIFACT_VERSION.iso /tmp/apex-iso/
@@ -54,18 +62,32 @@ fi
 
 if ! echo $ARTIFACT_VERSION | grep "dev" 1> /dev/null; then
   echo "Writing opnfv.properties file"
-  # save information regarding artifact into file
-  (
-    echo "OPNFV_ARTIFACT_VERSION=$OPNFV_ARTIFACT_VERSION"
-    echo "OPNFV_GIT_URL=$(git config --get remote.origin.url)"
-    echo "OPNFV_GIT_SHA1=$(git rev-parse HEAD)"
-    echo "OPNFV_ARTIFACT_URL=$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso"
-    echo "OPNFV_ARTIFACT_SHA512SUM=$(sha512sum $BUILD_DIRECTORY/../.build/release/OPNFV-CentOS-7-x86_64-$OPNFV_ARTIFACT_VERSION.iso | cut -d' ' -f1)"
-    echo "OPNFV_SRPM_URL=$GS_URL/opnfv-apex-$RPM_VERSION.src.rpm"
-    echo "OPNFV_RPM_URL=$GS_URL/opnfv-apex-$RPM_VERSION.noarch.rpm"
-    echo "OPNFV_RPM_SHA512SUM=$(sha512sum $BUILD_DIRECTORY/../.build/noarch/opnfv-apex-$RPM_VERSION.noarch.rpm | cut -d' ' -f1)"
-    echo "OPNFV_BUILD_URL=$BUILD_URL"
-  ) > $WORKSPACE/opnfv.properties
+  if [ "$BRANCH" != master ]; then
+    # save information regarding artifact into file
+    (
+      echo "OPNFV_ARTIFACT_VERSION=$OPNFV_ARTIFACT_VERSION"
+      echo "OPNFV_GIT_URL=$(git config --get remote.origin.url)"
+      echo "OPNFV_GIT_SHA1=$(git rev-parse HEAD)"
+      echo "OPNFV_ARTIFACT_URL=$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso"
+      echo "OPNFV_ARTIFACT_SHA512SUM=$(sha512sum $BUILD_DIRECTORY/../.build/release/OPNFV-CentOS-7-x86_64-$OPNFV_ARTIFACT_VERSION.iso | cut -d' ' -f1)"
+      echo "OPNFV_SRPM_URL=$GS_URL/opnfv-apex-$RPM_VERSION.src.rpm"
+      echo "OPNFV_RPM_URL=$GS_URL/opnfv-apex-$RPM_VERSION.noarch.rpm"
+      echo "OPNFV_RPM_SHA512SUM=$(sha512sum $BUILD_DIRECTORY/../.build/noarch/opnfv-apex-$RPM_VERSION.noarch.rpm | cut -d' ' -f1)"
+      echo "OPNFV_BUILD_URL=$BUILD_URL"
+    ) > $WORKSPACE/opnfv.properties
+  else
+    # save information regarding artifact into file
+    # we only generate the python package for master
+    (
+      echo "OPNFV_ARTIFACT_VERSION=$OPNFV_ARTIFACT_VERSION"
+      echo "OPNFV_GIT_URL=$(git config --get remote.origin.url)"
+      echo "OPNFV_GIT_SHA1=$(git rev-parse HEAD)"
+      echo "OPNFV_SRPM_URL=$GS_URL/python34-opnfv-apex-$RPM_VERSION.src.rpm"
+      echo "OPNFV_RPM_URL=$GS_URL/python34-opnfv-apex-$RPM_VERSION.noarch.rpm"
+      echo "OPNFV_RPM_SHA512SUM=$(sha512sum $BUILD_DIRECTORY/../.build/noarch/python34-opnfv-apex-$RPM_VERSION.noarch.rpm | cut -d' ' -f1)"
+      echo "OPNFV_BUILD_URL=$BUILD_URL"
+    ) > $WORKSPACE/opnfv.properties
+  fi
 fi
 echo "--------------------------------------------------------"
 echo "Done!"
diff --git a/jjb/apex/apex-deploy.sh b/jjb/apex/apex-deploy.sh
index 958e4bf5e..64877b0d5 100755
--- a/jjb/apex/apex-deploy.sh
+++ b/jjb/apex/apex-deploy.sh
@@ -18,15 +18,22 @@ else
 fi
 
 # Dev or RPM/ISO build
-# For upstream deployments we currently only use git repo and not RPM
-# Need to decide after Fraser if we want to use RPM or not for upstream
-if [[ "$ARTIFACT_VERSION" =~ dev || "$DEPLOY_SCENARIO" =~ "upstream" ]]; then
+if [[ "$ARTIFACT_VERSION" =~ dev ]]; then
   # Settings for deploying from git workspace
   DEPLOY_SETTINGS_DIR="${WORKSPACE}/config/deploy"
   NETWORK_SETTINGS_DIR="${WORKSPACE}/config/network"
   DEPLOY_CMD="opnfv-deploy --image-dir ${WORKSPACE}/.build"
   CLEAN_CMD="opnfv-clean"
-  RESOURCES="${WORKSPACE}/.build/"
+  # if we are using master, then we are downloading/caching upstream images
+  # we want to use that built in mechanism to avoid re-downloading every job
+  # so we use a dedicated folder to hold the upstream cache
+  UPSTREAM_CACHE=$HOME/upstream_cache
+  if [ "$BRANCH" == 'master' ]; then
+    mkdir -p ${UPSTREAM_CACHE}
+    RESOURCES=$UPSTREAM_CACHE
+  else
+    RESOURCES="${WORKSPACE}/.build/"
+  fi
   CONFIG="${WORKSPACE}/build"
   BASE=$CONFIG
   IMAGES=$RESOURCES
@@ -45,7 +52,15 @@ else
   NETWORK_SETTINGS_DIR="/etc/opnfv-apex/"
   DEPLOY_CMD="opnfv-deploy"
   CLEAN_CMD="opnfv-clean"
-  RESOURCES="/var/opt/opnfv/images"
+  # set to use different directory here because upon RPM removal this
+  # directory will be wiped in daily
+  UPSTREAM_CACHE=$HOME/upstream_cache
+  if [ "$BRANCH" == 'master' ]; then
+    mkdir -p ${UPSTREAM_CACHE}
+    RESOURCES=$UPSTREAM_CACHE
+  else
+    RESOURCES="/var/opt/opnfv/images"
+  fi
   CONFIG="/var/opt/opnfv"
   BASE=$CONFIG
   IMAGES=$RESOURCES
@@ -123,7 +138,7 @@ else
   DEPLOY_CMD="${DEPLOY_CMD} -i ${INVENTORY_FILE}"
 fi
 
-if [[ "$DEPLOY_SCENARIO" =~ "upstream" ]]; then
+if [[ "$BRANCH" == "master" ]]; then
   echo "Upstream deployment detected"
   DEPLOY_CMD="${DEPLOY_CMD} --upstream"
 fi
diff --git a/jjb/apex/apex-download-artifact.sh b/jjb/apex/apex-download-artifact.sh
index 6c0f8fe26..3efe1cbc7 100755
--- a/jjb/apex/apex-download-artifact.sh
+++ b/jjb/apex/apex-download-artifact.sh
@@ -17,19 +17,20 @@ else
   echo "Deploy scenario: ${DEPLOY_SCENARIO}"
 fi
 
-# if upstream we do not need to download anything
-if [[ "$DEPLOY_SCENARIO" =~ upstream ]]; then
-  echo "Upstream deployment detected, skipping download artifact"
-elif [[ "$ARTIFACT_VERSION" =~ dev ]]; then
-  # dev build
-  GERRIT_PATCHSET_NUMBER=$(echo $GERRIT_REFSPEC | grep -Eo '[0-9]+$')
-  export OPNFV_ARTIFACT_VERSION="dev${GERRIT_CHANGE_NUMBER}_${GERRIT_PATCHSET_NUMBER}"
-  # get build artifact
-  pushd ${BUILD_DIRECTORY} > /dev/null
-  echo "Downloading packaged dev build: apex-${OPNFV_ARTIFACT_VERSION}.tar.gz"
-  curl --fail -s -o $BUILD_DIRECTORY/apex-${OPNFV_ARTIFACT_VERSION}.tar.gz http://$GS_URL/apex-${OPNFV_ARTIFACT_VERSION}.tar.gz
-  tar -xvf apex-${OPNFV_ARTIFACT_VERSION}.tar.gz
-  popd > /dev/null
+if [[ "$ARTIFACT_VERSION" =~ dev ]]; then
+  if [ "$BRANCH" == 'master' ]; then
+    echo "Skipping download of artifacts for master branch"
+  else
+    # dev build
+    GERRIT_PATCHSET_NUMBER=$(echo $GERRIT_REFSPEC | grep -Eo '[0-9]+$')
+    export OPNFV_ARTIFACT_VERSION="dev${GERRIT_CHANGE_NUMBER}_${GERRIT_PATCHSET_NUMBER}"
+    # get build artifact
+    pushd ${BUILD_DIRECTORY} > /dev/null
+    echo "Downloading packaged dev build: apex-${OPNFV_ARTIFACT_VERSION}.tar.gz"
+    curl --fail -s -o $BUILD_DIRECTORY/apex-${OPNFV_ARTIFACT_VERSION}.tar.gz http://$GS_URL/apex-${OPNFV_ARTIFACT_VERSION}.tar.gz
+    tar -xvf apex-${OPNFV_ARTIFACT_VERSION}.tar.gz
+    popd > /dev/null
+  fi
 else
   echo "Will use RPMs..."
 
@@ -45,20 +46,16 @@ else
   RPM_INSTALL_PATH=$(echo "http://"$OPNFV_RPM_URL | sed 's/\/'"$(basename $OPNFV_RPM_URL)"'//')
   RPM_LIST=$(basename $OPNFV_RPM_URL)
 
-  # find version of RPM
-  VERSION_EXTENSION=$(echo $(basename $RPM_LIST) | grep -Eo '[0-9]+\.[0-9]+-([0-9]{8}|[a-z]+-[0-9]\.[0-9]+)')
-  # build RPM List which already includes base Apex RPM
-  RPM_LIST+=" opnfv-apex-undercloud-${VERSION_EXTENSION}.noarch.rpm"
-
-  # add back legacy support for danube
-  if [ "$BRANCH" == 'stable/danube' ]; then
-    RPM_LIST+=" opnfv-apex-common-${VERSION_EXTENSION}.noarch.rpm"
-  else
+  if [ "$BRANCH" != 'master' ]; then
+    # find version of RPM
+    VERSION_EXTENSION=$(echo $(basename $RPM_LIST) | grep -Eo '[0-9]+\.[0-9]+-([0-9]{8}|[a-z]+-[0-9]\.[0-9]+)')
+    # build RPM List which already includes base Apex RPM
+    RPM_LIST+=" opnfv-apex-undercloud-${VERSION_EXTENSION}.noarch.rpm"
     RPM_LIST+=" python34-opnfv-apex-${VERSION_EXTENSION}.noarch.rpm"
   fi
 
   # remove old / install new RPMs
-  if rpm -q opnfv-apex > /dev/null; then
+  if rpm -q python34-opnfv-apex > /dev/null; then
     INSTALLED_RPMS=$(rpm -qa | grep apex)
     if [ -n "$INSTALLED_RPMS" ]; then
       sudo yum remove -y ${INSTALLED_RPMS}
diff --git a/jjb/apex/apex-upload-artifact.sh b/jjb/apex/apex-upload-artifact.sh
index 4037d25ad..8743368fb 100755
--- a/jjb/apex/apex-upload-artifact.sh
+++ b/jjb/apex/apex-upload-artifact.sh
@@ -109,8 +109,8 @@ fi
 if [ "$ARTIFACT_TYPE" == 'snapshot' ]; then
   uploadsnap
 elif [ "$ARTIFACT_TYPE" == 'iso' ]; then
-  if [[ "$ARTIFACT_VERSION" =~ dev ]]; then
-    echo "Skipping artifact upload for ${ARTIFACT_TYPE} due to dev build"
+  if [[ "$ARTIFACT_VERSION" =~ dev || "$BRANCH" == 'master' ]]; then
+    echo "Skipping ISO artifact upload for ${ARTIFACT_TYPE} due to dev/master build"
     exit 0
   fi
   if [[ -n "$SIGN_ARTIFACT" && "$SIGN_ARTIFACT" == "true" ]]; then
@@ -119,20 +119,28 @@ elif [ "$ARTIFACT_TYPE" == 'iso' ]; then
   uploadiso
 elif [ "$ARTIFACT_TYPE" == 'rpm' ]; then
   if [[ "$ARTIFACT_VERSION" =~ dev ]]; then
-    echo "dev build detected, will upload image tarball"
-    ARTIFACT_TYPE=tarball
-    uploadimages
+    if [ "$BRANCH" == 'master' ]; then
+      echo "will not upload artifacts, master uses upstream"
+      ARTIFACT_TYPE=none
+    else
+      echo "dev build detected, will upload image tarball"
+      ARTIFACT_TYPE=tarball
+      uploadimages
+    fi
   else
     RPM_INSTALL_PATH=$BUILD_DIRECTORY/noarch
+    # RPM URL should be python package for master, and is only package we need
     RPM_LIST=$RPM_INSTALL_PATH/$(basename $OPNFV_RPM_URL)
-    VERSION_EXTENSION=$(echo $(basename $OPNFV_RPM_URL) | sed 's/opnfv-apex-//')
-    RPM_LIST+=" ${RPM_INSTALL_PATH}/opnfv-apex-undercloud-${VERSION_EXTENSION}"
-    RPM_LIST+=" ${RPM_INSTALL_PATH}/python34-opnfv-apex-${VERSION_EXTENSION}"
     SRPM_INSTALL_PATH=$BUILD_DIRECTORY
     SRPM_LIST=$SRPM_INSTALL_PATH/$(basename $OPNFV_SRPM_URL)
-    VERSION_EXTENSION=$(echo $(basename $OPNFV_SRPM_URL) | sed 's/opnfv-apex-//')
-    SRPM_LIST+=" ${SRPM_INSTALL_PATH}/opnfv-apex-undercloud-${VERSION_EXTENSION}"
-    SRPM_LIST+=" ${SRPM_INSTALL_PATH}/python34-opnfv-apex-${VERSION_EXTENSION}"
+    if [ "$BRANCH" != 'master' ]; then
+      VERSION_EXTENSION=$(echo $(basename $OPNFV_RPM_URL) | sed 's/opnfv-apex-//')
+      RPM_LIST+=" ${RPM_INSTALL_PATH}/opnfv-apex-undercloud-${VERSION_EXTENSION}"
+      RPM_LIST+=" ${RPM_INSTALL_PATH}/python34-opnfv-apex-${VERSION_EXTENSION}"
+      VERSION_EXTENSION=$(echo $(basename $OPNFV_SRPM_URL) | sed 's/opnfv-apex-//')
+      SRPM_LIST+=" ${SRPM_INSTALL_PATH}/opnfv-apex-undercloud-${VERSION_EXTENSION}"
+      SRPM_LIST+=" ${SRPM_INSTALL_PATH}/python34-opnfv-apex-${VERSION_EXTENSION}"
+    fi
 
     if [[ -n "$SIGN_ARTIFACT" && "$SIGN_ARTIFACT" == "true" ]]; then
       signrpm
diff --git a/jjb/apex/scenarios.yaml.hidden b/jjb/apex/scenarios.yaml.hidden
index 91cd789c5..2650eafbb 100644
--- a/jjb/apex/scenarios.yaml.hidden
+++ b/jjb/apex/scenarios.yaml.hidden
@@ -1,6 +1,6 @@
 master:
-  - 'os-odl-master_upstream-noha'
-  - 'os-odl-queens_upstream-noha'
+  - 'os-odl-nofeature-noha'
+  - 'os-odl-queens-noha'
 fraser:
   - 'os-nosdn-nofeature-noha'
   - 'os-nosdn-nofeature-ha'