Updates to Caching

- updates cache to use the cache dir as a linked representation
  of the build dir
- fixing cache build/extract

JIRA: APEX-325

Change-Id: I80bf67e5d55ca2e069ca72e6b47303368bda0433
Signed-off-by: Dan Radez <dradez@redhat.com>

3 files changed, 25 insertions, 11 deletions

diff --git a/build/cache.sh b/build/cache.sh
index 5c7e5e73..0c2da399 100644
--- a/build/cache.sh
+++ b/build/cache.sh
@@ -8,12 +8,14 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-CACHE_DIR="$(pwd)/cache"
+CACHE_DIR="$(pwd)/.cache"
+CACHE_HISTORY=".cache_history"
 
 # Make sure the cache dir exists
 function cache_dir {
+    if [ -f $CACHE_DIR ]; then rm -rf $CACHE_DIR; fi
     if [ ! -d $CACHE_DIR/ ]; then mkdir $CACHE_DIR/; fi
-    if [ ! -f $CACHE_DIR/.cache ]; then touch $CACHE_DIR/.cache; fi
+    if [ ! -f $CACHE_DIR/$CACHE_HISTORY ]; then touch $CACHE_DIR/$CACHE_HISTORY; fi
     echo "Cache Dir: $CACHE_DIR"
 }
 
@@ -33,8 +35,8 @@ function curl_file {
     until curl -C- -L -o $CACHE_DIR/$2 $1  || (( count++ >= 20 )); do
         echo -n '' #do nothing, we just want to loop
     done
-    sed -i "/$2/d" $CACHE_DIR/.cache
-    echo "$(md5sum $CACHE_DIR/$2) $2" >> $CACHE_DIR/.cache
+    sed -i "/$2/d" $CACHE_DIR/$CACHE_HISTORY
+    echo "$(md5sum $CACHE_DIR/$2) $2" >> $CACHE_DIR/$CACHE_HISTORY
 }
 
 # $1 =  download url
@@ -63,9 +65,14 @@ function populate_cache {
             echo "Got empty MD5 from remote for $filename, skipping MD5 check"
             curl_file $1 $filename
         else
-            my_md5=$(grep ${filename} $CACHE_DIR/.cache | awk {'print $1'})
+            my_md5=$(grep ${filename} $CACHE_HISTORY | awk {'print $1'})
+            if [ -z "$my_md5" ]; then
+                echo "${filename} missing in $CACHE_HISTORY file. Caculating md5..."
+                my_md5=$(md5sum ${CACHE_DIR}/${filename} | awk {'print $1'})
+            fi
             if [ "$remote_md5" != "$my_md5" ]; then
-                echo "MD5 mismatch, cache file MD5 is ${my_md5}"
+                echo "MD5 mismatch, local cache file MD5 is ${my_md5}"
+                echo "              remote cache file MD5 is ${remote_md5}"
                 echo "Downloading $filename"
                 curl_file $1 $filename
             else
@@ -76,6 +83,14 @@ function populate_cache {
 }
 
 # $1 = filename to get from cache
+# $2 = destintation
 function get_cached_file {
-  cp -f $CACHE_DIR/$1 .
+    if [ ! -f $CACHE_DIR/$1 ]; then
+        echo "Cache file: ${CACHE_DIR}/$1 is not in cache."
+    else
+        echo "Cache file: Using cached file ${CACHE_DIR}/$1."
+        dest='.'
+        if [ -n $2 ]; then dest=$2; fi
+        cp -f $CACHE_DIR/$1 $dest
+    fi
 }
diff --git a/build/overcloud-opendaylight.sh b/build/overcloud-opendaylight.sh
index 4152c195..49c5f21e 100755
--- a/build/overcloud-opendaylight.sh
+++ b/build/overcloud-opendaylight.sh
@@ -53,8 +53,7 @@ gpgcheck=0
 EOF
 
 # SDNVPN - Copy tunnel setup script
-wget https://raw.githubusercontent.com/openstack/fuel-plugin-opendaylight/brahmaputra-sr2/deployment_scripts/puppet/modules/opendaylight/templates/setup_TEPs.py
-
+curl -fO https://raw.githubusercontent.com/openstack/fuel-plugin-opendaylight/brahmaputra-sr2/deployment_scripts/puppet/modules/opendaylight/templates/setup_TEPs.py
 
 # install ODL packages
 # install Jolokia for ODL HA
diff --git a/build/undercloud.sh b/build/undercloud.sh
index 39885faf..d0bab527 100755
--- a/build/undercloud.sh
+++ b/build/undercloud.sh
@@ -40,7 +40,7 @@ LIBGUESTFS_BACKEND=direct virt-customize \
     --run-command "cd /usr/share && rm -rf openstack-tripleo-heat-templates && tar xzf opnfv-tht.tar.gz" \
     --run-command "sed -i '/ControllerEnableCephStorage/c\\  ControllerEnableCephStorage: true' /usr/share/openstack-tripleo-heat-templates/environments/storage-environment.yaml" \
     --run-command "sed -i '/ComputeEnableCephStorage/c\\  ComputeEnableCephStorage: true' /usr/share/openstack-tripleo-heat-templates/environments/storage-environment.yaml" \
-    --run-command "curl http://download.opensuse.org/repositories/Openwsman/CentOS_CentOS-7/Openwsman.repo > /etc/yum.repos.d/wsman.repo" \
+    --run-command "curl -f http://download.opensuse.org/repositories/Openwsman/CentOS_CentOS-7/Openwsman.repo > /etc/yum.repos.d/wsman.repo" \
     --run-command "yum update -y openwsman*" \
     --run-command "cp /usr/share/instack-undercloud/undercloud.conf.sample /home/stack/undercloud.conf && chown stack:stack /home/stack/undercloud.conf" \
     --upload ../opnfv-environment.yaml:/home/stack/ \
@@ -63,7 +63,7 @@ LIBGUESTFS_BACKEND=direct virt-customize \
     -a undercloud_build.qcow2
 
 # Add custom IPA to allow kernel params
-wget https://raw.githubusercontent.com/trozet/ironic-python-agent/opnfv_kernel/ironic_python_agent/extensions/image.py
+curl -fO https://raw.githubusercontent.com/trozet/ironic-python-agent/opnfv_kernel/ironic_python_agent/extensions/image.py
 python3 -c 'import py_compile; py_compile.compile("image.py", cfile="image.pyc")'
 
 # Add performance image scripts