Fuel build cache handling

- Cache invalidation supported in case cache is outdated compared to upstream.
- Fully automated build.sh integration tests added (build.sh -t or build.sh -T)
- Rebased to Daniel Smiths's patch-set: genesis refs/changes/71/171/4
- Yet untested, will perform automated integration tests over night!
- Updated based on Stefan's comments (no more revs.)
- Fixed a bug in f_odl_docker/Makefile
- validate-cache target added in f_odl_docker/Makefile

JIRA: BGS-1

Change-Id: I9f8f30c2a85fd7bf04dab1b566065e5739d4fcb1
Signed-off-by: Jonas Bjurel <jonas.bjurel@ericsson.com>

17 files changed, 551 insertions, 206 deletions

diff --git a/fuel/build/Makefile b/fuel/build/Makefile
index b6b0eac..364fe31 100644
--- a/fuel/build/Makefile
+++ b/fuel/build/Makefile
@@ -8,11 +8,10 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
+SHELL = /bin/bash
 ############################################################################
 # BEGIN of variables to customize
 #
-SHELL = /bin/bash
-
 #Input args
 export UNIT_TEST = FALSE
 export INTERACTIVE = TRUE
@@ -31,20 +30,10 @@ export DEB_DEST := $(BUILD_BASE)/release/packages/ubuntu/pool/main
 export PUPPET_DEST := $(BUILD_BASE)/release/puppet/modules
 export CACHE_DIR := $(BUILD_BASE)/cache
 export VERSION_FILE := $(BUILD_BASE)/.versions
-DOCKERIMG = opnfv.org/ubuntu-builder:14.04
+export DOCKERIMG = opnfv.org/ubuntu-builder:14.04
 export TOPDIR := $(shell pwd)
 
-
-#
-# END of variables to customize
-#############################################################################
-
-#Include definitions
-include config.mk
-include cache.mk
-
-
-ORIGDIR := $(TOPDIR)/origiso
+#Build subclasses
 
 SUBDIRS := f_isoroot
 SUBDIRS += f_opnfv_puppet
@@ -53,32 +42,43 @@ SUBDIRS += f_l23network
 SUBDIRS += f_resolvconf
 SUBDIRS += f_ntp
 SUBDIRS += f_odl_docker
-
-
+#SUBDIRS += f_odl
 
 # f_example is only an example of how to generate a .deb package and
 # should not be enabled in official builds.
 #SUBDIRS += f_example
 
+ORIGDIR := $(TOPDIR)/origiso
+#
+# END of variables to customize
+#############################################################################
+
 SUBCLEAN = $(addsuffix .clean,$(SUBDIRS))
 
+
 .PHONY: all
 all:
 	@docker version >/dev/null 2>&1 || (echo 'No Docker installation available'; exit 1)
-	@echo "Versions of cached build results built by" $(shell hostname) "at" $(shell date -u) > $(BUILD_BASE)/.versions
-	@echo "cache.mk" $(shell md5sum $(BUILD_BASE)/cache.mk | cut -f1 -d " ") >> $(BUILD_BASE)/.versions
-	@echo "config.mk" $(shell md5sum $(BUILD_BASE)/config.mk | cut -f1 -d " ") >> $(BUILD_BASE)/.versions
+	@echo "Versions of cached build results built by" $(shell hostname) "at" $(shell date -u) > $(VERSION_FILE)
+	@echo "cache.mk" $(shell md5sum $(BUILD_BASE)/cache.mk | cut -f1 -d " ") >> $(VERSION_FILE)
+	@echo "config.mk" $(shell md5sum $(BUILD_BASE)/config.mk | cut -f1 -d " ") >> $(VERSION_FILE)
 	@make -C docker
 	@docker/runcontext $(DOCKERIMG) $(MAKE) $(MAKEFLAGS) iso
 
-
+############################################################################
+# BEGIN of Include definitions
+#
+include config.mk
+include cache.mk
+#
+# END Include definitions
+#############################################################################
 
 $(ISOCACHE):
 	# Clone Fuel to non-persistent location and build
 	cd /tmp && git clone $(FUEL_MAIN_REPO)
 	cd /tmp/fuel-main && git checkout $(FUEL_MAIN_TAG)
-	@echo "fuel" `git -C /tmp/fuel-main show | grep commit | rev | cut -f1 -d " "` >> ${BUILD_BASE}/.versions
-
+	@echo "fuel" `git -C /tmp/fuel-main show | grep commit | head -1 | cut -d " " -f2` >> $(VERSION_FILE)
 	# Setup cgroups for docker-in-docker
 	sudo /root/enable_dockerx2
 	# Patch to fix race condition when doing "Docker-in-Docker" build
@@ -130,7 +130,7 @@ build-clean: $(SUBCLEAN)
 	@rm -f $(NEWISO)
 
 .PHONY: clean $(SUBCLEAN)
-clean:  prepare $(SUBCLEAN)
+clean:  clean-cache prepare $(SUBCLEAN)
 	$(MAKE) -C patch-packages -f Makefile clean
 	#$(MAKE) -C opendaylight -f Makefile clean
 	@rm -f *.iso
diff --git a/fuel/build/cache.mk b/fuel/build/cache.mk
index a3f3516..a2aa1d5 100644
--- a/fuel/build/cache.mk
+++ b/fuel/build/cache.mk
@@ -8,23 +8,22 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
+SHELL = /bin/bash
+CACHEVALIDATE := $(addsuffix .validate,$(SUBDIRS))
+CACHECLEAN := $(addsuffix .clean,$(CACHEFILES) $(CACHEDIRS))
+
 ############################################################################
 # BEGIN of variables to customize
 #
-SHELL = /bin/bash
-
-#export BUILD_BASE = $(shell pwd)
-#export CACHE_DIR = "$(BUILD_BASE)/cache"
-
 CACHEDIRS := opendaylight/f_odl/package
 
 CACHEFILES := opendaylight/.odl-build-history
 CACHEFILES += opendaylight/.odl-build.log
 CACHEFILES += .versions
-CACHEFILES += fuel-6.0.1.iso
-#CACHEFILES += $(ISOSRC)
-
-CACHECLEAN = $(addsuffix .clean,$(CACHEDIRS))
+CACHEFILES += $(shell basename $(ISOSRC))
+#
+# END of variables to customize
+############################################################################
 
 .PHONY: prepare-cache
 prepare-cache: make-cache-dir $(CACHEDIRS) $(CACHEFILES)
@@ -34,54 +33,66 @@ make-cache-dir:
 	@rm -rf ${CACHE_DIR}
 	@mkdir ${CACHE_DIR}
 
-
 .PHONY: clean-cache
 clean-cache: $(CACHECLEAN)
 	@rm -rf ${CACHE_DIR}
 
 .PHONY: $(CACHEDIRS)
 $(CACHEDIRS):
-	@mkdir -p $(dir ${CACHE_DIR}/$@)
-	@if [ ! -d ${BUILD_BASE}/$@ ]; then\
-	   mkdir -p $(dir ${BUILD_BASE}/$@);\
- 	   ln -s ${BUILD_BASE}/$@ ${CACHE_DIR}/$@;\
-	   rm -rf ${BUILD_BASE}/$@;\
-	else\
-	   ln -s ${BUILD_BASE}/$@ ${CACHE_DIR}/$@;\
+	@mkdir -p $(dir $(CACHE_DIR)/$@)
+	@if [ ! -d $(BUILD_BASE)/$@ ]; then\
+	   mkdir -p $(BUILD_BASE)/$@;\
 	fi
+	@ln -s $(BUILD_BASE)/$@ $(CACHE_DIR)/$@
 
 .PHONY: $(CACHEFILES)
 $(CACHEFILES):
-	@mkdir -p $(dir ${CACHE_DIR}/$@)
-	@if [ ! -f ${BUILD_BASE}/$@ ]; then\
-	   mkdir $(dir ${BUILD_BASE}/$@);\
-	   echo " " > ${BUILD_BASE}/$@;\
- 	   ln -s ${BUILD_BASE}/$@ ${CACHE_DIR}/$@;\
-	   rm -f ${BUILD_BASE}/$@;\
+	@mkdir -p $(dir $(CACHE_DIR)/$@)
+	@if [ ! -d $(dir $(BUILD_BASE)/$@) ]; then\
+	   mkdir -p $(dir $(BUILD_BASE)/$@);\
+	fi
+
+	@if [ ! -f $(BUILD_BASE)/$@ ]; then\
+	   echo " " > $(BUILD_BASE)/$@;\
+ 	   ln -s $(BUILD_BASE)/$@ $(CACHE_DIR)/$@;\
+	   rm -f $(BUILD_BASE)/$@;\
 	else\
-	   ln -s ${BUILD_BASE}/$@ ${CACHE_DIR}/$@;\
+	   ln -s $(BUILD_BASE)/$@ $(CACHE_DIR)/$@;\
 	fi
 
 .PHONY: validate-cache
-validate-cache:
-#	if [ $(shell md5sum ${BUILD_BASE}/config.mk) -ne $(shell cat ${CACHE_DIR}/.versions | grep config.mk awk '{print $NF}') ]; then\
+validate-cache: prepare $(CACHEVALIDATE)
+	@if [[ $(shell md5sum $(BUILD_BASE)/config.mk | cut -f1 -d " ") != $(shell cat $(VERSION_FILE) | grep config.mk | awk '{print $$NF}') ]]; then\
 	   echo "Cache does not match current config.mk definition, cache must be rebuilt";\
 	   exit 1;\
 	fi;
 
-#	if [ $(shell md5sum ${BUILD_BASE}/cache.mk) -ne $(shell cat ${CACHE_DIR}/.versions | grep config.mk awk '{print $NF}') ]; then\
+	@if [[ $(shell md5sum $(BUILD_BASE)/cache.mk | cut -f1 -d " ") != $(shell cat $(VERSION_FILE) | grep cache.mk | awk '{print $$NF}') ]]; then\
 	   echo "Cache does not match current cache.mk definition, cache must be rebuilt";\
 	   exit 1;\
 	fi;
 
-#	$(MAKE) -C opendaylight validate-cache
-#	if [ $? -ne 0 ]; then\
-	   echo "Cache does not match current OpenDaylight version, cach must be rebuilt";\
-	   exit 1;\
-	fi;
+# Once the Make structure is refactored, this should go in as a validate-cache
+# taget in the fuel Makefile
+
+	@REMOTE_ID=$(shell git ls-remote $(FUEL_MAIN_REPO) $(FUEL_MAIN_TAG)^{} | awk '{print $$(NF-1)}'); \
+	if [ -z $$REMOTE_ID ] || [ $$REMOTE_ID = " " ]; \
+	then \
+	   REMOTE_ID=$(shell git ls-remote $(FUEL_MAIN_REPO) $(FUEL_MAIN_TAG) | awk '{print $$(NF-1)}'); \
+	fi; \
+	if [ $$REMOTE_ID != $(shell cat $(VERSION_FILE) | grep fuel | awk '{print $$NF}') ]; \
+	then \
+	   echo "Cache does not match upstream Fuel, cache must be rebuilt!"; \
+	   exit 1; \
+	fi
+
+	#$(MAKE) -C opendaylight validate-cache
 
-#       $(SUBDIRS)
+.PHONY: $(CACHEVALIDATE)
+$(CACHEVALIDATE): %.validate:
+	@echo VALIDATE $(CACHEVALIDATE)
+	$(MAKE) -C $* -f Makefile validate-cache
 
 .PHONY: $(CACHECLEAN)
 $(CACHECLEAN): %.clean:
-	rm -f ${CACHE_DIR}/$*
+	rm -rf ${CACHE_DIR}/$*
diff --git a/fuel/build/config.mk b/fuel/build/config.mk
index 49b0173..19f502d 100644
--- a/fuel/build/config.mk
+++ b/fuel/build/config.mk
@@ -16,3 +16,12 @@ FUEL_MAIN_TAG = stable/6.0
 
 DOCKER_REPO := http://get.docker.com/builds/Linux/x86_64
 DOCKER_TAG := docker-latest
+
+.PHONY: get-odl-repo
+get-odl-repo:
+	@echo $(ODL_MAIN_REPO) $(ODL_MAIN_TAG)
+
+.PHONY: get-fuel-repo
+get-fuel-repo:
+	@echo $(FUEL_MAIN_REPO) $(FUEL_MAIN_TAG)
+
diff --git a/fuel/build/f_example_control_bond/Makefile b/fuel/build/f_example_control_bond/Makefile
index c311a2d..0949737 100644
--- a/fuel/build/f_example_control_bond/Makefile
+++ b/fuel/build/f_example_control_bond/Makefile
@@ -18,6 +18,10 @@ clean:
 	@rm -rf tmp
 	@rm -rf release
 
+.PHONY: validate-cache
+validate-cache:
+	@echo "No cache validation schema available for $(shell pwd)"
+	@echo "Continuing ..."
 
 .PHONY: release
 release:
diff --git a/fuel/build/f_example_packadd/Makefile b/fuel/build/f_example_packadd/Makefile
index eadc83f..a64293d 100644
--- a/fuel/build/f_example_packadd/Makefile
+++ b/fuel/build/f_example_packadd/Makefile
@@ -31,6 +31,10 @@ clean:
 	@rm -rf release
 	@rm -f ../release/packages/ubuntu/pool/main/$(DEB_NAME)
 
+.PHONY: validate-cache
+validate-cache:
+	@echo "No cache validation schema available for $(shell pwd)"
+	@echo "Continuing ..."
 
 .PHONY: release
 release:release/pool/main/$(DEB_NAME)
diff --git a/fuel/build/f_isoroot/Makefile b/fuel/build/f_isoroot/Makefile
index 2fdbbae..bde8e64 100644
--- a/fuel/build/f_isoroot/Makefile
+++ b/fuel/build/f_isoroot/Makefile
@@ -26,6 +26,10 @@ clean: $(SUBCLEAN)
 $(SUBCLEAN): %.clean:
 	$(MAKE) -C $* -f Makefile clean
 
+validate-cache:
+	@echo "No cache validation schema available for $(shell pwd)"
+	@echo "Continuing ..."
+
 .PHONY: release
 release:clean $(SUBDIRS)
 	@cp -Rvp release/* ../release/isoroot
diff --git a/fuel/build/f_l23network/Makefile b/fuel/build/f_l23network/Makefile
index c311a2d..0949737 100644
--- a/fuel/build/f_l23network/Makefile
+++ b/fuel/build/f_l23network/Makefile
@@ -18,6 +18,10 @@ clean:
 	@rm -rf tmp
 	@rm -rf release
 
+.PHONY: validate-cache
+validate-cache:
+	@echo "No cache validation schema available for $(shell pwd)"
+	@echo "Continuing ..."
 
 .PHONY: release
 release:
diff --git a/fuel/build/f_ntp/Makefile b/fuel/build/f_ntp/Makefile
index c311a2d..0949737 100644
--- a/fuel/build/f_ntp/Makefile
+++ b/fuel/build/f_ntp/Makefile
@@ -18,6 +18,10 @@ clean:
 	@rm -rf tmp
 	@rm -rf release
 
+.PHONY: validate-cache
+validate-cache:
+	@echo "No cache validation schema available for $(shell pwd)"
+	@echo "Continuing ..."
 
 .PHONY: release
 release:
diff --git a/fuel/build/f_odl_docker/Makefile b/fuel/build/f_odl_docker/Makefile
index b227bee..6135e71 100755
--- a/fuel/build/f_odl_docker/Makefile
+++ b/fuel/build/f_odl_docker/Makefile
@@ -12,7 +12,7 @@ TOP := $(shell pwd)
 BUILDTAG := loving_daniel
 
 # Edit this to match the GENESIS / OPNFV in your environment
-export OPNFV_PUPPET := /home/lmcdasm/MAR24/genesis/common/puppet-opnfv
+export OPNFV_PUPPET := $(BUILD_BASE)/../../common/puppet-opnfv
 include ../config.mk
 
 .PHONY: all
@@ -25,7 +25,7 @@ all:
 	@docker save ${BUILDTAG} > puppet/modules/opnfv/odl_docker/odl_docker_image.tar
 	@wget ${DOCKER_REPO}/${DOCKER_TAG} -O  puppet/modules/opnfv/odl_docker/docker-latest
 	@echo "OPFNV_PUPPET is: ${OPNFV_PUPPET}"
-	@cp -Rvp ${OPNFV_PUPPET}/manifests/templates/dockerfile/compute_scripts  puppet/modules/opnfv
+	@cp -Rvp ${OPNFV_PUPPET}/manifests/templates/dockerfile/container_scripts  puppet/modules/opnfv
 
 .PHONY: clean
 clean:
@@ -39,9 +39,13 @@ build-clean:
 	@rm -rf puppet/modules/opnfv/odl_docker/odl_docker_image.tar
 	@rm -rf puppet/modules/opnfv/odl_docker/docker-latest
 
+.PHONY: validate-cache
+validate-cache:
+	@echo "No cache validation schema available for $(shell pwd)"
+	@echo "Continuing ..."
 
 .PHONY: release
 release:
 	# Fetch PP from OPNFV Common
-	@cp -Rvp ${OPNFV_PUPPET}/common/puppet-opnfv/manifests/odl_docker.pp ${PUPPET_DEST}
+	@cp -Rvp ${OPNFV_PUPPET}/manifests/odl_docker.pp ${PUPPET_DEST}
 	@cp -Rvp puppet/modules/* $(PUPPET_DEST)
diff --git a/fuel/build/f_opnfv_puppet/Makefile b/fuel/build/f_opnfv_puppet/Makefile
index c311a2d..0949737 100644
--- a/fuel/build/f_opnfv_puppet/Makefile
+++ b/fuel/build/f_opnfv_puppet/Makefile
@@ -18,6 +18,10 @@ clean:
 	@rm -rf tmp
 	@rm -rf release
 
+.PHONY: validate-cache
+validate-cache:
+	@echo "No cache validation schema available for $(shell pwd)"
+	@echo "Continuing ..."
 
 .PHONY: release
 release:
diff --git a/fuel/build/f_osnaily/Makefile b/fuel/build/f_osnaily/Makefile
index c311a2d..0949737 100644
--- a/fuel/build/f_osnaily/Makefile
+++ b/fuel/build/f_osnaily/Makefile
@@ -18,6 +18,10 @@ clean:
 	@rm -rf tmp
 	@rm -rf release
 
+.PHONY: validate-cache
+validate-cache:
+	@echo "No cache validation schema available for $(shell pwd)"
+	@echo "Continuing ..."
 
 .PHONY: release
 release:
diff --git a/fuel/build/f_resolvconf/Makefile b/fuel/build/f_resolvconf/Makefile
index c311a2d..0949737 100644
--- a/fuel/build/f_resolvconf/Makefile
+++ b/fuel/build/f_resolvconf/Makefile
@@ -18,6 +18,10 @@ clean:
 	@rm -rf tmp
 	@rm -rf release
 
+.PHONY: validate-cache
+validate-cache:
+	@echo "No cache validation schema available for $(shell pwd)"
+	@echo "Continuing ..."
 
 .PHONY: release
 release:
diff --git a/fuel/build/opendaylight/  b/fuel/build/opendaylight/ 
new file mode 120000
index 0000000..0519ecb
--- /dev/null
+++ b/fuel/build/opendaylight/ 
@@ -0,0 +1 @@
+ 
\ No newline at end of file
diff --git a/fuel/build/opendaylight/Makefile b/fuel/build/opendaylight/Makefile
index c112ff6..bd2eeb5 100644
--- a/fuel/build/opendaylight/Makefile
+++ b/fuel/build/opendaylight/Makefile
@@ -12,7 +12,7 @@
 # BEGIN of variables to customize
 #
 SHELL = /bin/bash
-include ../config.mk
+
 
 BUILD_DIR := $(shell pwd)
 GIT_DIR := /tmp
@@ -32,14 +32,31 @@ MAVEN_SPEC = $(BUILD_DIR)/odl_maven/settings.xml
 .PHONY: all
 all:	odl
 
+############################################################################
+# BEGIN of Include definitions
+#
+include ../config.mk
+#
+# END Include definitions
+#############################################################################
+
 .PHONY: setup
 setup:
 	rm -f "$(BUILD_BASE)/f_odl"
 	ln -s "$(shell readlink -e $(BUILD_DIR))/f_odl" "$(shell readlink -e $(BUILD_BASE))/f_odl"
 
-.PHONY: check-cache-validity
-check-cache-validity:
-	@if [ $(git ls-remote $(ODL_MAIN_REPO) $(ODL_MAIN_TAG) | awk '{print $(NF-1)}') -ne $(cat $(CACHE_DIR)/.versions | grep odl | awk '{print $NF}') ]; then @echo "Cache is not up to date, stopping!"; exit 1; fi
+.PHONY: validate-cache
+validate-cache:
+	@REMOTE_ID=$(shell git ls-remote $(ODL_MAIN_REPO) $(ODL_MAIN_TAG)^{} | awk '{print $$(NF-1)}'); \
+	if [ -z $$REMOTE_ID ] || [ $$REMOTE_ID = " " ]; \
+	then \
+	   REMOTE_ID=$(shell git ls-remote $(ODL_MAIN_REPO) $(ODL_MAIN_TAG) | awk '{print $$(NF-1)}'); \
+	fi; \
+	if [ $$REMOTE_ID != $(shell cat $(VERSION_FILE) | grep odl | awk '{print $$NF}') ]; \
+	then \
+	   echo "Cache does not match upstream OpenDaylight, cache must be rebuilt!"; \
+	   exit 1; \
+	fi
 
 .PHONY: odl
 odl:
@@ -69,7 +86,7 @@ else
 	   mvn -gs $(MAVEN_SPEC) clean install;\
 	fi;
 
-	@echo "odl" `git -C /tmp/controller show | grep commit | rev | cut -f1 -d " "` >> $(BUILD_BASE)/.versions
+	@echo "odl" `git -C /tmp/controller show | grep commit | head -1 | cut -d " " -f2` >> $(VERSION_FILE)
 	@./make-odl-deb.sh -N $(ODL_SHORT_NAME)_`cd /tmp/controller; git rev-parse --short HEAD` -n $(ODL_SHORT_NAME) -v "$(ODL_VERSION)" -t "$(ODL_MAIN_TAG)" -m $(MAINTAINER) -d $(DEPEND) -p $(TARGET_BUILD_PATH)
 	@echo $(ODL_MAIN_TAG) > .odl-build.log
 endif
diff --git a/fuel/build/opendaylight/f_odl/Makefile b/fuel/build/opendaylight/f_odl/Makefile
index 6983f0c..f7ebd3e 100644
--- a/fuel/build/opendaylight/f_odl/Makefile
+++ b/fuel/build/opendaylight/f_odl/Makefile
@@ -36,6 +36,10 @@ clean:
 	@rm -rf release
 	@rm -f $(DEB_DEST)/$(DEB_NAME).deb
 
+.PHONY: validate-cache
+validate-cache:
+	@echo "No cache validation schema available for $(shell pwd)"
+	@echo "Continuing ..."
 
 .PHONY: release
 release:release/pool/main/$(DEB_NAME).deb
diff --git a/fuel/ci/build.sh b/fuel/ci/build.sh
index bb0d124..cdaa6ef 100755
--- a/fuel/ci/build.sh
+++ b/fuel/ci/build.sh
@@ -17,6 +17,9 @@ if [ -f ${LOCK_FILE} ]; then \
    fi; \
 fi;' EXIT
 
+############################################################################
+# BEGIN of usage description
+#
 usage ()
 {
 cat << EOF
@@ -31,6 +34,8 @@ OPTIONS:
   -l log-file ($BUILD_LOG), specifies the output log-file (stdout and stderr), if not specified logs are output to console as normal
   -v version tag to be applied to the build result
   -r alternative remote access method script/program. curl is default.
+  -t run small build-script unit test.
+  -T run large build-script unit test.
   -f build flags ($BUILD_FLAGS):
      o s: Do nothing, succeed
      o f: Do nothing, fail
@@ -70,136 +75,13 @@ build -c http://opnfv.org/artifactory/fuel/cache -d ~/jenkins/genesis/fuel/ci/ou
 NOTE: At current the build scope is set to the git root of the repository, -d destination locations outside that scope will not work
 EOF
 }
+#
+# END of usage description
+############################################################################
 
-
-
-debug_make () {
-    make -C ${BUILD_BASE} clean
-    echo "This is a fake debug fuel .iso image" > ${BUILD_BASE}/fuel-6.0.1.iso
-    echo "This is a fake debug versions file" > ${BUILD_BASE}/.versions
-    rm -rf ${BUILD_BASE}/release
-    mkdir ${BUILD_BASE}/release
-    echo "This is a fake debug OPNFV .iso image"  > ${BUILD_BASE}/release/fake-debug.iso
-    echo "This is a fake debug OPNFV .iso.txt message"  > ${BUILD_BASE}/release/fake-debug.iso.txt
-    echo "This a fake debug odl build history" > ${BUILD_BASE}/opendaylight/.odl-build-history
-    echo "This a fake debug odl build log" > ${BUILD_BASE}/opendaylight/.odl-build.log
-}
-
-cache_check () {
-    if [ ! -f ${BUILD_BASE}/fuel-6.0.1.iso ] || \
-       [ ! -f ${BUILD_BASE}/.versions ] || \
-       [ ! -f ${BUILD_BASE}/opendaylight/.odl-build-history ]; \
-       [ ! -f ${BUILD_BASE}/opendaylight/.odl-build.log ]; then
-	echo "Cache not present in the build system"
-	echo "TEST FAILED"
-	exit $rc
-    fi
-}
-
-integration-test () {
-
-##### Always succeed integration test ####
-    make -C ${BUILD_BASE} clean
-    echo "TEST - $0 - ALWAYS SUCCEED"
-    set +e
-    $0 -f s tmp/output
-    rc=$?
-    set -e
-    if [ $rc -ne 0 ]; then
-	echo "TEST FAILED"
-	rc=150
-	exit $rc
-    fi
-
-##### Always fail integration test ####
-    make -C ${BUILD_BASE} clean
-    echo "TEST - $0 - ALWAYS FAIL"
-    set +e
-    $0 -f f tmp/output
-    rc=$?
-    set -e
-    if [ $rc -eq 0 ]; then
-	echo "TEST FAILED"
-	rc=151
-	exit $rc
-    fi
-
-##### Populate cache integration test ####
-    make -C ${BUILD_BASE} clean
-    echo "TEST - $0 - POPULATE CACHE"
-    rm -rf tmp
-    mkdir tmp
-    mkdir tmp/cache
-    set +e
-    $0 -c ${BUILD_CACHE_URI} -f PD tmp/output
-    rc=$?
-    set -e
-    if [ $rc -ne 0 ]; then
-	echo "TEST FAILED"
-	rc=152
-	exit $rc
-    fi
-
-    if [ ! -f  tmp/output/fake-debug.iso ] || [ ! -f  tmp/output/fake-debug.iso ]; then
-	echo "TEST FAILED"
-	rc=153
-	exit $rc
-    fi
-    rc=154
-    cache_check
-
-##### Build uinge cache integration test ####
-    make -C ${BUILD_BASE} clean
-    echo "TEST - $0 - BUILD USING CACHE"
-    set +e
-    $0 -c ${BUILD_CACHE_URI} -f D tmp/output
-    rc=$?
-    set -e
-    if [ $rc -ne 0 ]; then
-	echo "TEST FAILED"
-	rc=155
-	exit $rc
-    fi
-
-    if [ ! -f  tmp/output/fake-debug.iso ] || [ ! -f  tmp/output/fake-debug.iso ]; then
-	echo "TEST FAILED"
-	rc=156
-	exit $rc
-    fi
-    rc=157
-    cache_check
-
-#### Repopulate cache because cache non existant ####
-    make -C ${BUILD_BASE} clean
-    echo "TEST - $0 - BUILD USING CACHE"
-    rm -rf tmp/cache/*
-    set +e
-    $0 -c ${BUILD_CACHE_URI} -f D tmp/output
-    rc=$?
-    set -e
-    if [ $rc -ne 0 ]; then
-	echo "TEST FAILED"
-	rc=158
-	exit $rc
-    fi
-
-    if [ ! -f  tmp/output/fake-debug.iso ] || [ ! -f  tmp/output/fake-debug.iso ]; then
-	echo "TEST FAILED"
-	rc=160
-	exit $rc
-    fi
-    rc=161
-    cache_check
-
-# Repopulate cache because cach is ivalidated
-#TBD
-rm -rf tmp
-echo "All tests passed!"
-rc=0
-exit $rc
-}
-
-#DEFAULT VALUES
+############################################################################
+# BEGIN of variables to customize
+#
 BUILD_BASE=$(readlink -e ../build/)
 RESULT_DIR="${BUILD_BASE}/release"
 BUILD_SPEC="${BUILD_BASE}/config.mk"
@@ -207,8 +89,14 @@ CACHE_DIR="cache"
 LOCAL_CACHE_ARCH_NAME="fuel-cache"
 REMOTE_CACHE_ARCH_NAME="fuel_cache-$(md5sum ${BUILD_SPEC}| cut -f1 -d " ")"
 REMOTE_ACCESS_METHD=curl
-
-#SCRIPT ASSIGNED VARIABLES
+INCLUDE_DIR=../include
+#
+# END of variables to customize
+############################################################################
+
+############################################################################
+# BEGIN of script assigned variables
+#
 SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 LOCK_FILE="${SCRIPT_DIR}/.build.lck"
 TEST_SUCCEED=0
@@ -220,6 +108,7 @@ RECURSIV=0
 DETACH=0
 DEBUG=0
 INTEGRATION_TEST=0
+FULL_INTEGRATION_TEST=0
 INTERACTIVE=0
 BUILD_CACHE_URI=
 BUILD_SPEC=
@@ -227,8 +116,22 @@ BUILD_DIR=
 BUILD_LOG=
 BUILD_VERSION=
 MAKE_ARGS=
-
-while getopts "s:c:v:f:l:r:RTh" OPTION
+#
+# END of script assigned variables
+############################################################################
+
+############################################################################
+# BEGIN of include pragmas
+#
+source ${INCLUDE_DIR}/build.sh.debug
+#
+# END of include
+############################################################################
+
+############################################################################
+# BEGIN of main
+#
+while getopts "s:c:v:f:l:r:RtTh" OPTION
 do
     case $OPTION in
 	h)
@@ -264,8 +167,13 @@ do
 	    RECURSIVE=1
 	    ;;
 
+	t)
+	    INTEGRATION_TEST=1
+	    ;;
+
 	T)
 	    INTEGRATION_TEST=1
+	    FULL_INTEGRATION_TEST=1
 	    ;;
 
 	*)
@@ -316,7 +224,7 @@ for ((i=0; i<${#BUILD_FLAGS};i++)); do
 	    ;;
 
 	*)
-	    echo "${BUILD_FLAGS:$i:1} is not a valid build flag"
+	    echo "${BUILD_FLAGS:$i:1} is not a valid build flag - exiting ...."
 	    rc=100
 	    exit $rc
 	    ;;
@@ -330,13 +238,13 @@ if [ ${INTEGRATION_TEST} -eq 1 ]; then
 fi
 
 if [ ! -f ${BUILD_SPEC} ]; then
-    echo "spec file does not exist: $BUILD_SPEC"
+    echo "spec file does not exist: $BUILD_SPEC - exiting ...."
     rc=100
     exit $rc
 fi
 
 if [ -z ${BUILD_DIR} ]; then
-    echo "Missing build directory"
+    echo "Missing build directory - exiting ...."
     rc=100
     exit $rc
 fi
@@ -379,7 +287,7 @@ if [ ! -z ${BUILD_CACHE_URI} ]; then
 	mkdir /tmp/cache
 	echo "Downloading cach file ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME} ..."
 	set +e
-	${REMOTE_ACCESS_METHD} -o /tmp/cache/${LOCAL_CACHE_ARCH_NAME}.tgz ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME}
+	${REMOTE_ACCESS_METHD} -o /tmp/cache/${LOCAL_CACHE_ARCH_NAME}.tgz ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME}.tgz
 	rc=$?
 	set -e
 	if [ $rc -ne 0 ]; then
@@ -389,10 +297,11 @@ if [ ! -z ${BUILD_CACHE_URI} ]; then
 	    echo "Unpacking cache file ..."
 	    tar -C /tmp/cache -xvf /tmp/cache/${LOCAL_CACHE_ARCH_NAME}.tgz
 	    cp /tmp/cache/cache/.versions ${BUILD_BASE}/.
-	    echo "Validating cache content ..."
 	    set +e
        	    make -C ${BUILD_BASE} validate-cache;
 	    rc=$?
+	    set -e
+
 	    if [ $rc -ne 0 ]; then
 		echo "Cache invalid - a new cache will be built "
 		POPULATE_CACHE=1
@@ -455,13 +364,13 @@ rc=$?
 set -e
 
 if [ $rc -gt 0 ]; then
-    echo "Build: make prepare-cache failed, exiting ..."
+    echo "Build: make prepare-cache failed - exiting ..."
     rc=100
     exit $rc
 fi
 echo "Copying built OPNFV .iso file to target directory ${BUILD_DIR} ..."
 rm -rf ${BUILD_DIR}
-mkdir ${BUILD_DIR}
+mkdir -p ${BUILD_DIR}
 cp ${BUILD_BASE}/.versions ${BUILD_DIR}
 cp ${RESULT_DIR}/*.iso* ${BUILD_DIR}
 
@@ -470,9 +379,12 @@ if [ $POPULATE_CACHE -eq 1 ]; then
 	echo "Building cache ..."
 	tar --dereference -C ${BUILD_BASE} -caf ${BUILD_BASE}/${LOCAL_CACHE_ARCH_NAME}.tgz ${CACHE_DIR}
 	echo "Uploading cache ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME}"
-	${REMOTE_ACCESS_METHD} -T ${BUILD_BASE}/${LOCAL_CACHE_ARCH_NAME}.tgz ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME}
+	${REMOTE_ACCESS_METHD} -T ${BUILD_BASE}/${LOCAL_CACHE_ARCH_NAME}.tgz ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME}.tgz
 	rm ${BUILD_BASE}/${LOCAL_CACHE_ARCH_NAME}.tgz
     fi
 fi
 echo "Success!!!"
 exit 0
+#
+# END of main
+############################################################################
diff --git a/fuel/include/build.sh.debug b/fuel/include/build.sh.debug
new file mode 100644
index 0000000..b6594cb
--- /dev/null
+++ b/fuel/include/build.sh.debug
@@ -0,0 +1,355 @@
+##############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+# stefan.k.berg@ericsson.com
+# jonas.bjurel@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+############################################################################
+# BEGIN of build script integration test
+#
+integration-test () {
+
+test_clean () {
+    make -C ${BUILD_BASE} clean
+    test_cache_clean
+    test_output_clean
+}
+
+test_cache_clean () {
+    rm -rf ${TEST_BUILD_CACHE_URI_DIR}
+}
+
+test_output_clean () {
+    rm -rf ${TEST_BUILD_RESULT_DIR}
+}
+
+test_prepare () {
+    test_clean
+    test_cache_clean
+    test_output_clean
+    mkdir -p ${TEST_BUILD_CACHE_URI_DIR}
+}
+
+test_cache_check () {
+    if [ ! -f ${BUILD_BASE}/fuel-6.0.1.iso ] || \
+       [ ! -f ${BUILD_BASE}/.versions ] || \
+       [ ! -f ${BUILD_BASE}/opendaylight/.odl-build-history ]; \
+       [ ! -f ${BUILD_BASE}/opendaylight/.odl-build.log ]; then
+	echo "Cache not present in the build system"
+	echo "TEST FAILED"
+	exit $rc
+    fi
+
+    if [ ! -f ${TEST_BUILD_CACHE_URI_DIR}/${REMOTE_CACHE_ARCH_NAME}.tgz ]; then
+	echo "Cache archive not present"
+	echo "TEST FAILED"
+	exit $rc
+    fi
+
+    if tar -tf ${TEST_BUILD_CACHE_URI_DIR}/${REMOTE_CACHE_ARCH_NAME}.tgz ${CACHE_DIR}/.mark > /dev/null 2>&1; then
+	echo "Cache archive is invalidated"
+	echo "TEST FAILED"
+	exit $rc
+    fi
+}
+
+test_output_check () {
+    if [ ! -f  ${TEST_BUILD_RESULT_DIR}/*.iso ] || \
+       [ ! -f  ${TEST_BUILD_RESULT_DIR}/*.txt ] || \
+       [ ! -f  ${TEST_BUILD_RESULT_DIR}/.versions ]; then
+	echo "Build output not produced by the build system"
+	echo "TEST FAILED"
+	exit $rc
+    fi
+}
+
+test_invalidate_cache () {
+    tar -C ${TEST_BUILD_CACHE_URI_DIR} -xf ${TEST_BUILD_CACHE_URI_DIR}/${REMOTE_CACHE_ARCH_NAME}.tgz
+    echo "cache.mk 123" > ${TEST_BUILD_CACHE_URI_DIR}/cache/.versions
+    echo "config.mk 123" >> ${TEST_BUILD_CACHE_URI_DIR}/cache/.versions
+    echo "fuel 123" >> ${TEST_BUILD_CACHE_URI_DIR}/cache/.versions
+    echo "odl 123" >> ${TEST_BUILD_CACHE_URI_DIR}/cache/.versions
+
+    echo "This file marks the invalid cache" > ${TEST_BUILD_CACHE_URI_DIR}/cache/.mark
+    rm -f ${TEST_BUILD_CACHE_URI_DIR}/${REMOTE_CACHE_ARCH_NAME}.tgz
+    tar --dereference -C ${TEST_BUILD_CACHE_URI_DIR} -caf ${TEST_BUILD_CACHE_URI_DIR}/${REMOTE_CACHE_ARCH_NAME}.tgz ${CACHE_DIR}
+}
+
+##### Always succeed integration test ####
+build_tc1 () {
+    echo "TEST - $0 - ALWAYS SUCCEED"
+    set +e
+    $0 -f s tmp/output
+    rc=$?
+    set -e
+    if [ $rc -ne 0 ]; then
+	echo "TEST FAILED"
+	rc=151
+	exit $rc
+    fi
+}
+
+##### Always fail integration test (TC2) ####
+build_tc2 () {
+    echo "TEST - $0 - ALWAYS FAIL"
+    set +e
+    $0 -f f tmp/output
+    rc=$?
+    set -e
+    if [ $rc -eq 0 ]; then
+	echo "TEST FAILED"
+	rc=152
+	exit $rc
+    fi
+}
+
+##### Fake build and populate dummy cache - integration test (TC3) ####
+build_tc3 () {
+    echo "TEST - $0 - FAKE BUILD AND POPULATE DUMMY CACHE"
+    rm -rf tmp
+    mkdir -p ${TEST_BUILD_CACHE_URI_DIR}
+    set +e
+    $0 -c ${TEST_BUILD_CACHE_URI} -f PD ${TEST_BUILD_RESULT_DIR}
+    rc=$?
+    set -e
+    if [ $rc -ne 0 ]; then
+	echo "Build failed"
+	echo "TEST FAILED"
+	rc=152
+	exit $rc
+    fi
+
+    rc=153
+    test_output_check
+
+    rc=154
+    test_cache_check
+}
+
+##### Fake build usng dummy cache integration test (TC4) ####
+build_tc4 () {
+    echo "TEST - $0 - FAKE BUILD USING DUMMY CACHE"
+    set +e
+    $0 -c ${TEST_BUILD_CACHE_URI} -f D ${TEST_BUILD_RESULT_DIR}
+    rc=$?
+    set -e
+    if [ $rc -ne 0 ]; then
+	echo "Build failed"
+	echo "TEST FAILED"
+	rc=155
+	exit $rc
+    fi
+
+    rc=156
+    test_output_check
+
+    rc=157
+    test_cache_check
+}
+
+#### Fake build and repopulate cache because cache non existant (TC5) ####
+build_tc5 () {
+    echo "TEST - $0 - FAKE BUILD AND REPOPULATE CACHE BECAUSE NON EXISTANT"
+    rm -rf ${TEST_BUILD_CACHE_URI_DIR}/*
+    set +e
+    $0 -c ${TEST_BUILD_CACHE_URI} -f D ${TEST_BUILD_RESULT_DIR}
+    rc=$?
+    set -e
+    if [ $rc -ne 0 ]; then
+	echo "Build failed"
+	echo "TEST FAILED"
+	rc=158
+	exit $rc
+    fi
+
+    rc=159
+    test_output_check
+
+    rc=160
+    test_cache_check
+}
+
+#### Fake build and repopulate cache because cache is ivalidated (TC6) ####
+build_tc6 () {
+    echo "TEST - $0 - FAKE BUILD AND REPOPULATE DUE TO CACHE INVALIDATION"
+    test_invalidate_cache
+    set +e
+    $0 -c ${TEST_BUILD_CACHE_URI} -f D ${TEST_BUILD_RESULT_DIR}
+    rc=$?
+    set -e
+    if [ $rc -ne 0 ]; then
+	echo "Build failed"
+	echo "TEST FAILED"
+	rc=161
+	exit $rc
+    fi
+
+    rc=162
+    test_output_check
+
+    rc=163
+    test_cache_check
+}
+
+#### Real build and populate cache (TC7) ####
+build_tc7 () {
+    echo "TEST - $0 - FULL BUILD AND POPULATE CACHE"
+    test_prepare
+    BUILD_START=$(date +%s)
+    set +e
+    $0 -c ${TEST_BUILD_CACHE_URI} -f Pi ${TEST_BUILD_RESULT_DIR}
+    rc=$?
+    set -e
+    if [ $rc -ne 0 ]; then
+	echo "Build failed"
+	echo "TEST FAILED"
+	rc=164
+	exit $rc
+    fi
+
+    BUILD_TIME=$(($(date +%s)-${BUILD_START}))
+    rc=165
+    test_output_check
+    rc=166
+    test_cache_check
+}
+
+#### Real build using cache from TC7 (TC8) ####
+build_tc8 () {
+    echo "TEST - $0 - INCREMENTAL BUILD AND USING CACHE"
+    BUILD_START=$(date +%s)
+    set +e
+    $0 -c ${TEST_BUILD_CACHE_URI} -f i ${TEST_BUILD_RESULT_DIR}
+    rc=$?
+    set -e
+    if [ $rc -ne 0 ]; then
+	echo "Build failed"
+	echo "TEST FAILED"
+	rc=167
+	exit $rc
+    fi
+    BUILD_TIME=$(($(date +%s)-${BUILD_START}))
+    test_output_check
+    rc=168
+    test_cache_check
+    rc=169
+}
+
+TEST_BUILD_ARTIFACT_DIR=${BUILD_BASE}/test
+TEST_BUILD_CACHE_URI_DIR=${TEST_BUILD_ARTIFACT_DIR}
+TEST_BUILD_CACHE_URI=file://${TEST_BUILD_CACHE_URI_DIR}
+TEST_BUILD_RESULT_DIR=${BUILD_BASE}/tmp/output
+
+TEST_START=$(date +%s)
+# Run test case 1: Always succeed integration test
+echo ==========================================================================
+echo ========================= RUNNING TEST CASE 1 ============================
+echo ==========================================================================
+test_prepare
+build_tc1
+
+# Run test case 2: Always fail integration test
+echo ==========================================================================
+echo ========================= RUNNING TEST CASE 2 ============================
+echo ==========================================================================
+test_prepare
+build_tc2
+
+# Run test case 3: Fake build and populate a build cach
+echo ==========================================================================
+echo ========================= RUNNING TEST CASE 3 ============================
+echo ==========================================================================
+test_prepare
+build_tc3
+
+# Run test case 4: Fake build using the build cache produced in TC 3
+echo ==========================================================================
+echo ========================= RUNNING TEST CASE 4 ============================
+echo ==========================================================================
+build_tc4
+
+# Run test case 5: Fake build and repopulate the build cache as there is no existing one
+echo ==========================================================================
+echo ========================= RUNNING TEST CASE 5 ============================
+echo ==========================================================================
+test_prepare
+build_tc5
+
+# Run test case 6: Fake build and repopulate the build cache as there the current one is invalidated/outdated
+echo ==========================================================================
+echo ========================= RUNNING TEST CASE 6 ============================
+echo ==========================================================================
+test_invalidate_cache
+build_tc4
+
+# Run test case 7: Build from upstream repos and populate the build cache
+if [ ${FULL_INTEGRATION_TEST} -eq 1 ]; then
+echo ==========================================================================
+echo ========================= RUNNING TEST CASE 7 ============================
+echo ==========================================================================
+    test_prepare
+    build_tc7
+    POPULATE_CACHE_TIME=${BUILD_TIME}
+fi
+
+# Run test case 8: Build from cache produced in TC7
+if [ ${FULL_INTEGRATION_TEST} -eq 1 ]; then
+echo ==========================================================================
+echo ========================= RUNNING TEST CASE 8 ============================
+echo ==========================================================================
+    build_tc8
+    PARTIAL_BUILD_TIME=${BUILD_TIME}
+fi
+
+# Run test case 9: Build and repopulate the cache as the current one is invalidated/not up to date
+if [ ${FULL_INTEGRATION_TEST} -eq 1 ]; then
+echo ==========================================================================
+echo ========================= RUNNING TEST CASE 9 ============================
+echo ==========================================================================
+    echo "TEST - $0 - REPOPULATING CACHE DUE TO CACHE INVALIDATION"
+    test_invalidate_cache
+    build_tc8
+fi
+
+TEST_TIME=$(($(date +%s)-${TEST_START}))
+
+rm -rf tmp
+echo "All tests passed!, test duration was ${TEST_TIME} seconds"
+if [ ${FULL_INTEGRATION_TEST} -eq 1 ]; then
+    echo "Full build took ${POPULATE_CACHE_TIME} second"
+    echo "Incremental build took ${PARTIAL_BUILD_TIME} seconds"
+fi
+rc=0
+exit $rc
+}
+#
+# END of build script integration test
+############################################################################
+
+############################################################################
+# BEGIN of debug_make - a fake make initiated with -D for debugging
+#
+debug_make () {
+    make -C ${BUILD_BASE} clean
+    echo "This is a fake debug fuel .iso image" > ${BUILD_BASE}/fuel-6.0.1.iso
+
+    echo "This is a fake debug versions file" > ${BUILD_BASE}/.versions
+    echo "cache.mk" $(md5sum ${BUILD_BASE}/cache.mk | cut -f1 -d " ") >> ${BUILD_BASE}/.versions
+    echo "config.mk" $(md5sum ${BUILD_BASE}/config.mk | cut -f1 -d " ") >> ${BUILD_BASE}/.versions
+    echo "fuel" $(git ls-remote $(make -C ${BUILD_BASE} get-fuel-repo | grep https) | awk '{print $(NF-1)}') >> ${BUILD_BASE}/.versions
+    echo "odl" $(git ls-remote $(make -C ${BUILD_BASE} get-odl-repo | grep https) | awk '{print $(NF-1)}') >> ${BUILD_BASE}/.versions
+
+    rm -rf ${BUILD_BASE}/release
+    mkdir ${BUILD_BASE}/release
+    echo "This is a fake debug OPNFV .iso image"  > ${BUILD_BASE}/release/fake-debug.iso
+    echo "This is a fake debug OPNFV .iso.txt message"  > ${BUILD_BASE}/release/fake-debug.iso.txt
+    echo "This a fake debug odl build history" > ${BUILD_BASE}/opendaylight/.odl-build-history
+    echo "This a fake debug odl build log" > ${BUILD_BASE}/opendaylight/.odl-build.log
+}
+#
+# END of debug_make
+############################################################################