summaryrefslogtreecommitdiffstats
path: root/fuel
diff options
context:
space:
mode:
authorJonas Bjurel <jonas.bjurel@ericsson.com>2015-03-22 23:04:43 +0100
committerJonas Bjurel <jonas.bjurel@ericsson.com>2015-03-26 17:26:27 +0100
commit2de197be04b871b437e35b55ca7f48d921595303 (patch)
treeb2f812d507ac3037153c11a5f19ea6db5c27009e /fuel
parenta5a094a199226f312e795c3019c5793094e5378e (diff)
Fuel build cache handling
- Cache invalidation supported in case cache is outdated compared to upstream. - Fully automated build.sh integration tests added (build.sh -t or build.sh -T) - Rebased to Daniel Smiths's patch-set: genesis refs/changes/71/171/4 - Yet untested, will perform automated integration tests over night! - Updated based on Stefan's comments (no more revs.) - Fixed a bug in f_odl_docker/Makefile - validate-cache target added in f_odl_docker/Makefile JIRA: BGS-1 Change-Id: I9f8f30c2a85fd7bf04dab1b566065e5739d4fcb1 Signed-off-by: Jonas Bjurel <jonas.bjurel@ericsson.com>
Diffstat (limited to 'fuel')
-rw-r--r--fuel/build/Makefile46
-rw-r--r--fuel/build/cache.mk79
-rw-r--r--fuel/build/config.mk9
-rw-r--r--fuel/build/f_example_control_bond/Makefile4
-rw-r--r--fuel/build/f_example_packadd/Makefile4
-rw-r--r--fuel/build/f_isoroot/Makefile4
-rw-r--r--fuel/build/f_l23network/Makefile4
-rw-r--r--fuel/build/f_ntp/Makefile4
-rwxr-xr-xfuel/build/f_odl_docker/Makefile10
-rw-r--r--fuel/build/f_opnfv_puppet/Makefile4
-rw-r--r--fuel/build/f_osnaily/Makefile4
-rw-r--r--fuel/build/f_resolvconf/Makefile4
l---------fuel/build/opendaylight/ 1
-rw-r--r--fuel/build/opendaylight/Makefile27
-rw-r--r--fuel/build/opendaylight/f_odl/Makefile4
-rwxr-xr-xfuel/ci/build.sh194
-rw-r--r--fuel/include/build.sh.debug355
17 files changed, 551 insertions, 206 deletions
diff --git a/fuel/build/Makefile b/fuel/build/Makefile
index b6b0eac..364fe31 100644
--- a/fuel/build/Makefile
+++ b/fuel/build/Makefile
@@ -8,11 +8,10 @@
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
+SHELL = /bin/bash
############################################################################
# BEGIN of variables to customize
#
-SHELL = /bin/bash
-
#Input args
export UNIT_TEST = FALSE
export INTERACTIVE = TRUE
@@ -31,20 +30,10 @@ export DEB_DEST := $(BUILD_BASE)/release/packages/ubuntu/pool/main
export PUPPET_DEST := $(BUILD_BASE)/release/puppet/modules
export CACHE_DIR := $(BUILD_BASE)/cache
export VERSION_FILE := $(BUILD_BASE)/.versions
-DOCKERIMG = opnfv.org/ubuntu-builder:14.04
+export DOCKERIMG = opnfv.org/ubuntu-builder:14.04
export TOPDIR := $(shell pwd)
-
-#
-# END of variables to customize
-#############################################################################
-
-#Include definitions
-include config.mk
-include cache.mk
-
-
-ORIGDIR := $(TOPDIR)/origiso
+#Build subclasses
SUBDIRS := f_isoroot
SUBDIRS += f_opnfv_puppet
@@ -53,32 +42,43 @@ SUBDIRS += f_l23network
SUBDIRS += f_resolvconf
SUBDIRS += f_ntp
SUBDIRS += f_odl_docker
-
-
+#SUBDIRS += f_odl
# f_example is only an example of how to generate a .deb package and
# should not be enabled in official builds.
#SUBDIRS += f_example
+ORIGDIR := $(TOPDIR)/origiso
+#
+# END of variables to customize
+#############################################################################
+
SUBCLEAN = $(addsuffix .clean,$(SUBDIRS))
+
.PHONY: all
all:
@docker version >/dev/null 2>&1 || (echo 'No Docker installation available'; exit 1)
- @echo "Versions of cached build results built by" $(shell hostname) "at" $(shell date -u) > $(BUILD_BASE)/.versions
- @echo "cache.mk" $(shell md5sum $(BUILD_BASE)/cache.mk | cut -f1 -d " ") >> $(BUILD_BASE)/.versions
- @echo "config.mk" $(shell md5sum $(BUILD_BASE)/config.mk | cut -f1 -d " ") >> $(BUILD_BASE)/.versions
+ @echo "Versions of cached build results built by" $(shell hostname) "at" $(shell date -u) > $(VERSION_FILE)
+ @echo "cache.mk" $(shell md5sum $(BUILD_BASE)/cache.mk | cut -f1 -d " ") >> $(VERSION_FILE)
+ @echo "config.mk" $(shell md5sum $(BUILD_BASE)/config.mk | cut -f1 -d " ") >> $(VERSION_FILE)
@make -C docker
@docker/runcontext $(DOCKERIMG) $(MAKE) $(MAKEFLAGS) iso
-
+############################################################################
+# BEGIN of Include definitions
+#
+include config.mk
+include cache.mk
+#
+# END Include definitions
+#############################################################################
$(ISOCACHE):
# Clone Fuel to non-persistent location and build
cd /tmp && git clone $(FUEL_MAIN_REPO)
cd /tmp/fuel-main && git checkout $(FUEL_MAIN_TAG)
- @echo "fuel" `git -C /tmp/fuel-main show | grep commit | rev | cut -f1 -d " "` >> ${BUILD_BASE}/.versions
-
+ @echo "fuel" `git -C /tmp/fuel-main show | grep commit | head -1 | cut -d " " -f2` >> $(VERSION_FILE)
# Setup cgroups for docker-in-docker
sudo /root/enable_dockerx2
# Patch to fix race condition when doing "Docker-in-Docker" build
@@ -130,7 +130,7 @@ build-clean: $(SUBCLEAN)
@rm -f $(NEWISO)
.PHONY: clean $(SUBCLEAN)
-clean: prepare $(SUBCLEAN)
+clean: clean-cache prepare $(SUBCLEAN)
$(MAKE) -C patch-packages -f Makefile clean
#$(MAKE) -C opendaylight -f Makefile clean
@rm -f *.iso
diff --git a/fuel/build/cache.mk b/fuel/build/cache.mk
index a3f3516..a2aa1d5 100644
--- a/fuel/build/cache.mk
+++ b/fuel/build/cache.mk
@@ -8,23 +8,22 @@
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
+SHELL = /bin/bash
+CACHEVALIDATE := $(addsuffix .validate,$(SUBDIRS))
+CACHECLEAN := $(addsuffix .clean,$(CACHEFILES) $(CACHEDIRS))
+
############################################################################
# BEGIN of variables to customize
#
-SHELL = /bin/bash
-
-#export BUILD_BASE = $(shell pwd)
-#export CACHE_DIR = "$(BUILD_BASE)/cache"
-
CACHEDIRS := opendaylight/f_odl/package
CACHEFILES := opendaylight/.odl-build-history
CACHEFILES += opendaylight/.odl-build.log
CACHEFILES += .versions
-CACHEFILES += fuel-6.0.1.iso
-#CACHEFILES += $(ISOSRC)
-
-CACHECLEAN = $(addsuffix .clean,$(CACHEDIRS))
+CACHEFILES += $(shell basename $(ISOSRC))
+#
+# END of variables to customize
+############################################################################
.PHONY: prepare-cache
prepare-cache: make-cache-dir $(CACHEDIRS) $(CACHEFILES)
@@ -34,54 +33,66 @@ make-cache-dir:
@rm -rf ${CACHE_DIR}
@mkdir ${CACHE_DIR}
-
.PHONY: clean-cache
clean-cache: $(CACHECLEAN)
@rm -rf ${CACHE_DIR}
.PHONY: $(CACHEDIRS)
$(CACHEDIRS):
- @mkdir -p $(dir ${CACHE_DIR}/$@)
- @if [ ! -d ${BUILD_BASE}/$@ ]; then\
- mkdir -p $(dir ${BUILD_BASE}/$@);\
- ln -s ${BUILD_BASE}/$@ ${CACHE_DIR}/$@;\
- rm -rf ${BUILD_BASE}/$@;\
- else\
- ln -s ${BUILD_BASE}/$@ ${CACHE_DIR}/$@;\
+ @mkdir -p $(dir $(CACHE_DIR)/$@)
+ @if [ ! -d $(BUILD_BASE)/$@ ]; then\
+ mkdir -p $(BUILD_BASE)/$@;\
fi
+ @ln -s $(BUILD_BASE)/$@ $(CACHE_DIR)/$@
.PHONY: $(CACHEFILES)
$(CACHEFILES):
- @mkdir -p $(dir ${CACHE_DIR}/$@)
- @if [ ! -f ${BUILD_BASE}/$@ ]; then\
- mkdir $(dir ${BUILD_BASE}/$@);\
- echo " " > ${BUILD_BASE}/$@;\
- ln -s ${BUILD_BASE}/$@ ${CACHE_DIR}/$@;\
- rm -f ${BUILD_BASE}/$@;\
+ @mkdir -p $(dir $(CACHE_DIR)/$@)
+ @if [ ! -d $(dir $(BUILD_BASE)/$@) ]; then\
+ mkdir -p $(dir $(BUILD_BASE)/$@);\
+ fi
+
+ @if [ ! -f $(BUILD_BASE)/$@ ]; then\
+ echo " " > $(BUILD_BASE)/$@;\
+ ln -s $(BUILD_BASE)/$@ $(CACHE_DIR)/$@;\
+ rm -f $(BUILD_BASE)/$@;\
else\
- ln -s ${BUILD_BASE}/$@ ${CACHE_DIR}/$@;\
+ ln -s $(BUILD_BASE)/$@ $(CACHE_DIR)/$@;\
fi
.PHONY: validate-cache
-validate-cache:
-# if [ $(shell md5sum ${BUILD_BASE}/config.mk) -ne $(shell cat ${CACHE_DIR}/.versions | grep config.mk awk '{print $NF}') ]; then\
+validate-cache: prepare $(CACHEVALIDATE)
+ @if [[ $(shell md5sum $(BUILD_BASE)/config.mk | cut -f1 -d " ") != $(shell cat $(VERSION_FILE) | grep config.mk | awk '{print $$NF}') ]]; then\
echo "Cache does not match current config.mk definition, cache must be rebuilt";\
exit 1;\
fi;
-# if [ $(shell md5sum ${BUILD_BASE}/cache.mk) -ne $(shell cat ${CACHE_DIR}/.versions | grep config.mk awk '{print $NF}') ]; then\
+ @if [[ $(shell md5sum $(BUILD_BASE)/cache.mk | cut -f1 -d " ") != $(shell cat $(VERSION_FILE) | grep cache.mk | awk '{print $$NF}') ]]; then\
echo "Cache does not match current cache.mk definition, cache must be rebuilt";\
exit 1;\
fi;
-# $(MAKE) -C opendaylight validate-cache
-# if [ $? -ne 0 ]; then\
- echo "Cache does not match current OpenDaylight version, cach must be rebuilt";\
- exit 1;\
- fi;
+# Once the Make structure is refactored, this should go in as a validate-cache
+# taget in the fuel Makefile
+
+ @REMOTE_ID=$(shell git ls-remote $(FUEL_MAIN_REPO) $(FUEL_MAIN_TAG)^{} | awk '{print $$(NF-1)}'); \
+ if [ -z $$REMOTE_ID ] || [ $$REMOTE_ID = " " ]; \
+ then \
+ REMOTE_ID=$(shell git ls-remote $(FUEL_MAIN_REPO) $(FUEL_MAIN_TAG) | awk '{print $$(NF-1)}'); \
+ fi; \
+ if [ $$REMOTE_ID != $(shell cat $(VERSION_FILE) | grep fuel | awk '{print $$NF}') ]; \
+ then \
+ echo "Cache does not match upstream Fuel, cache must be rebuilt!"; \
+ exit 1; \
+ fi
+
+ #$(MAKE) -C opendaylight validate-cache
-# $(SUBDIRS)
+.PHONY: $(CACHEVALIDATE)
+$(CACHEVALIDATE): %.validate:
+ @echo VALIDATE $(CACHEVALIDATE)
+ $(MAKE) -C $* -f Makefile validate-cache
.PHONY: $(CACHECLEAN)
$(CACHECLEAN): %.clean:
- rm -f ${CACHE_DIR}/$*
+ rm -rf ${CACHE_DIR}/$*
diff --git a/fuel/build/config.mk b/fuel/build/config.mk
index 49b0173..19f502d 100644
--- a/fuel/build/config.mk
+++ b/fuel/build/config.mk
@@ -16,3 +16,12 @@ FUEL_MAIN_TAG = stable/6.0
DOCKER_REPO := http://get.docker.com/builds/Linux/x86_64
DOCKER_TAG := docker-latest
+
+.PHONY: get-odl-repo
+get-odl-repo:
+ @echo $(ODL_MAIN_REPO) $(ODL_MAIN_TAG)
+
+.PHONY: get-fuel-repo
+get-fuel-repo:
+ @echo $(FUEL_MAIN_REPO) $(FUEL_MAIN_TAG)
+
diff --git a/fuel/build/f_example_control_bond/Makefile b/fuel/build/f_example_control_bond/Makefile
index c311a2d..0949737 100644
--- a/fuel/build/f_example_control_bond/Makefile
+++ b/fuel/build/f_example_control_bond/Makefile
@@ -18,6 +18,10 @@ clean:
@rm -rf tmp
@rm -rf release
+.PHONY: validate-cache
+validate-cache:
+ @echo "No cache validation schema available for $(shell pwd)"
+ @echo "Continuing ..."
.PHONY: release
release:
diff --git a/fuel/build/f_example_packadd/Makefile b/fuel/build/f_example_packadd/Makefile
index eadc83f..a64293d 100644
--- a/fuel/build/f_example_packadd/Makefile
+++ b/fuel/build/f_example_packadd/Makefile
@@ -31,6 +31,10 @@ clean:
@rm -rf release
@rm -f ../release/packages/ubuntu/pool/main/$(DEB_NAME)
+.PHONY: validate-cache
+validate-cache:
+ @echo "No cache validation schema available for $(shell pwd)"
+ @echo "Continuing ..."
.PHONY: release
release:release/pool/main/$(DEB_NAME)
diff --git a/fuel/build/f_isoroot/Makefile b/fuel/build/f_isoroot/Makefile
index 2fdbbae..bde8e64 100644
--- a/fuel/build/f_isoroot/Makefile
+++ b/fuel/build/f_isoroot/Makefile
@@ -26,6 +26,10 @@ clean: $(SUBCLEAN)
$(SUBCLEAN): %.clean:
$(MAKE) -C $* -f Makefile clean
+validate-cache:
+ @echo "No cache validation schema available for $(shell pwd)"
+ @echo "Continuing ..."
+
.PHONY: release
release:clean $(SUBDIRS)
@cp -Rvp release/* ../release/isoroot
diff --git a/fuel/build/f_l23network/Makefile b/fuel/build/f_l23network/Makefile
index c311a2d..0949737 100644
--- a/fuel/build/f_l23network/Makefile
+++ b/fuel/build/f_l23network/Makefile
@@ -18,6 +18,10 @@ clean:
@rm -rf tmp
@rm -rf release
+.PHONY: validate-cache
+validate-cache:
+ @echo "No cache validation schema available for $(shell pwd)"
+ @echo "Continuing ..."
.PHONY: release
release:
diff --git a/fuel/build/f_ntp/Makefile b/fuel/build/f_ntp/Makefile
index c311a2d..0949737 100644
--- a/fuel/build/f_ntp/Makefile
+++ b/fuel/build/f_ntp/Makefile
@@ -18,6 +18,10 @@ clean:
@rm -rf tmp
@rm -rf release
+.PHONY: validate-cache
+validate-cache:
+ @echo "No cache validation schema available for $(shell pwd)"
+ @echo "Continuing ..."
.PHONY: release
release:
diff --git a/fuel/build/f_odl_docker/Makefile b/fuel/build/f_odl_docker/Makefile
index b227bee..6135e71 100755
--- a/fuel/build/f_odl_docker/Makefile
+++ b/fuel/build/f_odl_docker/Makefile
@@ -12,7 +12,7 @@ TOP := $(shell pwd)
BUILDTAG := loving_daniel
# Edit this to match the GENESIS / OPNFV in your environment
-export OPNFV_PUPPET := /home/lmcdasm/MAR24/genesis/common/puppet-opnfv
+export OPNFV_PUPPET := $(BUILD_BASE)/../../common/puppet-opnfv
include ../config.mk
.PHONY: all
@@ -25,7 +25,7 @@ all:
@docker save ${BUILDTAG} > puppet/modules/opnfv/odl_docker/odl_docker_image.tar
@wget ${DOCKER_REPO}/${DOCKER_TAG} -O puppet/modules/opnfv/odl_docker/docker-latest
@echo "OPFNV_PUPPET is: ${OPNFV_PUPPET}"
- @cp -Rvp ${OPNFV_PUPPET}/manifests/templates/dockerfile/compute_scripts puppet/modules/opnfv
+ @cp -Rvp ${OPNFV_PUPPET}/manifests/templates/dockerfile/container_scripts puppet/modules/opnfv
.PHONY: clean
clean:
@@ -39,9 +39,13 @@ build-clean:
@rm -rf puppet/modules/opnfv/odl_docker/odl_docker_image.tar
@rm -rf puppet/modules/opnfv/odl_docker/docker-latest
+.PHONY: validate-cache
+validate-cache:
+ @echo "No cache validation schema available for $(shell pwd)"
+ @echo "Continuing ..."
.PHONY: release
release:
# Fetch PP from OPNFV Common
- @cp -Rvp ${OPNFV_PUPPET}/common/puppet-opnfv/manifests/odl_docker.pp ${PUPPET_DEST}
+ @cp -Rvp ${OPNFV_PUPPET}/manifests/odl_docker.pp ${PUPPET_DEST}
@cp -Rvp puppet/modules/* $(PUPPET_DEST)
diff --git a/fuel/build/f_opnfv_puppet/Makefile b/fuel/build/f_opnfv_puppet/Makefile
index c311a2d..0949737 100644
--- a/fuel/build/f_opnfv_puppet/Makefile
+++ b/fuel/build/f_opnfv_puppet/Makefile
@@ -18,6 +18,10 @@ clean:
@rm -rf tmp
@rm -rf release
+.PHONY: validate-cache
+validate-cache:
+ @echo "No cache validation schema available for $(shell pwd)"
+ @echo "Continuing ..."
.PHONY: release
release:
diff --git a/fuel/build/f_osnaily/Makefile b/fuel/build/f_osnaily/Makefile
index c311a2d..0949737 100644
--- a/fuel/build/f_osnaily/Makefile
+++ b/fuel/build/f_osnaily/Makefile
@@ -18,6 +18,10 @@ clean:
@rm -rf tmp
@rm -rf release
+.PHONY: validate-cache
+validate-cache:
+ @echo "No cache validation schema available for $(shell pwd)"
+ @echo "Continuing ..."
.PHONY: release
release:
diff --git a/fuel/build/f_resolvconf/Makefile b/fuel/build/f_resolvconf/Makefile
index c311a2d..0949737 100644
--- a/fuel/build/f_resolvconf/Makefile
+++ b/fuel/build/f_resolvconf/Makefile
@@ -18,6 +18,10 @@ clean:
@rm -rf tmp
@rm -rf release
+.PHONY: validate-cache
+validate-cache:
+ @echo "No cache validation schema available for $(shell pwd)"
+ @echo "Continuing ..."
.PHONY: release
release:
diff --git a/fuel/build/opendaylight/ b/fuel/build/opendaylight/
new file mode 120000
index 0000000..0519ecb
--- /dev/null
+++ b/fuel/build/opendaylight/
@@ -0,0 +1 @@
+ \ No newline at end of file
diff --git a/fuel/build/opendaylight/Makefile b/fuel/build/opendaylight/Makefile
index c112ff6..bd2eeb5 100644
--- a/fuel/build/opendaylight/Makefile
+++ b/fuel/build/opendaylight/Makefile
@@ -12,7 +12,7 @@
# BEGIN of variables to customize
#
SHELL = /bin/bash
-include ../config.mk
+
BUILD_DIR := $(shell pwd)
GIT_DIR := /tmp
@@ -32,14 +32,31 @@ MAVEN_SPEC = $(BUILD_DIR)/odl_maven/settings.xml
.PHONY: all
all: odl
+############################################################################
+# BEGIN of Include definitions
+#
+include ../config.mk
+#
+# END Include definitions
+#############################################################################
+
.PHONY: setup
setup:
rm -f "$(BUILD_BASE)/f_odl"
ln -s "$(shell readlink -e $(BUILD_DIR))/f_odl" "$(shell readlink -e $(BUILD_BASE))/f_odl"
-.PHONY: check-cache-validity
-check-cache-validity:
- @if [ $(git ls-remote $(ODL_MAIN_REPO) $(ODL_MAIN_TAG) | awk '{print $(NF-1)}') -ne $(cat $(CACHE_DIR)/.versions | grep odl | awk '{print $NF}') ]; then @echo "Cache is not up to date, stopping!"; exit 1; fi
+.PHONY: validate-cache
+validate-cache:
+ @REMOTE_ID=$(shell git ls-remote $(ODL_MAIN_REPO) $(ODL_MAIN_TAG)^{} | awk '{print $$(NF-1)}'); \
+ if [ -z $$REMOTE_ID ] || [ $$REMOTE_ID = " " ]; \
+ then \
+ REMOTE_ID=$(shell git ls-remote $(ODL_MAIN_REPO) $(ODL_MAIN_TAG) | awk '{print $$(NF-1)}'); \
+ fi; \
+ if [ $$REMOTE_ID != $(shell cat $(VERSION_FILE) | grep odl | awk '{print $$NF}') ]; \
+ then \
+ echo "Cache does not match upstream OpenDaylight, cache must be rebuilt!"; \
+ exit 1; \
+ fi
.PHONY: odl
odl:
@@ -69,7 +86,7 @@ else
mvn -gs $(MAVEN_SPEC) clean install;\
fi;
- @echo "odl" `git -C /tmp/controller show | grep commit | rev | cut -f1 -d " "` >> $(BUILD_BASE)/.versions
+ @echo "odl" `git -C /tmp/controller show | grep commit | head -1 | cut -d " " -f2` >> $(VERSION_FILE)
@./make-odl-deb.sh -N $(ODL_SHORT_NAME)_`cd /tmp/controller; git rev-parse --short HEAD` -n $(ODL_SHORT_NAME) -v "$(ODL_VERSION)" -t "$(ODL_MAIN_TAG)" -m $(MAINTAINER) -d $(DEPEND) -p $(TARGET_BUILD_PATH)
@echo $(ODL_MAIN_TAG) > .odl-build.log
endif
diff --git a/fuel/build/opendaylight/f_odl/Makefile b/fuel/build/opendaylight/f_odl/Makefile
index 6983f0c..f7ebd3e 100644
--- a/fuel/build/opendaylight/f_odl/Makefile
+++ b/fuel/build/opendaylight/f_odl/Makefile
@@ -36,6 +36,10 @@ clean:
@rm -rf release
@rm -f $(DEB_DEST)/$(DEB_NAME).deb
+.PHONY: validate-cache
+validate-cache:
+ @echo "No cache validation schema available for $(shell pwd)"
+ @echo "Continuing ..."
.PHONY: release
release:release/pool/main/$(DEB_NAME).deb
diff --git a/fuel/ci/build.sh b/fuel/ci/build.sh
index bb0d124..cdaa6ef 100755
--- a/fuel/ci/build.sh
+++ b/fuel/ci/build.sh
@@ -17,6 +17,9 @@ if [ -f ${LOCK_FILE} ]; then \
fi; \
fi;' EXIT
+############################################################################
+# BEGIN of usage description
+#
usage ()
{
cat << EOF
@@ -31,6 +34,8 @@ OPTIONS:
-l log-file ($BUILD_LOG), specifies the output log-file (stdout and stderr), if not specified logs are output to console as normal
-v version tag to be applied to the build result
-r alternative remote access method script/program. curl is default.
+ -t run small build-script unit test.
+ -T run large build-script unit test.
-f build flags ($BUILD_FLAGS):
o s: Do nothing, succeed
o f: Do nothing, fail
@@ -70,136 +75,13 @@ build -c http://opnfv.org/artifactory/fuel/cache -d ~/jenkins/genesis/fuel/ci/ou
NOTE: At current the build scope is set to the git root of the repository, -d destination locations outside that scope will not work
EOF
}
+#
+# END of usage description
+############################################################################
-
-
-debug_make () {
- make -C ${BUILD_BASE} clean
- echo "This is a fake debug fuel .iso image" > ${BUILD_BASE}/fuel-6.0.1.iso
- echo "This is a fake debug versions file" > ${BUILD_BASE}/.versions
- rm -rf ${BUILD_BASE}/release
- mkdir ${BUILD_BASE}/release
- echo "This is a fake debug OPNFV .iso image" > ${BUILD_BASE}/release/fake-debug.iso
- echo "This is a fake debug OPNFV .iso.txt message" > ${BUILD_BASE}/release/fake-debug.iso.txt
- echo "This a fake debug odl build history" > ${BUILD_BASE}/opendaylight/.odl-build-history
- echo "This a fake debug odl build log" > ${BUILD_BASE}/opendaylight/.odl-build.log
-}
-
-cache_check () {
- if [ ! -f ${BUILD_BASE}/fuel-6.0.1.iso ] || \
- [ ! -f ${BUILD_BASE}/.versions ] || \
- [ ! -f ${BUILD_BASE}/opendaylight/.odl-build-history ]; \
- [ ! -f ${BUILD_BASE}/opendaylight/.odl-build.log ]; then
- echo "Cache not present in the build system"
- echo "TEST FAILED"
- exit $rc
- fi
-}
-
-integration-test () {
-
-##### Always succeed integration test ####
- make -C ${BUILD_BASE} clean
- echo "TEST - $0 - ALWAYS SUCCEED"
- set +e
- $0 -f s tmp/output
- rc=$?
- set -e
- if [ $rc -ne 0 ]; then
- echo "TEST FAILED"
- rc=150
- exit $rc
- fi
-
-##### Always fail integration test ####
- make -C ${BUILD_BASE} clean
- echo "TEST - $0 - ALWAYS FAIL"
- set +e
- $0 -f f tmp/output
- rc=$?
- set -e
- if [ $rc -eq 0 ]; then
- echo "TEST FAILED"
- rc=151
- exit $rc
- fi
-
-##### Populate cache integration test ####
- make -C ${BUILD_BASE} clean
- echo "TEST - $0 - POPULATE CACHE"
- rm -rf tmp
- mkdir tmp
- mkdir tmp/cache
- set +e
- $0 -c ${BUILD_CACHE_URI} -f PD tmp/output
- rc=$?
- set -e
- if [ $rc -ne 0 ]; then
- echo "TEST FAILED"
- rc=152
- exit $rc
- fi
-
- if [ ! -f tmp/output/fake-debug.iso ] || [ ! -f tmp/output/fake-debug.iso ]; then
- echo "TEST FAILED"
- rc=153
- exit $rc
- fi
- rc=154
- cache_check
-
-##### Build uinge cache integration test ####
- make -C ${BUILD_BASE} clean
- echo "TEST - $0 - BUILD USING CACHE"
- set +e
- $0 -c ${BUILD_CACHE_URI} -f D tmp/output
- rc=$?
- set -e
- if [ $rc -ne 0 ]; then
- echo "TEST FAILED"
- rc=155
- exit $rc
- fi
-
- if [ ! -f tmp/output/fake-debug.iso ] || [ ! -f tmp/output/fake-debug.iso ]; then
- echo "TEST FAILED"
- rc=156
- exit $rc
- fi
- rc=157
- cache_check
-
-#### Repopulate cache because cache non existant ####
- make -C ${BUILD_BASE} clean
- echo "TEST - $0 - BUILD USING CACHE"
- rm -rf tmp/cache/*
- set +e
- $0 -c ${BUILD_CACHE_URI} -f D tmp/output
- rc=$?
- set -e
- if [ $rc -ne 0 ]; then
- echo "TEST FAILED"
- rc=158
- exit $rc
- fi
-
- if [ ! -f tmp/output/fake-debug.iso ] || [ ! -f tmp/output/fake-debug.iso ]; then
- echo "TEST FAILED"
- rc=160
- exit $rc
- fi
- rc=161
- cache_check
-
-# Repopulate cache because cach is ivalidated
-#TBD
-rm -rf tmp
-echo "All tests passed!"
-rc=0
-exit $rc
-}
-
-#DEFAULT VALUES
+############################################################################
+# BEGIN of variables to customize
+#
BUILD_BASE=$(readlink -e ../build/)
RESULT_DIR="${BUILD_BASE}/release"
BUILD_SPEC="${BUILD_BASE}/config.mk"
@@ -207,8 +89,14 @@ CACHE_DIR="cache"
LOCAL_CACHE_ARCH_NAME="fuel-cache"
REMOTE_CACHE_ARCH_NAME="fuel_cache-$(md5sum ${BUILD_SPEC}| cut -f1 -d " ")"
REMOTE_ACCESS_METHD=curl
-
-#SCRIPT ASSIGNED VARIABLES
+INCLUDE_DIR=../include
+#
+# END of variables to customize
+############################################################################
+
+############################################################################
+# BEGIN of script assigned variables
+#
SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
LOCK_FILE="${SCRIPT_DIR}/.build.lck"
TEST_SUCCEED=0
@@ -220,6 +108,7 @@ RECURSIV=0
DETACH=0
DEBUG=0
INTEGRATION_TEST=0
+FULL_INTEGRATION_TEST=0
INTERACTIVE=0
BUILD_CACHE_URI=
BUILD_SPEC=
@@ -227,8 +116,22 @@ BUILD_DIR=
BUILD_LOG=
BUILD_VERSION=
MAKE_ARGS=
-
-while getopts "s:c:v:f:l:r:RTh" OPTION
+#
+# END of script assigned variables
+############################################################################
+
+############################################################################
+# BEGIN of include pragmas
+#
+source ${INCLUDE_DIR}/build.sh.debug
+#
+# END of include
+############################################################################
+
+############################################################################
+# BEGIN of main
+#
+while getopts "s:c:v:f:l:r:RtTh" OPTION
do
case $OPTION in
h)
@@ -264,8 +167,13 @@ do
RECURSIVE=1
;;
+ t)
+ INTEGRATION_TEST=1
+ ;;
+
T)
INTEGRATION_TEST=1
+ FULL_INTEGRATION_TEST=1
;;
*)
@@ -316,7 +224,7 @@ for ((i=0; i<${#BUILD_FLAGS};i++)); do
;;
*)
- echo "${BUILD_FLAGS:$i:1} is not a valid build flag"
+ echo "${BUILD_FLAGS:$i:1} is not a valid build flag - exiting ...."
rc=100
exit $rc
;;
@@ -330,13 +238,13 @@ if [ ${INTEGRATION_TEST} -eq 1 ]; then
fi
if [ ! -f ${BUILD_SPEC} ]; then
- echo "spec file does not exist: $BUILD_SPEC"
+ echo "spec file does not exist: $BUILD_SPEC - exiting ...."
rc=100
exit $rc
fi
if [ -z ${BUILD_DIR} ]; then
- echo "Missing build directory"
+ echo "Missing build directory - exiting ...."
rc=100
exit $rc
fi
@@ -379,7 +287,7 @@ if [ ! -z ${BUILD_CACHE_URI} ]; then
mkdir /tmp/cache
echo "Downloading cach file ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME} ..."
set +e
- ${REMOTE_ACCESS_METHD} -o /tmp/cache/${LOCAL_CACHE_ARCH_NAME}.tgz ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME}
+ ${REMOTE_ACCESS_METHD} -o /tmp/cache/${LOCAL_CACHE_ARCH_NAME}.tgz ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME}.tgz
rc=$?
set -e
if [ $rc -ne 0 ]; then
@@ -389,10 +297,11 @@ if [ ! -z ${BUILD_CACHE_URI} ]; then
echo "Unpacking cache file ..."
tar -C /tmp/cache -xvf /tmp/cache/${LOCAL_CACHE_ARCH_NAME}.tgz
cp /tmp/cache/cache/.versions ${BUILD_BASE}/.
- echo "Validating cache content ..."
set +e
make -C ${BUILD_BASE} validate-cache;
rc=$?
+ set -e
+
if [ $rc -ne 0 ]; then
echo "Cache invalid - a new cache will be built "
POPULATE_CACHE=1
@@ -455,13 +364,13 @@ rc=$?
set -e
if [ $rc -gt 0 ]; then
- echo "Build: make prepare-cache failed, exiting ..."
+ echo "Build: make prepare-cache failed - exiting ..."
rc=100
exit $rc
fi
echo "Copying built OPNFV .iso file to target directory ${BUILD_DIR} ..."
rm -rf ${BUILD_DIR}
-mkdir ${BUILD_DIR}
+mkdir -p ${BUILD_DIR}
cp ${BUILD_BASE}/.versions ${BUILD_DIR}
cp ${RESULT_DIR}/*.iso* ${BUILD_DIR}
@@ -470,9 +379,12 @@ if [ $POPULATE_CACHE -eq 1 ]; then
echo "Building cache ..."
tar --dereference -C ${BUILD_BASE} -caf ${BUILD_BASE}/${LOCAL_CACHE_ARCH_NAME}.tgz ${CACHE_DIR}
echo "Uploading cache ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME}"
- ${REMOTE_ACCESS_METHD} -T ${BUILD_BASE}/${LOCAL_CACHE_ARCH_NAME}.tgz ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME}
+ ${REMOTE_ACCESS_METHD} -T ${BUILD_BASE}/${LOCAL_CACHE_ARCH_NAME}.tgz ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME}.tgz
rm ${BUILD_BASE}/${LOCAL_CACHE_ARCH_NAME}.tgz
fi
fi
echo "Success!!!"
exit 0
+#
+# END of main
+############################################################################
diff --git a/fuel/include/build.sh.debug b/fuel/include/build.sh.debug
new file mode 100644
index 0000000..b6594cb
--- /dev/null
+++ b/fuel/include/build.sh.debug
@@ -0,0 +1,355 @@
+##############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+# stefan.k.berg@ericsson.com
+# jonas.bjurel@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+############################################################################
+# BEGIN of build script integration test
+#
+integration-test () {
+
+test_clean () {
+ make -C ${BUILD_BASE} clean
+ test_cache_clean
+ test_output_clean
+}
+
+test_cache_clean () {
+ rm -rf ${TEST_BUILD_CACHE_URI_DIR}
+}
+
+test_output_clean () {
+ rm -rf ${TEST_BUILD_RESULT_DIR}
+}
+
+test_prepare () {
+ test_clean
+ test_cache_clean
+ test_output_clean
+ mkdir -p ${TEST_BUILD_CACHE_URI_DIR}
+}
+
+test_cache_check () {
+ if [ ! -f ${BUILD_BASE}/fuel-6.0.1.iso ] || \
+ [ ! -f ${BUILD_BASE}/.versions ] || \
+ [ ! -f ${BUILD_BASE}/opendaylight/.odl-build-history ]; \
+ [ ! -f ${BUILD_BASE}/opendaylight/.odl-build.log ]; then
+ echo "Cache not present in the build system"
+ echo "TEST FAILED"
+ exit $rc
+ fi
+
+ if [ ! -f ${TEST_BUILD_CACHE_URI_DIR}/${REMOTE_CACHE_ARCH_NAME}.tgz ]; then
+ echo "Cache archive not present"
+ echo "TEST FAILED"
+ exit $rc
+ fi
+
+ if tar -tf ${TEST_BUILD_CACHE_URI_DIR}/${REMOTE_CACHE_ARCH_NAME}.tgz ${CACHE_DIR}/.mark > /dev/null 2>&1; then
+ echo "Cache archive is invalidated"
+ echo "TEST FAILED"
+ exit $rc
+ fi
+}
+
+test_output_check () {
+ if [ ! -f ${TEST_BUILD_RESULT_DIR}/*.iso ] || \
+ [ ! -f ${TEST_BUILD_RESULT_DIR}/*.txt ] || \
+ [ ! -f ${TEST_BUILD_RESULT_DIR}/.versions ]; then
+ echo "Build output not produced by the build system"
+ echo "TEST FAILED"
+ exit $rc
+ fi
+}
+
+test_invalidate_cache () {
+ tar -C ${TEST_BUILD_CACHE_URI_DIR} -xf ${TEST_BUILD_CACHE_URI_DIR}/${REMOTE_CACHE_ARCH_NAME}.tgz
+ echo "cache.mk 123" > ${TEST_BUILD_CACHE_URI_DIR}/cache/.versions
+ echo "config.mk 123" >> ${TEST_BUILD_CACHE_URI_DIR}/cache/.versions
+ echo "fuel 123" >> ${TEST_BUILD_CACHE_URI_DIR}/cache/.versions
+ echo "odl 123" >> ${TEST_BUILD_CACHE_URI_DIR}/cache/.versions
+
+ echo "This file marks the invalid cache" > ${TEST_BUILD_CACHE_URI_DIR}/cache/.mark
+ rm -f ${TEST_BUILD_CACHE_URI_DIR}/${REMOTE_CACHE_ARCH_NAME}.tgz
+ tar --dereference -C ${TEST_BUILD_CACHE_URI_DIR} -caf ${TEST_BUILD_CACHE_URI_DIR}/${REMOTE_CACHE_ARCH_NAME}.tgz ${CACHE_DIR}
+}
+
+##### Always succeed integration test ####
+build_tc1 () {
+ echo "TEST - $0 - ALWAYS SUCCEED"
+ set +e
+ $0 -f s tmp/output
+ rc=$?
+ set -e
+ if [ $rc -ne 0 ]; then
+ echo "TEST FAILED"
+ rc=151
+ exit $rc
+ fi
+}
+
+##### Always fail integration test (TC2) ####
+build_tc2 () {
+ echo "TEST - $0 - ALWAYS FAIL"
+ set +e
+ $0 -f f tmp/output
+ rc=$?
+ set -e
+ if [ $rc -eq 0 ]; then
+ echo "TEST FAILED"
+ rc=152
+ exit $rc
+ fi
+}
+
+##### Fake build and populate dummy cache - integration test (TC3) ####
+build_tc3 () {
+ echo "TEST - $0 - FAKE BUILD AND POPULATE DUMMY CACHE"
+ rm -rf tmp
+ mkdir -p ${TEST_BUILD_CACHE_URI_DIR}
+ set +e
+ $0 -c ${TEST_BUILD_CACHE_URI} -f PD ${TEST_BUILD_RESULT_DIR}
+ rc=$?
+ set -e
+ if [ $rc -ne 0 ]; then
+ echo "Build failed"
+ echo "TEST FAILED"
+ rc=152
+ exit $rc
+ fi
+
+ rc=153
+ test_output_check
+
+ rc=154
+ test_cache_check
+}
+
+##### Fake build usng dummy cache integration test (TC4) ####
+build_tc4 () {
+ echo "TEST - $0 - FAKE BUILD USING DUMMY CACHE"
+ set +e
+ $0 -c ${TEST_BUILD_CACHE_URI} -f D ${TEST_BUILD_RESULT_DIR}
+ rc=$?
+ set -e
+ if [ $rc -ne 0 ]; then
+ echo "Build failed"
+ echo "TEST FAILED"
+ rc=155
+ exit $rc
+ fi
+
+ rc=156
+ test_output_check
+
+ rc=157
+ test_cache_check
+}
+
+#### Fake build and repopulate cache because cache non existant (TC5) ####
+build_tc5 () {
+ echo "TEST - $0 - FAKE BUILD AND REPOPULATE CACHE BECAUSE NON EXISTANT"
+ rm -rf ${TEST_BUILD_CACHE_URI_DIR}/*
+ set +e
+ $0 -c ${TEST_BUILD_CACHE_URI} -f D ${TEST_BUILD_RESULT_DIR}
+ rc=$?
+ set -e
+ if [ $rc -ne 0 ]; then
+ echo "Build failed"
+ echo "TEST FAILED"
+ rc=158
+ exit $rc
+ fi
+
+ rc=159
+ test_output_check
+
+ rc=160
+ test_cache_check
+}
+
+#### Fake build and repopulate cache because cache is ivalidated (TC6) ####
+build_tc6 () {
+ echo "TEST - $0 - FAKE BUILD AND REPOPULATE DUE TO CACHE INVALIDATION"
+ test_invalidate_cache
+ set +e
+ $0 -c ${TEST_BUILD_CACHE_URI} -f D ${TEST_BUILD_RESULT_DIR}
+ rc=$?
+ set -e
+ if [ $rc -ne 0 ]; then
+ echo "Build failed"
+ echo "TEST FAILED"
+ rc=161
+ exit $rc
+ fi
+
+ rc=162
+ test_output_check
+
+ rc=163
+ test_cache_check
+}
+
+#### Real build and populate cache (TC7) ####
+build_tc7 () {
+ echo "TEST - $0 - FULL BUILD AND POPULATE CACHE"
+ test_prepare
+ BUILD_START=$(date +%s)
+ set +e
+ $0 -c ${TEST_BUILD_CACHE_URI} -f Pi ${TEST_BUILD_RESULT_DIR}
+ rc=$?
+ set -e
+ if [ $rc -ne 0 ]; then
+ echo "Build failed"
+ echo "TEST FAILED"
+ rc=164
+ exit $rc
+ fi
+
+ BUILD_TIME=$(($(date +%s)-${BUILD_START}))
+ rc=165
+ test_output_check
+ rc=166
+ test_cache_check
+}
+
+#### Real build using cache from TC7 (TC8) ####
+build_tc8 () {
+ echo "TEST - $0 - INCREMENTAL BUILD AND USING CACHE"
+ BUILD_START=$(date +%s)
+ set +e
+ $0 -c ${TEST_BUILD_CACHE_URI} -f i ${TEST_BUILD_RESULT_DIR}
+ rc=$?
+ set -e
+ if [ $rc -ne 0 ]; then
+ echo "Build failed"
+ echo "TEST FAILED"
+ rc=167
+ exit $rc
+ fi
+ BUILD_TIME=$(($(date +%s)-${BUILD_START}))
+ test_output_check
+ rc=168
+ test_cache_check
+ rc=169
+}
+
+TEST_BUILD_ARTIFACT_DIR=${BUILD_BASE}/test
+TEST_BUILD_CACHE_URI_DIR=${TEST_BUILD_ARTIFACT_DIR}
+TEST_BUILD_CACHE_URI=file://${TEST_BUILD_CACHE_URI_DIR}
+TEST_BUILD_RESULT_DIR=${BUILD_BASE}/tmp/output
+
+TEST_START=$(date +%s)
+# Run test case 1: Always succeed integration test
+echo ==========================================================================
+echo ========================= RUNNING TEST CASE 1 ============================
+echo ==========================================================================
+test_prepare
+build_tc1
+
+# Run test case 2: Always fail integration test
+echo ==========================================================================
+echo ========================= RUNNING TEST CASE 2 ============================
+echo ==========================================================================
+test_prepare
+build_tc2
+
+# Run test case 3: Fake build and populate a build cach
+echo ==========================================================================
+echo ========================= RUNNING TEST CASE 3 ============================
+echo ==========================================================================
+test_prepare
+build_tc3
+
+# Run test case 4: Fake build using the build cache produced in TC 3
+echo ==========================================================================
+echo ========================= RUNNING TEST CASE 4 ============================
+echo ==========================================================================
+build_tc4
+
+# Run test case 5: Fake build and repopulate the build cache as there is no existing one
+echo ==========================================================================
+echo ========================= RUNNING TEST CASE 5 ============================
+echo ==========================================================================
+test_prepare
+build_tc5
+
+# Run test case 6: Fake build and repopulate the build cache as there the current one is invalidated/outdated
+echo ==========================================================================
+echo ========================= RUNNING TEST CASE 6 ============================
+echo ==========================================================================
+test_invalidate_cache
+build_tc4
+
+# Run test case 7: Build from upstream repos and populate the build cache
+if [ ${FULL_INTEGRATION_TEST} -eq 1 ]; then
+echo ==========================================================================
+echo ========================= RUNNING TEST CASE 7 ============================
+echo ==========================================================================
+ test_prepare
+ build_tc7
+ POPULATE_CACHE_TIME=${BUILD_TIME}
+fi
+
+# Run test case 8: Build from cache produced in TC7
+if [ ${FULL_INTEGRATION_TEST} -eq 1 ]; then
+echo ==========================================================================
+echo ========================= RUNNING TEST CASE 8 ============================
+echo ==========================================================================
+ build_tc8
+ PARTIAL_BUILD_TIME=${BUILD_TIME}
+fi
+
+# Run test case 9: Build and repopulate the cache as the current one is invalidated/not up to date
+if [ ${FULL_INTEGRATION_TEST} -eq 1 ]; then
+echo ==========================================================================
+echo ========================= RUNNING TEST CASE 9 ============================
+echo ==========================================================================
+ echo "TEST - $0 - REPOPULATING CACHE DUE TO CACHE INVALIDATION"
+ test_invalidate_cache
+ build_tc8
+fi
+
+TEST_TIME=$(($(date +%s)-${TEST_START}))
+
+rm -rf tmp
+echo "All tests passed!, test duration was ${TEST_TIME} seconds"
+if [ ${FULL_INTEGRATION_TEST} -eq 1 ]; then
+ echo "Full build took ${POPULATE_CACHE_TIME} second"
+ echo "Incremental build took ${PARTIAL_BUILD_TIME} seconds"
+fi
+rc=0
+exit $rc
+}
+#
+# END of build script integration test
+############################################################################
+
+############################################################################
+# BEGIN of debug_make - a fake make initiated with -D for debugging
+#
+debug_make () {
+ make -C ${BUILD_BASE} clean
+ echo "This is a fake debug fuel .iso image" > ${BUILD_BASE}/fuel-6.0.1.iso
+
+ echo "This is a fake debug versions file" > ${BUILD_BASE}/.versions
+ echo "cache.mk" $(md5sum ${BUILD_BASE}/cache.mk | cut -f1 -d " ") >> ${BUILD_BASE}/.versions
+ echo "config.mk" $(md5sum ${BUILD_BASE}/config.mk | cut -f1 -d " ") >> ${BUILD_BASE}/.versions
+ echo "fuel" $(git ls-remote $(make -C ${BUILD_BASE} get-fuel-repo | grep https) | awk '{print $(NF-1)}') >> ${BUILD_BASE}/.versions
+ echo "odl" $(git ls-remote $(make -C ${BUILD_BASE} get-odl-repo | grep https) | awk '{print $(NF-1)}') >> ${BUILD_BASE}/.versions
+
+ rm -rf ${BUILD_BASE}/release
+ mkdir ${BUILD_BASE}/release
+ echo "This is a fake debug OPNFV .iso image" > ${BUILD_BASE}/release/fake-debug.iso
+ echo "This is a fake debug OPNFV .iso.txt message" > ${BUILD_BASE}/release/fake-debug.iso.txt
+ echo "This a fake debug odl build history" > ${BUILD_BASE}/opendaylight/.odl-build-history
+ echo "This a fake debug odl build log" > ${BUILD_BASE}/opendaylight/.odl-build.log
+}
+#
+# END of debug_make
+############################################################################