aboutsummaryrefslogtreecommitdiffstats
path: root/fuel/build/Makefile
diff options
context:
space:
mode:
authorStefan K. Berg <stefan.k.berg@ericsson.com>2015-10-15 12:51:26 +0200
committerStefan K. Berg <stefan.k.berg@ericsson.com>2015-10-15 17:01:59 +0200
commite532d93d6fef7f23f3f227eaf855fff2be846ae7 (patch)
tree0c278a94ac8d7b15147255d3b213a6eba26160e5 /fuel/build/Makefile
parent66fae447879604849beb2108242e33cc0c6fc867 (diff)
Improved caching functionality and tracability
In order to enable the caching of more build artifacts than the Fuel iso during CI builds, the caching functionality and the CI build frontend "build.sh" has been rewritten. The build.sh script will now rely on the "make cache" functionality of the top Makefile to make sure that the build is using cache handling. The underlying cache logic is implemented in "cache.mk" for the top makefile and those recursive levels that do not produce cachable artifacts themselves in that they are only calling the SUBDIRS of their lower level (like f_isoroot). All "leaf" Makefiles will however need to implement three cache targets in their top Makefile (for visibility): clean-cache: Clean all files relating to the handling of caches. get-cache: Attempt to fetch a cached artifact using a SHA1 key. put-cache: Store a built artifact into the cache. They can just implement a simple "no-op" functionality for thes targets if they do not have any use of the caching functionality. If they are to use the caching functionality, they must make sure to implement a logic that produces a SHA1 hash based on the source of the data they are to cache - for upstream repos one suggestion is to use the commit ID of the used upstream HEAD. For examples, see the top Makefile that implements this logic for the Fuel ISO build, taking into consideration the commit IDs of all the upstream repositories used. To improve tracability, the root directory of the ISO will contain the file gitinfo.txt, which is meant to detail the upstream repo and commit ID used for all upstream dependencies of a build. If you are adding additional upstream dependencies, make sure to use the repo_info.sh tool to add this data. The cache tool cache.sh has no notion of the cache data it is storing - from the tools perspective, cache data is just a binary blob piped in or out of the tool. This blob is stored by the cache tool at the cache location as <SHA1>.blob, together with an associated meta file <SHA1>.meta. The cache meta file currently holds just one line: Expires: <epoch time> This file is expected to be used to iterate through the cache objects and retire those who's expiry date has passed. Currently objects will always have an expiry date of two weeks into the future, but down the road the "cache put" functionality may be amended with an optional "age" argument that can set a different expiration time. New tools in this commit: cache.sh - the cache logic repo_info.sh - the (optionally recursive) repo information logger Change-Id: I8a40546c21febeecc9de6d82c0ceb6bc60b04205 Signed-off-by: Stefan K. Berg <stefan.k.berg@ericsson.com>
Diffstat (limited to 'fuel/build/Makefile')
-rw-r--r--fuel/build/Makefile65
1 files changed, 46 insertions, 19 deletions
diff --git a/fuel/build/Makefile b/fuel/build/Makefile
index 437b793b1..767b930d7 100644
--- a/fuel/build/Makefile
+++ b/fuel/build/Makefile
@@ -13,8 +13,6 @@ SHELL = /bin/bash
# BEGIN of variables to customize
#
#Input args
-export UNIT_TEST = FALSE
-export INTERACTIVE = TRUE
export ISOSRC = file:$(shell pwd)/fuel-6.1.iso
export ISOCACHE = $(shell pwd)/$(shell basename $(ISOSRC))
export PRODNO = "OPNFV_BGS"
@@ -24,15 +22,15 @@ export NEWISO = $(shell pwd)/release/opnfv-${REVSTATE}.iso
# Note! Invoke with "make REVSTATE=RXXXX all" to make release build!
# Invoke with ICOCACHE=/full/path/to/iso if cached ISO is in non-standard location.
-#Build variables
+# Build variables
export BUILD_BASE := $(shell pwd)
export DEB_DEST := $(BUILD_BASE)/release/packages/ubuntu/pool/main
export UDEB_DEST := $(BUILD_BASE)/release/packages/ubuntu/pool/debian-installer
export PUPPET_DEST := $(BUILD_BASE)/release/puppet/modules
-export CACHE_DIR := $(BUILD_BASE)/cache
export VERSION_FILE := $(BUILD_BASE)/.versions
export DOCKERIMG = opnfv.org/ubuntu-builder:14.04
export TOPDIR := $(shell pwd)
+export REPOINFO := $(BUILD_BASE)/repo_info.sh
#Build subclasses
@@ -49,16 +47,13 @@ ORIGDIR := $(TOPDIR)/origiso
SUBCLEAN = $(addsuffix .clean,$(SUBDIRS))
-
.PHONY: all
all:
@docker version >/dev/null 2>&1 || (echo 'No Docker installation available'; exit 1)
- @echo "Versions of cached build results built by" $(shell hostname) "at" $(shell date -u) > $(VERSION_FILE)
- @echo "cache.mk" $(shell md5sum $(BUILD_BASE)/cache.mk | cut -f1 -d " ") >> $(VERSION_FILE)
- @echo "config.mk" $(shell md5sum $(BUILD_BASE)/config.mk | cut -f1 -d " ") >> $(VERSION_FILE)
@make -C docker
@docker/runcontext $(DOCKERIMG) $(MAKE) $(MAKEFLAGS) iso
+
############################################################################
# BEGIN of Include definitions
#
@@ -70,7 +65,9 @@ include cache.mk
$(ISOCACHE):
# Clone Fuel to non-persistent location and build
- cd /tmp && git clone $(FUEL_MAIN_REPO)
+ if [ ! -d /tmp/fuel-main ]; then \
+ cd /tmp && git clone $(FUEL_MAIN_REPO); \
+ fi
cd /tmp/fuel-main && git checkout $(FUEL_MAIN_TAG)
@echo "fuel" `git -C /tmp/fuel-main show | grep commit | head -1 | cut -d " " -f2` >> $(VERSION_FILE)
# Setup cgroups for docker-in-docker
@@ -86,6 +83,7 @@ $(ISOCACHE):
#
cd /tmp/fuel-main && ./prepare-build-env.sh
cd /tmp/fuel-main && make repos
+ $(REPOINFO) -r /tmp/fuel-main > gitinfo_fuel.txt
#
cd /tmp/fuel-main && make iso
mv /tmp/fuel-main/build/artifacts/fuel*.iso .
@@ -110,21 +108,14 @@ $(SUBDIRS):
patch-packages:
ORIGISO=$(ISOCACHE) REVSTATE=$(REVSTATE) $(MAKE) -C $@ -f Makefile release
-.PHONY: build-clean $(SUBCLEAN)
-build-clean: $(SUBCLEAN)
- $(MAKE) -C patch-packages -f Makefile clean
- @rm -Rf release
- @rm -Rf newiso
- @rm -f $(NEWISO)
-
.PHONY: clean $(SUBCLEAN)
-clean: clean-cache $(SUBCLEAN)
+clean: $(SUBCLEAN)
$(MAKE) -C patch-packages -f Makefile clean
@rm -f *.iso
@rm -Rf release
@rm -Rf newiso
@rm -f $(NEWISO)
- @rm -f $(BUILD_BASE)/.versions
+ @rm -f $(BUILD_BASE)/gitinfo_*.txt
$(SUBCLEAN): %.clean:
$(MAKE) -C $* -f Makefile clean
@@ -132,7 +123,8 @@ $(SUBCLEAN): %.clean:
# Todo: Make things smarter - we shouldn't need to clean everything
# betwen make invocations.
.PHONY: iso
-iso: build-clean $(ISOCACHE) $(SUBDIRS) patch-packages
+iso: $(ISOCACHE) $(SUBDIRS) patch-packages
+ $(REPOINFO) . > gitinfo_main.txt
install/install.sh iso $(ISOCACHE) $(NEWISO) $(PRODNO) $(REVSTATE)
@printf "\n\nProduct ISO is $(NEWISO)\n\n"
@@ -141,3 +133,38 @@ iso: build-clean $(ISOCACHE) $(SUBDIRS) patch-packages
debug:
@docker version >/dev/null 2>&1 || (echo 'No Docker installation available'; exit 1)
docker/runcontext $(DOCKERIMG) bash
+
+#############################################################################
+# Cache operations - only used when building through ci/build.sh
+#############################################################################
+
+# Create a unique hash to be used for getting and putting cache, based on:
+# - The commit ID of the full Fuel repo structre
+# - The contents of all local Fuel patches
+.cacheid:
+ cd /tmp && git clone $(FUEL_MAIN_REPO)
+ cd /tmp/fuel-main && git checkout $(FUEL_MAIN_TAG)
+ cd /tmp/fuel-main && make repos
+ $(REPOINFO) -r /tmp/fuel-main > .cachedata
+ sha1sum fuel-main*.patch >> .cachedata
+ cat .cachedata | $(CACHETOOL) getid > .cacheid
+ # Not removing fuel-main as it is re-used in build
+
+# Clean local data related to caching - called prior to ordinary build
+.PHONY: clean-cache
+clean-cache:
+ rm -f .cachedata .cacheid
+
+# Try to download cache - called prior to ordinary build
+.PHONY: get-cache
+get-cache: .cacheid
+ @if $(CACHETOOL) check $(shell cat .cacheid); then \
+ $(CACHETOOL) get $(shell cat .cacheid) | tar xf -;\
+ else \
+ exit 0;\
+ fi
+
+# Store cache if not already stored - called after ordinary build
+.PHONY: put-cache
+put-cache: .cacheid
+ @tar cf - fuel*.iso gitinfo_fuel.txt | $(CACHETOOL) put $(shell cat .cacheid)