diff options
| -rw-r--r-- | fuel/build/Makefile | 65 | ||||
| -rw-r--r-- | fuel/build/cache.mk | 111 | ||||
| -rwxr-xr-x | fuel/build/cache.sh | 112 | ||||
| -rw-r--r-- | fuel/build/config.mk | 5 | ||||
| -rwxr-xr-x | fuel/build/docker/runcontext | 10 | ||||
| -rw-r--r-- | fuel/build/f_isoroot/Makefile | 8 | ||||
| -rw-r--r-- | fuel/build/f_isoroot/cache.mk | 37 | ||||
| -rw-r--r-- | fuel/build/f_isoroot/f_bootstrap/Makefile | 19 | ||||
| -rw-r--r-- | fuel/build/f_isoroot/f_kscfg/Makefile | 19 | ||||
| -rw-r--r-- | fuel/build/f_isoroot/f_odlpluginbuild/Makefile | 48 | ||||
| -rw-r--r-- | fuel/build/f_isoroot/f_repobuild/Makefile | 45 | ||||
| -rwxr-xr-x | fuel/build/install/install.sh | 6 | ||||
| -rwxr-xr-x | fuel/build/repo_info.sh | 63 | ||||
| -rwxr-xr-x | fuel/ci/build.sh | 542 | ||||
| -rw-r--r-- | fuel/include/build.sh.debug | 355 |
15 files changed, 592 insertions, 853 deletions
diff --git a/fuel/build/Makefile b/fuel/build/Makefile index 437b793b1..767b930d7 100644 --- a/fuel/build/Makefile +++ b/fuel/build/Makefile @@ -13,8 +13,6 @@ SHELL = /bin/bash # BEGIN of variables to customize # #Input args -export UNIT_TEST = FALSE -export INTERACTIVE = TRUE export ISOSRC = file:$(shell pwd)/fuel-6.1.iso export ISOCACHE = $(shell pwd)/$(shell basename $(ISOSRC)) export PRODNO = "OPNFV_BGS" @@ -24,15 +22,15 @@ export NEWISO = $(shell pwd)/release/opnfv-${REVSTATE}.iso # Note! Invoke with "make REVSTATE=RXXXX all" to make release build! # Invoke with ICOCACHE=/full/path/to/iso if cached ISO is in non-standard location. -#Build variables +# Build variables export BUILD_BASE := $(shell pwd) export DEB_DEST := $(BUILD_BASE)/release/packages/ubuntu/pool/main export UDEB_DEST := $(BUILD_BASE)/release/packages/ubuntu/pool/debian-installer export PUPPET_DEST := $(BUILD_BASE)/release/puppet/modules -export CACHE_DIR := $(BUILD_BASE)/cache export VERSION_FILE := $(BUILD_BASE)/.versions export DOCKERIMG = opnfv.org/ubuntu-builder:14.04 export TOPDIR := $(shell pwd) +export REPOINFO := $(BUILD_BASE)/repo_info.sh #Build subclasses @@ -49,16 +47,13 @@ ORIGDIR := $(TOPDIR)/origiso SUBCLEAN = $(addsuffix .clean,$(SUBDIRS)) - .PHONY: all all: @docker version >/dev/null 2>&1 || (echo 'No Docker installation available'; exit 1) - @echo "Versions of cached build results built by" $(shell hostname) "at" $(shell date -u) > $(VERSION_FILE) - @echo "cache.mk" $(shell md5sum $(BUILD_BASE)/cache.mk | cut -f1 -d " ") >> $(VERSION_FILE) - @echo "config.mk" $(shell md5sum $(BUILD_BASE)/config.mk | cut -f1 -d " ") >> $(VERSION_FILE) @make -C docker @docker/runcontext $(DOCKERIMG) $(MAKE) $(MAKEFLAGS) iso + ############################################################################ # BEGIN of Include definitions # @@ -70,7 +65,9 @@ include cache.mk $(ISOCACHE): # Clone Fuel to non-persistent location and build - cd /tmp && git clone $(FUEL_MAIN_REPO) + if [ ! -d /tmp/fuel-main ]; then \ + cd /tmp && git clone $(FUEL_MAIN_REPO); \ + fi cd /tmp/fuel-main && git checkout $(FUEL_MAIN_TAG) @echo "fuel" `git -C /tmp/fuel-main show | grep commit | head -1 | cut -d " " -f2` >> $(VERSION_FILE) # Setup cgroups for docker-in-docker @@ -86,6 +83,7 @@ $(ISOCACHE): # cd /tmp/fuel-main && ./prepare-build-env.sh cd /tmp/fuel-main && make repos + $(REPOINFO) -r /tmp/fuel-main > gitinfo_fuel.txt # cd /tmp/fuel-main && make iso mv /tmp/fuel-main/build/artifacts/fuel*.iso . @@ -110,21 +108,14 @@ $(SUBDIRS): patch-packages: ORIGISO=$(ISOCACHE) REVSTATE=$(REVSTATE) $(MAKE) -C $@ -f Makefile release -.PHONY: build-clean $(SUBCLEAN) -build-clean: $(SUBCLEAN) - $(MAKE) -C patch-packages -f Makefile clean - @rm -Rf release - @rm -Rf newiso - @rm -f $(NEWISO) - .PHONY: clean $(SUBCLEAN) -clean: clean-cache $(SUBCLEAN) +clean: $(SUBCLEAN) $(MAKE) -C patch-packages -f Makefile clean @rm -f *.iso @rm -Rf release @rm -Rf newiso @rm -f $(NEWISO) - @rm -f $(BUILD_BASE)/.versions + @rm -f $(BUILD_BASE)/gitinfo_*.txt $(SUBCLEAN): %.clean: $(MAKE) -C $* -f Makefile clean @@ -132,7 +123,8 @@ $(SUBCLEAN): %.clean: # Todo: Make things smarter - we shouldn't need to clean everything # betwen make invocations. .PHONY: iso -iso: build-clean $(ISOCACHE) $(SUBDIRS) patch-packages +iso: $(ISOCACHE) $(SUBDIRS) patch-packages + $(REPOINFO) . > gitinfo_main.txt install/install.sh iso $(ISOCACHE) $(NEWISO) $(PRODNO) $(REVSTATE) @printf "\n\nProduct ISO is $(NEWISO)\n\n" @@ -141,3 +133,38 @@ iso: build-clean $(ISOCACHE) $(SUBDIRS) patch-packages debug: @docker version >/dev/null 2>&1 || (echo 'No Docker installation available'; exit 1) docker/runcontext $(DOCKERIMG) bash + +############################################################################# +# Cache operations - only used when building through ci/build.sh +############################################################################# + +# Create a unique hash to be used for getting and putting cache, based on: +# - The commit ID of the full Fuel repo structre +# - The contents of all local Fuel patches +.cacheid: + cd /tmp && git clone $(FUEL_MAIN_REPO) + cd /tmp/fuel-main && git checkout $(FUEL_MAIN_TAG) + cd /tmp/fuel-main && make repos + $(REPOINFO) -r /tmp/fuel-main > .cachedata + sha1sum fuel-main*.patch >> .cachedata + cat .cachedata | $(CACHETOOL) getid > .cacheid + # Not removing fuel-main as it is re-used in build + +# Clean local data related to caching - called prior to ordinary build +.PHONY: clean-cache +clean-cache: + rm -f .cachedata .cacheid + +# Try to download cache - called prior to ordinary build +.PHONY: get-cache +get-cache: .cacheid + @if $(CACHETOOL) check $(shell cat .cacheid); then \ + $(CACHETOOL) get $(shell cat .cacheid) | tar xf -;\ + else \ + exit 0;\ + fi + +# Store cache if not already stored - called after ordinary build +.PHONY: put-cache +put-cache: .cacheid + @tar cf - fuel*.iso gitinfo_fuel.txt | $(CACHETOOL) put $(shell cat .cacheid) diff --git a/fuel/build/cache.mk b/fuel/build/cache.mk index b88ac2fc4..f6db01797 100644 --- a/fuel/build/cache.mk +++ b/fuel/build/cache.mk @@ -1,4 +1,4 @@ -############################################################################# +############################################################################## # Copyright (c) 2015 Ericsson AB and others. # stefan.k.berg@ericsson.com # jonas.bjurel@ericsson.com @@ -8,87 +8,48 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -SHELL = /bin/bash -CACHEVALIDATE := $(addsuffix .validate,$(SUBDIRS)) -CACHECLEAN := $(addsuffix .clean,$(CACHEFILES) $(CACHEDIRS)) - -############################################################################ -# BEGIN of variables to customize -# -#CACHEDIRS := foo/bar - -CACHEFILES += .versions -CACHEFILES += $(shell basename $(ISOSRC)) +############################################################################# +# Cache operations - only used when building through ci/build.sh # -# END of variables to customize -############################################################################ - -.PHONY: prepare-cache -prepare-cache: make-cache-dir $(CACHEDIRS) $(CACHEFILES) +# This is the global cache implementation, providing the main target "cache" +# which is called from ci/build.sh, and recursively calling the cache +# operations clean-cache, get-cache and put-cache on all $(SUBDIRS). +############################################################################# -.PHONY: make-cache-dir -make-cache-dir: - @rm -rf ${CACHE_DIR} - @mkdir ${CACHE_DIR} -.PHONY: clean-cache -clean-cache: $(CACHECLEAN) - @rm -rf ${CACHE_DIR} +export CACHETOOL := $(BUILD_BASE)/cache.sh -.PHONY: $(CACHEDIRS) -$(CACHEDIRS): - @mkdir -p $(dir $(CACHE_DIR)/$@) - @if [ ! -d $(BUILD_BASE)/$@ ]; then\ - mkdir -p $(BUILD_BASE)/$@;\ - fi - @ln -s $(BUILD_BASE)/$@ $(CACHE_DIR)/$@ +# Call sub caches +SUBGETCACHE = $(addsuffix .getcache,$(SUBDIRS)) +$(SUBGETCACHE): %.getcache: + $(MAKE) -C $* -f Makefile get-cache -.PHONY: $(CACHEFILES) -$(CACHEFILES): - @mkdir -p $(dir $(CACHE_DIR)/$@) - @if [ ! -d $(dir $(BUILD_BASE)/$@) ]; then\ - mkdir -p $(dir $(BUILD_BASE)/$@);\ - fi +SUBPUTCACHE = $(addsuffix .putcache,$(SUBDIRS)) +$(SUBPUTCACHE): %.putcache: + $(MAKE) -C $* -f Makefile put-cache - @if [ ! -f $(BUILD_BASE)/$@ ]; then\ - echo " " > $(BUILD_BASE)/$@;\ - ln -s $(BUILD_BASE)/$@ $(CACHE_DIR)/$@;\ - rm -f $(BUILD_BASE)/$@;\ - else\ - ln -s $(BUILD_BASE)/$@ $(CACHE_DIR)/$@;\ - fi +SUBCLEANCACHE = $(addsuffix .cleancache,$(SUBDIRS)) +$(SUBCLEANCACHE): %.cleancache: + $(MAKE) -C $* -f Makefile clean-cache -.PHONY: validate-cache -validate-cache: $(CACHEVALIDATE) - @if [ "$(shell md5sum $(BUILD_BASE)/config.mk | cut -f1 -d " ")" != "$(shell cat $(VERSION_FILE) | grep config.mk | awk '{print $$NF}')" ]; then\ - echo "Cache does not match current config.mk definition, cache must be rebuilt";\ - exit 1;\ - fi; +# Overlay implementation: +# - clean +# - clean cache identities +# - get caches +# - build iso +# - store caches +.PHONY: cached-all +cached-all: clean clean-cache $(SUBCLEANCACHE) get-cache $(SUBGETCACHE) iso put-cache $(SUBPUTCACHE) + @echo "Cached build is complete" - @if [ "$(shell md5sum $(BUILD_BASE)/cache.mk | cut -f1 -d " ")" != "$(shell cat $(VERSION_FILE) | grep cache.mk | awk '{print $$NF}')" ]; then\ - echo "Cache does not match current cache.mk definition, cache must be rebuilt";\ - exit 1;\ - fi; -# Once the Make structure is refactored, this should go in as a validate-cache -# taget in the fuel Makefile - - @REMOTE_ID=$(shell git ls-remote $(FUEL_MAIN_REPO) $(FUEL_MAIN_TAG)^{} | awk '{print $$(NF-1)}'); \ - if [ -z $$REMOTE_ID ] || [ $$REMOTE_ID = " " ]; \ - then \ - REMOTE_ID=$(shell git ls-remote $(FUEL_MAIN_REPO) $(FUEL_MAIN_TAG) | awk '{print $$(NF-1)}'); \ - fi; \ - if [[ $$REMOTE_ID != $(shell cat $(VERSION_FILE) | grep fuel | awk '{print $$NF}') ]]; \ - then \ - echo "Cache does not match upstream Fuel, cache must be rebuilt!"; \ - exit 1; \ +# cache: The target for ci/build.sh +.PHONY: cache +cache: + @if [ -z "${CACHEBASE}" ]; then \ + echo "CACHEBASE not set, are you really building through build.sh?"; \ + exit 1; \ fi - -.PHONY: $(CACHEVALIDATE) -$(CACHEVALIDATE): %.validate: - @echo VALIDATE $(CACHEVALIDATE) - $(MAKE) -C $* -f Makefile validate-cache - -.PHONY: $(CACHECLEAN) -$(CACHECLEAN): %.clean: - rm -rf ${CACHE_DIR}/$* + @docker version >/dev/null 2>&1 || (echo 'No Docker installation available'; exit 1) + @make -C docker + docker/runcontext $(DOCKERIMG) $(MAKE) $(MAKEFLAGS) cached-all diff --git a/fuel/build/cache.sh b/fuel/build/cache.sh new file mode 100755 index 000000000..c8cd1b03d --- /dev/null +++ b/fuel/build/cache.sh @@ -0,0 +1,112 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +CACHETRANSPORT=${CACHETRANSPORT:-"curl --silent"} +CACHEBASE=${CACHEBASE:-"file://${HOME}/cache"} +CACHEMAXAGE=${CACHEMAXAGE:-$[14*24*3600]} +CACHEDEBUG=${CACHEDEBUG:-1} + +debugmsg () { + if [ "$CACHEDEBUG" -eq 1 ]; then + echo "$@" >&2 + fi +} + +errormsg () { + echo "$@" >&2 +} + +# Get a SHA1 based on what's piped into the cache command +getid() { + debugmsg "Generating sha1sum" + sha1sum | sed 's/ .*//' +} + + +# Put in cache +put() { + if check $1; then + debugmsg "SHA1 $1 already in cache, skipping storage" + else + debugmsg "Storing SHA1 $1 in cache" + ${CACHETRANSPORT} -T - ${CACHEBASE}/$1.blob + echo "Expires: $[`date +"%s"` + $CACHEMAXAGE]" | ${CACHETRANSPORT} -T - ${CACHEBASE}/$1.meta + fi + exit 0 +} + +# Get from cache +get() { + local rc + + ${CACHETRANSPORT} -o - ${CACHEBASE}/$1.blob 2>/dev/null + rc=$? + + if [ $rc -eq 0 ]; then + echo "Got SHA1 $1 from cache" 2>/dev/null + else + echo "Tried to get SHA1 $1 from cache but failed" 2>/dev/null + fi + + return $? +} + +# Check if in cache +check() { + local rc + + ${CACHETRANSPORT} ${CACHEBASE}/$1.meta &>/dev/null + rc=$? + + if [ $rc -eq 0 ]; then + debugmsg "Checking for SHA1 $1 in cache and found it, rc = $rc" + else + debugmsg "Checking for SHA1 $1 in cache and failed, rc = $rc" + fi + + return $rc +} + +# Verify that SHA1 seems to be a SHA1... +validSHA1() { + if [ $(echo $1 | wc -c) -ne 41 ]; then + return 1 + else + return 0 + fi +} + +case $1 in + getid) + if [ $# -ne 1 ]; then + errormsg "No arguments can be given to getid!" + exit 1 + fi + getid + ;; + get|check|put) + if [ $# -ne 2 ]; then + errormsg "Only one argument, the SHA1 sum, can be given to getid!" + exit 1 + else + if ! validSHA1 $2; then + errormsg "Invalid SHA1 format!" + exit 1 + fi + fi + + $1 $2 + exit $rc + ;; + *) + errormsg "I only know about getid, check, get and put!" + exit 1 +esac diff --git a/fuel/build/config.mk b/fuel/build/config.mk index 16354d578..e9a5320f4 100644 --- a/fuel/build/config.mk +++ b/fuel/build/config.mk @@ -8,11 +8,6 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -# Change the below value with "uuidgen" to force a full cache rebuild - a -# temporary as we work with improving the caching functionality for the next -# release. -CACHE_RND := 73b88a88-9714-4010-888e-e7cd4b26d5e4 - FUEL_MAIN_REPO := https://github.com/stackforge/fuel-main FUEL_MAIN_TAG = stable/6.1 diff --git a/fuel/build/docker/runcontext b/fuel/build/docker/runcontext index 77e4921e1..341612b4e 100755 --- a/fuel/build/docker/runcontext +++ b/fuel/build/docker/runcontext @@ -42,11 +42,15 @@ EOF` GITROOT=`git rev-parse --show-toplevel` IID=`echo $res | sed 's/.* //'` CID_FILE=`mktemp -u -t runcontext.XXXXXXXXXX` -RUN_CONTEXT_OPT="--cidfile $CID_FILE --privileged=true --rm -i -e HOME=$HOME -u $USER -w $PWD -v ${HOME}/.ssh:${HOME}/.ssh -v $GITROOT:$GITROOT" -if [ "${INTERACTIVE}" != "FALSE" ]; then - RUN_CONTEXT_OPT+=" -t" +# Evaluate the need for bind mounting the cache directory +if [ -n "$CACHEBASE" ]; then + if echo $CACHEBASE | grep -q '^file://'; then + CACHEMOUNT="-v $(echo $CACHEBASE | sed 's;file://;;'):$(echo $CACHEBASE | sed 's;file://;;')" + fi fi +RUN_CONTEXT_OPT="--cidfile $CID_FILE --privileged=true --rm -e HOME=$HOME -e CACHEDEBUG -e CACHETRANSPORT -e CACHEMAXAGE -e CACHEBASE -u $USER -w $PWD -v ${HOME}/.ssh:${HOME}/.ssh -v $GITROOT:$GITROOT $CACHEMOUNT" + echo command: docker run ${RUN_CONTEXT_OPT} $IID "$@" docker run ${RUN_CONTEXT_OPT} $IID "$@" diff --git a/fuel/build/f_isoroot/Makefile b/fuel/build/f_isoroot/Makefile index a9b12d927..3831a09f0 100644 --- a/fuel/build/f_isoroot/Makefile +++ b/fuel/build/f_isoroot/Makefile @@ -26,10 +26,8 @@ clean: $(SUBCLEAN) $(SUBCLEAN): %.clean: $(MAKE) -C $* -f Makefile clean -validate-cache: - @echo "No cache validation schema available for $(shell pwd)" - @echo "Continuing ..." - .PHONY: release -release:clean $(SUBDIRS) +release: $(SUBDIRS) @cp -Rvp release/* ../release/isoroot + +include cache.mk diff --git a/fuel/build/f_isoroot/cache.mk b/fuel/build/f_isoroot/cache.mk new file mode 100644 index 000000000..2df3b6bd1 --- /dev/null +++ b/fuel/build/f_isoroot/cache.mk @@ -0,0 +1,37 @@ +############################################################################## +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +############################################################################# +# Cache operations - only used when building through ci/build.sh +# +# This file is only meant for a top Makefile which is only calling its +# own SUBDIRS, without building any cachable artifact by itself. +############################################################################# + +# Call sub caches +SUBGETCACHE = $(addsuffix .getcache,$(SUBDIRS)) +$(SUBGETCACHE): %.getcache: + $(MAKE) -C $* -f Makefile get-cache + +SUBPUTCACHE = $(addsuffix .putcache,$(SUBDIRS)) +$(SUBPUTCACHE): %.putcache: + $(MAKE) -C $* -f Makefile put-cache + +SUBCLEANCACHE = $(addsuffix .cleancache,$(SUBDIRS)) +$(SUBCLEANCACHE): %.cleancache: + $(MAKE) -C $* -f Makefile clean-cache + +.PHONY: get-cache +get-cache: $(SUBGETCACHE) + +.PHONY: put-cache +put-cache: $(SUBPUTCACHE) + +.PHONY: clean-cache +clean-cache: $(SUBCLEANCACHE) diff --git a/fuel/build/f_isoroot/f_bootstrap/Makefile b/fuel/build/f_isoroot/f_bootstrap/Makefile index 7404319d9..a958aa2d6 100644 --- a/fuel/build/f_isoroot/f_bootstrap/Makefile +++ b/fuel/build/f_isoroot/f_bootstrap/Makefile @@ -27,3 +27,22 @@ clean: .PHONY: release release:all @cp -Rvp release/* ../release + +############################################################################# +# Cache operations - only used when building through ci/build.sh +############################################################################# + +# Clean local data related to caching - called prior to ordinary build +.PHONY: clean-cache +clean-cache: clean + @echo "clean-cache not implemented" + +# Try to download cache - called prior to ordinary build +.PHONY: get-cache +get-cache: + @echo "get-cache not implemented" + +# Store cache if not already stored - called after ordinary build +.PHONY: put-cache +put-cache: + @echo "put-cache not implemented" diff --git a/fuel/build/f_isoroot/f_kscfg/Makefile b/fuel/build/f_isoroot/f_kscfg/Makefile index 38305fd50..fe740418d 100644 --- a/fuel/build/f_isoroot/f_kscfg/Makefile +++ b/fuel/build/f_isoroot/f_kscfg/Makefile @@ -23,3 +23,22 @@ clean: .PHONY: release release: all @cp -Rvp release/* ../release + +############################################################################# +# Cache operations - only used when building through ci/build.sh +############################################################################# + +# Clean local data related to caching - called prior to ordinary build +.PHONY: clean-cache +clean-cache: clean + @echo "clean-cache not implemented" + +# Try to download cache - called prior to ordinary build +.PHONY: get-cache +get-cache: + @echo "get-cache not implemented" + +# Store cache if not already stored - called after ordinary build +.PHONY: put-cache +put-cache: + @echo "put-cache not implemented" diff --git a/fuel/build/f_isoroot/f_odlpluginbuild/Makefile b/fuel/build/f_isoroot/f_odlpluginbuild/Makefile index ce9cd73e1..bc925e087 100644 --- a/fuel/build/f_isoroot/f_odlpluginbuild/Makefile +++ b/fuel/build/f_isoroot/f_odlpluginbuild/Makefile @@ -8,9 +8,7 @@ ############################################################################## TOP := $(shell pwd) -DOCKNAME = fuelrepo -DOCKVERSION = 1.0 -ODL_BRANCH="juno/lithium-sr1" +ODL_BRANCH=juno/lithium-sr1 ODL_REPO="https://github.com/stackforge/fuel-plugin-opendaylight.git" .PHONY: all @@ -18,21 +16,57 @@ all: .odlbuild .PHONY: clean clean: - # Deliberately not cleaning nailgun directory to speed up multiple builds - @rm -f ../release/opnfv/opendaylight*.rpm + @rm -f .odlbuild ../release/opnfv/opendaylight*.rpm opendaylight*.rpm + @rm -f $(BUILD_BASE)/gitinfo_odlplugin.txt .PHONY: release release:.odlbuild @rm -f ../release/opnfv/opendaylight*.rpm @mkdir -p ../release/opnfv - @cp opendaylight*.rpm ../release/opnfv/ + @cp opendaylight*.rpm ../release/opnfv/ + cp gitinfo_odlplugin.txt $(BUILD_BASE) .odlbuild: rm -rf fuel-plugin-opendaylight sudo apt-get -y install build-essential ruby-dev rubygems-integration python-pip git rpm createrepo dpkg-dev sudo gem install fpm sudo pip install fuel-plugin-builder - git clone -b ${ODL_BRANCH} ${ODL_REPO} + git clone -b $(ODL_BRANCH) $(ODL_REPO) INCLUDE_DEPENDENCIES=true fpb --debug --build fuel-plugin-opendaylight/ mv fuel-plugin-opendaylight/opendaylight*.rpm . + $(REPOINFO) -r . > gitinfo_odlplugin.txt rm -rf fuel-plugin-opendaylight + touch .odlbuild + +############################################################################# +# Cache operations - only used when building through ci/build.sh +############################################################################# + + +# Create a unique hash to be used for getting and putting cache, based on: +# - The SHA1 hash of the HEAD on the plugin repo's $(ODL_BRANCH) +# - The contents of this Makefile +.cacheid: + git ls-remote --heads $(ODL_REPO) | grep $(ODL_BRANCH) > .cachedata + sha1sum Makefile >> .cachedata + cat .cachedata | $(CACHETOOL) getid > .cacheid + +# Clean local data related to caching - called prior to ordinary build +.PHONY: clean-cache +clean-cache: clean + rm -f .cachedata .cacheid + +# Try to download cache - called prior to ordinary build +.PHONY: get-cache +get-cache: .cacheid + @if $(CACHETOOL) check $(shell cat .cacheid); then \ + $(CACHETOOL) get $(shell cat .cacheid) | tar xf -;\ + else \ + echo "No cache item found for $(shell cat .cacheid)" ;\ + exit 0;\ + fi + +# Store cache if not already stored - called after ordinary build +.PHONY: put-cache +put-cache: .cacheid + @tar cf - .odlbuild opendaylight*.rpm gitinfo_odlplugin.txt | $(CACHETOOL) put $(shell cat .cacheid) diff --git a/fuel/build/f_isoroot/f_repobuild/Makefile b/fuel/build/f_isoroot/f_repobuild/Makefile index 03e4caecb..ad40d0b9a 100644 --- a/fuel/build/f_isoroot/f_repobuild/Makefile +++ b/fuel/build/f_isoroot/f_repobuild/Makefile @@ -10,17 +10,13 @@ SHELL := /bin/bash TOP := $(shell pwd) -DOCKNAME = fuelrepo -DOCKVERSION = 1.0 RSYNC_HOST := $(shell ./select_ubuntu_repo.sh) .PHONY: all -all: .nailgun +all: nailgun -.nailgun: - sudo apt-get update - sudo apt-get upgrade -y +nailgun: sudo apt-get install -y rsync python python-yaml dpkg-dev openssl rm -rf tmpiso tmpdir mkdir tmpiso @@ -37,16 +33,45 @@ all: .nailgun sudo su - -c /opt/fuel-createmirror-6.1/fuel-createmirror sudo chmod -R 755 /var/www/nailgun cp -Rp /var/www/nailgun . - touch .nailgun .PHONY: clean clean: - # Deliberately not cleaning nailgun directory to speed up multiple builds - @rm -rf ../release/opnfv/nailgun fuel-createmirror_6.1*.deb + @rm -rf ../release/opnfv/nailgun nailgun fuel-createmirror_6.1*.deb .PHONY: release -release:.nailgun +release:nailgun @rm -Rf ../release/opnfv/nailgun @mkdir -p ../release/opnfv @cp -Rp nailgun ../release/opnfv/nailgun +############################################################################# +# Cache operations - only used when building through ci/build.sh +############################################################################# + +# Create a unique hash to be used for getting and putting cache, based on: +# - Year and week (causing the cache to be rebuilt weekly) +# - The contents of this Makefile +.cacheid: + date +"Repocache %G%V" > .cachedata + sha1sum Makefile >> .cachedata + cat .cachedata | $(CACHETOOL) getid > .cacheid + +# Clean local data related to caching - called prior to ordinary build +.PHONY: clean-cache +clean-cache: clean + rm -f .cachedata .cacheid + +# Try to download cache - called prior to ordinary build +.PHONY: get-cache +get-cache: .cacheid + @if $(CACHETOOL) check $(shell cat .cacheid); then \ + $(CACHETOOL) get $(shell cat .cacheid) | tar xf -;\ + else \ + echo "No cache item found for $(shell cat .cacheid)" ;\ + exit 0;\ + fi + +# Store cache if not already stored - called after ordinary build +.PHONY: put-cache +put-cache: .cacheid + @tar cf - nailgun | $(CACHETOOL) put $(shell cat .cacheid) diff --git a/fuel/build/install/install.sh b/fuel/build/install/install.sh index f0bb84914..6155a4162 100755 --- a/fuel/build/install/install.sh +++ b/fuel/build/install/install.sh @@ -252,6 +252,10 @@ iso_modify_image () { cd $TOP/release/isoroot cp -Rvp . $DEST + + # Add all Git info files + sort $TOP/gitinfo*.txt > $DEST/gitinfo.txt + cp $DEST/gitinfo.txt $REPORTFILE } make_iso() { @@ -465,7 +469,7 @@ if [ $MODE = "iso" ]; then echo "Opening reportfile at $REPORTFILE" touch $REPORTFILE if [ ! -f $ORIGISO ]; then - echo "Can't find original MOS 5.1 iso at $ORIGISO" + echo "Can't find original iso at $ORIGISO" rm $CONF exit 1 fi diff --git a/fuel/build/repo_info.sh b/fuel/build/repo_info.sh new file mode 100755 index 000000000..347ede7f7 --- /dev/null +++ b/fuel/build/repo_info.sh @@ -0,0 +1,63 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +usage() { + cat <<EOF +Usage: `basename $0` [-r] <path> + + -r + Recursively list all repos found starting at <path> + -h + Display help text +EOF +} + +repoinfo() { + repotop=$(git -C $1 rev-parse --show-toplevel) + origin=$(git -C $repotop config --get remote.origin.url) + sha1=$(git -C $repotop rev-parse HEAD) + echo "$origin: $sha1" +} + + +if [ $# -eq 2 ]; then + case $1 in + -r) + RECURSE=1 + shift + ;; + -h) + usage + exit 0 + ;; + *) + echo "Error, argument $1 not known" >&2 + usage + exit 1 + esac +fi + +if [ $# -gt 1 ]; then + echo "Error, too many arguments" >&2 + usage + exit 1 +fi + +abspath=$(readlink -f $1) + +if [ -n "$RECURSE" ]; then + for dir in $(find $abspath -type d -name .git) + do + repoinfo $(readlink -f $dir/..) + done +else + repoinfo $abspath +fi diff --git a/fuel/ci/build.sh b/fuel/ci/build.sh index f8e164a76..62abba173 100755 --- a/fuel/ci/build.sh +++ b/fuel/ci/build.sh @@ -1,5 +1,4 @@ #!/bin/bash -set -e ############################################################################## # Copyright (c) 2015 Ericsson AB and others. # stefan.k.berg@ericsson.com @@ -10,56 +9,66 @@ set -e # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -trap 'echo "Exiting ..."; \ -if [ -f ${LOCK_FILE} ]; then \ - if [ $(cat ${LOCK_FILE}) -eq $$ ]; then \ - rm -f ${LOCK_FILE}; \ - fi; \ -fi;' EXIT - ############################################################################ # BEGIN of usage description # usage () { -cat << EOF -$0 Builds the Fuel@OPNFV stack +cat | more << EOF +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +`basename $0`: Builds the Fuel@OPNFV stack -usage: $0 [-s spec-file] [-c cache-URI] [-l log-file] [-f Flags] build-directory +usage: `basename $0` [-s spec-file] [-c cache-URI] [-l log-file] [-f flags] + [output-directory] OPTIONS: - -s spec-file ($BUILD_SPEC), define the build-spec file, default ../build/config.mk - -c cache base URI ($BUILD_CACHE_URI), specifies the base URI to a build cache to be used/updated - the name is automatically generated from the md5sum of the spec-file, http://, ftp://, file://[absolute path] suported. - - -l log-file ($BUILD_LOG), specifies the output log-file (stdout and stderr), if not specified logs are output to console as normal - -v version tag to be applied to the build result - -r alternative remote access method script/program. curl is default. - -t run small build-script unit test. - -T run large build-script unit test. - -f build flags ($BUILD_FLAGS): - o s: Do nothing, succeed - o f: Do nothing, fail - o t: run build unit tests - o i: run interactive (-t flag to docker run) - o P: Populate a new local cache and push it to the (-c cache-URI) cache artifactory if -c option is present, currently file://, http:// and ftp:// are supported - o d: Detatch - NOT YET SUPPORTED - - build-directory ($BUILD_DIR), specifies the directory for the output artifacts (.iso file). + -s spec-file (NOTE! DEPRECATED!) + define the build-spec file, default ../build/config.mk. The script only + verifies that the spec-file exists. + -c cache base URI + specifies the base URI to a build cache to be used/updated, supported + methods are http://, ftp:// and file:// + -l log-file + specifies the output log-file (stdout and stderr), if not specified + logs are output to console as normal + -v + version tag to be applied to the build result + -r + alternative remote access method script/program. curl is default. + -f flag[...] + build flags: + s: Do nothing, succeed + f: Do nothing, fail + D: Debug mode + P: Clear the local cache before building. This flag is only + valid if the "-c cache-URI" options has been specified and + and the method in the cache-URI is file:// (local cache). -h help, prints this help text + output-directory, specifies the directory for the output artifacts + (.iso file). If no output-directory is specified, the current path + when calling the script is used. + + Description: -build.sh builds opnfv .iso artifact. -To reduce build time it uses build cache on a local or remote location. The cache is rebuilt and uploaded if either of the below conditions are met: -1) The P(opulate) flag is set and the -c cache-base-URI is provided, if -c is not provided the cache will stay local. -2) If the cache is invalidated by one of the following conditions: - - The config spec md5sum does not compare to the md5sum for the spec which the cache was built. - - The git Commit-Id on the remote repos/HEAD defined in the spec file does not correspont with the Commit-Id for what the cache was built with. + +build.sh builds the opnfv .iso artifact. +To reduce build time it uses build caches on a local or remote location. A +cache is rebuilt and uploaded if either of the below conditions are met: +1) The P(opulate) flag is set and the -c cache-base-URI is provided and set + to the method file:// , if -c is + not provided the cache will stay local. +2) If a cache is invalidated by the make system - the exact logic is encoded + in the cache.mk of the different parts of the build. 3) A valid cache does not exist on the specified -c cache-base-URI. -The cache URI object name is fuel_cache-"md5sum(spec file)" +A cache has a blob (binary data) and a meta file in the format of: + <SHA1>.blob + <SHA1>.meta -Logging by default to console, but can be directed elsewhere with the -l option in which case both stdout and stderr is redirected to that destination. +Logging is by default to console, but can be directed elsewhere with the -l +option in which case both stdout and stderr is redirected to that destination. Built in unit testing of components is enabled by adding the t(est) flag. @@ -67,12 +76,15 @@ Return codes: - 0 Success! - 1-99 Unspecified build error - 100-199 Build system internal error (not build it self) - o 101 Build system instance busy + - 101 Build system instance busy - 200 Build failure Examples: -build -c http://opnfv.org/artifactory/fuel/cache -d ~/jenkins/genesis/fuel/ci/output -f ti -NOTE: At current the build scope is set to the git root of the repository, -d destination locations outside that scope will not work + build -c http://opnfv.org/artifactory/fuel/cache \ + -d ~/jenkins/genesis/fuel/ci/output -f ti + +NOTE: At current the build scope is set to the git root of the repository, -d + destination locations outside that scope will not work! EOF } # @@ -80,394 +92,178 @@ EOF ############################################################################ ############################################################################ -# Begin of string xor function -# -function xor() -{ - local res=(`echo "$1" | sed "s/../0x& /g"`) - shift 1 - while [[ "$1" ]]; do - local one=(`echo "$1" | sed "s/../0x& /g"`) - local count1=${#res[@]} - if [ $count1 -lt ${#one[@]} ] - then - count1=${#one[@]} - fi - for (( i = 0; i < $count1; i++ )) - do - res[$i]=$((${one[$i]:-0} ^ ${res[$i]:-0})) - done - shift 1 - done - printf "%02x" "${res[@]}" +# BEGIN of function error_exit + +error_exit() { + echo "$@" >&2 + exit 1 } + # -# END of string xor function +# END of function error_exit ############################################################################ + ############################################################################ -# BEGIN of variables to customize +# BEGIN of shorthand variables for internal use # -BUILD_BASE=$(readlink -e ../build/) +SCRIPT_DIR=$(readlink -f $(dirname ${BASH_SOURCE[0]})) +BUILD_BASE=$(readlink -e ${SCRIPT_DIR}/../build/) RESULT_DIR="${BUILD_BASE}/release" BUILD_SPEC="${BUILD_BASE}/config.mk" -CACHE_DIR="cache" LOCAL_CACHE_ARCH_NAME="fuel-cache" -REMOTE_ACCESS_METHD=curl -INCLUDE_DIR=../include # # END of variables to customize ############################################################################ ############################################################################ -# BEGIN of script assigned variables +# BEGIN of script assigned default variables # -SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) -LOCK_FILE="${SCRIPT_DIR}/.build.lck" -CACHE_TMP="${SCRIPT_DIR}/tmp" -TEST_SUCCEED=0 -TEST_FAIL=0 -UNIT_TEST=0 -UPDATE_CACHE=0 -POPULATE_CACHE=0 -RECURSIVE=0 -DETACH=0 -DEBUG=0 -INTEGRATION_TEST=0 -FULL_INTEGRATION_TEST=0 -INTERACTIVE=0 -BUILD_CACHE_URI= -BUILD_SPEC= -BUILD_DIR= -BUILD_LOG= -BUILD_VERSION= -MAKE_ARGS= -FUEL_GIT_SRC="$(make -f ../build/config.mk get-fuel-repo | cut -d " " -f1)" -FUEL_GIT_BRANCH="$(make -f ../build/config.mk get-fuel-repo | cut -d " " -f2)" -CACHE_MD5=$(md5sum ../build/cache.mk | cut -f1 -d " ") -CONFIG_MD5=$(md5sum ../build/config.mk | cut -f1 -d " ") -FUEL_COMMIT_ID=$(git ls-remote $FUEL_GIT_SRC -t $FUEL_GIT_BRANCH | cut -d $'\t' -f1) -REMOTE_CACHE_ARCH_HASH_TMP="$(xor $CACHE_MD5 $CONFIG_MD5)" -REMOTE_CACHE_ARCH_HASH="$(xor $REMOTE_CACHE_ARCH_HASH_TMP $FUEL_COMMIT_ID)" -REMOTE_CACHE_ARCH_NAME="fuel_cache-$REMOTE_CACHE_ARCH_HASH" +export CACHEBASE="file://$HOME/cache" +export CACHETRANSPORT="curl --silent" +CLEAR_CACHE=0 +MAKE_ARGS="" + # # END of script assigned variables ############################################################################ -############################################################################ -# BEGIN of include pragmas -# -source ${INCLUDE_DIR}/build.sh.debug -# -# END of include -############################################################################ +build() { + echo "CI build parameters:" + echo "SCRIPT_DIR = $SCRIPT_DIR" + echo "BUILD_BASE = $BUILD_BASE" + echo "RESULT_DIR = $RESULT_DIR" + echo "BUILD_SPEC = $BUILD_SPEC" + echo "LOCAL_CACHE_ARCH_NAME = $LOCAL_CACHE_ARCH_NAME" + echo "CLEAR_CACHE = $CLEAR_CACHE" + echo "DEBUG = $DEBUG" + echo "OUTPUT_DIR = $OUTPUT_DIR" + echo "BUILD_LOG = $BUILD_LOG" + echo "MAKE_ARGS = $MAKE_ARGS" + echo "CACHEBASE = $CACHEBASE" + echo "CACHETRANSPORT = $CACHETRANSPORT" + + + if [ "$CLEAR_CACHE" -eq 1 ]; then + echo $CACHEBASE | grep -q '^file://' $CACHE_BASE + if [ $? -ne 0 ]; then + error_exit "Can't clear a non-local cache!" + else + CACHEDIR=$(echo $CACHEBASE | sed 's;file://;;') + echo "Clearing local cache at $CACHEDIR..." + rm -rvf $CACHEDIR/* + fi + fi + + echo make ${MAKE_ARGS} cache + + cd ${BUILD_BASE} + if make ${MAKE_ARGS} cache; then + echo "Copying build result into $OUTPUT_DIR" + sort ${BUILD_BASE}/gitinfo*.txt > ${OUTPUT_DIR}/gitinfo.txt + cp ${RESULT_DIR}/*.iso ${OUTPUT_DIR} + cp ${RESULT_DIR}/*.iso.txt ${OUTPUT_DIR} + else + error_exit "Build failed" + fi +} ############################################################################ # BEGIN of main # -while getopts "s:c:v:f:l:r:RtTh" OPTION +while getopts "s:c:l:v:f:r:f:h" OPTION do case $OPTION in - h) - usage - rc=0 - exit $rc - ;; - s) BUILD_SPEC=${OPTARG} + if [ ! -f ${BUILD_SPEC} ]; then + echo "spec file does not exist: $BUILD_SPEC - exiting ...." + exit 100 + fi ;; - c) - BUILD_CACHE_URI=${OPTARG} + # This value is used by cache.sh + export CACHEBASE=${OPTARG} ;; - l) - BUILD_LOG=${OPTARG} + BUILD_LOG=$(readlink -f ${OPTARG}) ;; - v) - BUILD_VERSION=${OPTARG} - ;; - - f) - BUILD_FLAGS=${OPTARG} + MAKE_ARGS+="REVSTATE=${OPTARG}" ;; - - r) REMOTE_ACCESS_METHD=${OPTARG} - ;; - - R) - RECURSIVE=1 - ;; - - t) - INTEGRATION_TEST=1 - ;; - - T) - INTEGRATION_TEST=1 - FULL_INTEGRATION_TEST=1 + r) + # This value is used by cache.sh + export CACHETRANSPORT=${OPTARG} ;; - - *) - echo "${OPTION} is not a valid argument" - rc=100 - exit $rc - ;; - esac -done - -if [ -z $BUILD_DIR ]; then - BUILD_DIR=$(echo $@ | cut -d ' ' -f ${OPTIND}) -fi - -for ((i=0; i<${#BUILD_FLAGS};i++)); do - case ${BUILD_FLAGS:$i:1} in - s) + h) + usage rc=0 exit $rc ;; - f) - rc=1 - exit $rc - ;; - - t) - UNIT_TEST=1 - ;; - - i) - INTERACTIVE=1 - ;; - - P) - POPULATE_CACHE=1 - ;; - - d) - DETACH=1 - echo "Detach is not yet supported - exiting ...." - rc=100 - exit $rc - ;; - - D) - DEBUG=1 + BUILD_FLAGS=${OPTARG} + for ((i=0; i<${#BUILD_FLAGS};i++)); do + case ${BUILD_FLAGS:$i:1} in + s) + exit 0 + ;; + + f) + exit 1 + ;; + + P) + CLEAR_CACHE=1 + ;; + + D) + DEBUG=1 + ;; + + *) + error_exit "${BUILD_FLAGS:$i:1} is not a valid build flag - exiting ...." + ;; + esac + done ;; *) - echo "${BUILD_FLAGS:$i:1} is not a valid build flag - exiting ...." + echo "${OPTION} is not a valid argument" rc=100 exit $rc ;; esac done -if [ ${INTEGRATION_TEST} -eq 1 ]; then - integration-test - rc=0 - exit $rc -fi - -if [ ! -f ${BUILD_SPEC} ]; then - echo "spec file does not exist: $BUILD_SPEC - exiting ...." - rc=100 - exit $rc -fi - -if [ -z ${BUILD_DIR} ]; then - echo "Missing build directory - exiting ...." - rc=100 - exit $rc -fi - -if [ ! -z ${BUILD_LOG} ]; then - if [[ ${RECURSIVE} -ne 1 ]]; then - set +e - eval $0 -R $@ > ${BUILD_LOG} 2>&1 - rc=$? - set -e - if [ $rc -ne 0]; then - exit $rc - fi - fi -fi - -if [ ${TEST_SUCCEED} -eq 1 ]; then - sleep 1 - rc=0 - exit $rc -fi - -if [ ${TEST_FAIL} -eq 1 ]; then - sleep 1 - rc=1 - exit $rc -fi - -if [ -e ${LOCK_FILE} ]; then - echo "A build job is already running, exiting....." - rc=101 - exit $rc -fi - -echo $$ > ${LOCK_FILE} - -if [ ! -z ${BUILD_CACHE_URI} ]; then - if [ ${POPULATE_CACHE} -ne 1 ]; then - rm -rf ${CACHE_TMP}/cache - mkdir -p ${CACHE_TMP}/cache - echo "Downloading cache archive ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME} ..." - set +e - ${REMOTE_ACCESS_METHD} -o ${CACHE_TMP}/cache/${LOCAL_CACHE_ARCH_NAME}.tgz ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME}.tgz - rc=$? - set -e - if [ $rc -ne 0 ]; then - echo "Remote cache does not exist, or is not accessible - a new cache will be built ..." - POPULATE_CACHE=1 - else - echo "Unpacking cache archive ..." - set +e - tar -C ${CACHE_TMP}/cache -xvf ${CACHE_TMP}/cache/${LOCAL_CACHE_ARCH_NAME}.tgz - rc=$? - set -e - if [ $rc -ne 0 ]; then - echo "WARNING: The cache seems to be corrupt or has trailing garbage, will try to use brute force" - echo "Info about the cache below:" - set +e - file ${CACHE_TMP}/cache/${LOCAL_CACHE_ARCH_NAME}.tgz - tar -C ${CACHE_TMP}/cache -tvf ${CACHE_TMP}/cache/${LOCAL_CACHE_ARCH_NAME}.tgz - set -e - echo "Current time is: `date`" - set +e - pushd ${CACHE_TMP}/cache - gunzip -dcq ${CACHE_TMP}/cache/${LOCAL_CACHE_ARCH_NAME}.tgz | tar -xvf - - rc=$? - set -e - popd - if [ $rc -ne 0 ]; then - echo "ERROR: Not able to resolve the cache corruption" - POPULATE_CACHE=1 - else - echo "The chache corruption was resolved" - cp ${CACHE_TMP}/cache/cache/.versions ${BUILD_BASE}/. - set +e - make -C ${BUILD_BASE} validate-cache; - rc=$? - set -e - if [ $rc -ne 0 ]; then - echo "Cache invalid - a new cache will be built " - POPULATE_CACHE=1 - else - echo "Cache is up to date and will be used" - cp -rf ${CACHE_TMP}/cache/cache/. ${BUILD_BASE} - fi - fi - else - echo "Cache archive is intact" - cp ${CACHE_TMP}/cache/cache/.versions ${BUILD_BASE}/. - set +e - make -C ${BUILD_BASE} validate-cache; - rc=$? - set -e - - if [ $rc -ne 0 ]; then - echo "Cache invalid - a new cache will be built " - POPULATE_CACHE=1 - else - echo "Cache is up to date and will be used" - cp -rf ${CACHE_TMP}/cache/cache/. ${BUILD_BASE} - fi - fi - rm -rf ${CACHE_TMP}/cache - fi - fi -fi - -if [ ${POPULATE_CACHE} -eq 1 ]; then - if [ ${DEBUG} -eq 0 ]; then - set +e - cd ${BUILD_BASE} && make clean - rc=$? - set -e - if [ $rc -ne 0 ]; then - echo "Build - make clean failed, exiting ..." - rc=100 - exit $rc - fi - fi -fi - -if [ ! -z ${BUILD_VERSION} ]; then - MAKE_ARGS+="REVSTATE=${BUILD_VERSION} " -fi - -if [ ${UNIT_TEST} -eq 1 ]; then - MAKE_ARGS+="UNIT_TEST=TRUE " +# Get output directory +shift $[$OPTIND - 1] +case $# in + 0) + # No directory on command line + OUTPUT_DIR=$(pwd) + ;; + 1) + # Directory on command line + OUTPUT_DIR=$(readlink -f $1) + ;; + *) + error_exit "Too many arguments" + ;; +esac +mkdir -p $OUTPUT_DIR || error_exit "Could not access output directory $OUTPUT_DIR" + + +if [ -n "${BUILD_LOG}" ]; then + touch ${BUILD_LOG} || error_exit "Could not write to log file ${BUILD_LOG}" + build 2>&1 | tee ${BUILD_LOG} else - MAKE_ARGS+="UNIT_TEST=FALSE " + build fi -if [ ${INTERACTIVE} -eq 1 ]; then - MAKE_ARGS+="INTERACTIVE=TRUE " -else - MAKE_ARGS+="INTERACTIVE=FALSE " -fi - -MAKE_ARGS+=all - -if [ ${DEBUG} -eq 0 ]; then - set +e - cd ${BUILD_BASE} && make ${MAKE_ARGS} - rc=$? - set -e - if [ $rc -gt 0 ]; then - echo "Build: make all failed, exiting ..." - rc=200 - exit $rc - fi -else - debug_make -fi -set +e -make -C ${BUILD_BASE} prepare-cache rc=$? -set -e +exit $rc -if [ $rc -gt 0 ]; then - echo "Build: make prepare-cache failed - exiting ..." - rc=100 - exit $rc -fi -echo "Copying built OPNFV .iso file to target directory ${BUILD_DIR} ..." -rm -rf ${BUILD_DIR} -mkdir -p ${BUILD_DIR} -cp ${BUILD_BASE}/.versions ${BUILD_DIR} -cp ${RESULT_DIR}/*.iso* ${BUILD_DIR} - -if [ $POPULATE_CACHE -eq 1 ]; then - if [ ! -z ${BUILD_CACHE_URI} ]; then - echo "Building cache ..." - tar --dereference -C ${BUILD_BASE} -caf ${BUILD_BASE}/${LOCAL_CACHE_ARCH_NAME}.tgz ${CACHE_DIR} - set +e - tar -C ${CACHE_TMP}/cache -tvf ${BUILD_BASE}/${LOCAL_CACHE_ARCH_NAME}.tgz - rc=$? - set -e - if [ $rc -ne 0 ]; then - echo "WARNING the cache archive generated seems to be corrupt, or containing trailing garbage" - else - echo "The Cache archive build is intact" - fi - echo "Uploading cache ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME}" - ${REMOTE_ACCESS_METHD} -T ${BUILD_BASE}/${LOCAL_CACHE_ARCH_NAME}.tgz ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME}.tgz - rm ${BUILD_BASE}/${LOCAL_CACHE_ARCH_NAME}.tgz - fi -fi -echo "Success!!!" -exit 0 # # END of main ############################################################################ diff --git a/fuel/include/build.sh.debug b/fuel/include/build.sh.debug deleted file mode 100644 index b6594cb16..000000000 --- a/fuel/include/build.sh.debug +++ /dev/null @@ -1,355 +0,0 @@ -############################################################################## -# Copyright (c) 2015 Ericsson AB and others. -# stefan.k.berg@ericsson.com -# jonas.bjurel@ericsson.com -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -############################################################################ -# BEGIN of build script integration test -# -integration-test () { - -test_clean () { - make -C ${BUILD_BASE} clean - test_cache_clean - test_output_clean -} - -test_cache_clean () { - rm -rf ${TEST_BUILD_CACHE_URI_DIR} -} - -test_output_clean () { - rm -rf ${TEST_BUILD_RESULT_DIR} -} - -test_prepare () { - test_clean - test_cache_clean - test_output_clean - mkdir -p ${TEST_BUILD_CACHE_URI_DIR} -} - -test_cache_check () { - if [ ! -f ${BUILD_BASE}/fuel-6.0.1.iso ] || \ - [ ! -f ${BUILD_BASE}/.versions ] || \ - [ ! -f ${BUILD_BASE}/opendaylight/.odl-build-history ]; \ - [ ! -f ${BUILD_BASE}/opendaylight/.odl-build.log ]; then - echo "Cache not present in the build system" - echo "TEST FAILED" - exit $rc - fi - - if [ ! -f ${TEST_BUILD_CACHE_URI_DIR}/${REMOTE_CACHE_ARCH_NAME}.tgz ]; then - echo "Cache archive not present" - echo "TEST FAILED" - exit $rc - fi - - if tar -tf ${TEST_BUILD_CACHE_URI_DIR}/${REMOTE_CACHE_ARCH_NAME}.tgz ${CACHE_DIR}/.mark > /dev/null 2>&1; then - echo "Cache archive is invalidated" - echo "TEST FAILED" - exit $rc - fi -} - -test_output_check () { - if [ ! -f ${TEST_BUILD_RESULT_DIR}/*.iso ] || \ - [ ! -f ${TEST_BUILD_RESULT_DIR}/*.txt ] || \ - [ ! -f ${TEST_BUILD_RESULT_DIR}/.versions ]; then - echo "Build output not produced by the build system" - echo "TEST FAILED" - exit $rc - fi -} - -test_invalidate_cache () { - tar -C ${TEST_BUILD_CACHE_URI_DIR} -xf ${TEST_BUILD_CACHE_URI_DIR}/${REMOTE_CACHE_ARCH_NAME}.tgz - echo "cache.mk 123" > ${TEST_BUILD_CACHE_URI_DIR}/cache/.versions - echo "config.mk 123" >> ${TEST_BUILD_CACHE_URI_DIR}/cache/.versions - echo "fuel 123" >> ${TEST_BUILD_CACHE_URI_DIR}/cache/.versions - echo "odl 123" >> ${TEST_BUILD_CACHE_URI_DIR}/cache/.versions - - echo "This file marks the invalid cache" > ${TEST_BUILD_CACHE_URI_DIR}/cache/.mark - rm -f ${TEST_BUILD_CACHE_URI_DIR}/${REMOTE_CACHE_ARCH_NAME}.tgz - tar --dereference -C ${TEST_BUILD_CACHE_URI_DIR} -caf ${TEST_BUILD_CACHE_URI_DIR}/${REMOTE_CACHE_ARCH_NAME}.tgz ${CACHE_DIR} -} - -##### Always succeed integration test #### -build_tc1 () { - echo "TEST - $0 - ALWAYS SUCCEED" - set +e - $0 -f s tmp/output - rc=$? - set -e - if [ $rc -ne 0 ]; then - echo "TEST FAILED" - rc=151 - exit $rc - fi -} - -##### Always fail integration test (TC2) #### -build_tc2 () { - echo "TEST - $0 - ALWAYS FAIL" - set +e - $0 -f f tmp/output - rc=$? - set -e - if [ $rc -eq 0 ]; then - echo "TEST FAILED" - rc=152 - exit $rc - fi -} - -##### Fake build and populate dummy cache - integration test (TC3) #### -build_tc3 () { - echo "TEST - $0 - FAKE BUILD AND POPULATE DUMMY CACHE" - rm -rf tmp - mkdir -p ${TEST_BUILD_CACHE_URI_DIR} - set +e - $0 -c ${TEST_BUILD_CACHE_URI} -f PD ${TEST_BUILD_RESULT_DIR} - rc=$? - set -e - if [ $rc -ne 0 ]; then - echo "Build failed" - echo "TEST FAILED" - rc=152 - exit $rc - fi - - rc=153 - test_output_check - - rc=154 - test_cache_check -} - -##### Fake build usng dummy cache integration test (TC4) #### -build_tc4 () { - echo "TEST - $0 - FAKE BUILD USING DUMMY CACHE" - set +e - $0 -c ${TEST_BUILD_CACHE_URI} -f D ${TEST_BUILD_RESULT_DIR} - rc=$? - set -e - if [ $rc -ne 0 ]; then - echo "Build failed" - echo "TEST FAILED" - rc=155 - exit $rc - fi - - rc=156 - test_output_check - - rc=157 - test_cache_check -} - -#### Fake build and repopulate cache because cache non existant (TC5) #### -build_tc5 () { - echo "TEST - $0 - FAKE BUILD AND REPOPULATE CACHE BECAUSE NON EXISTANT" - rm -rf ${TEST_BUILD_CACHE_URI_DIR}/* - set +e - $0 -c ${TEST_BUILD_CACHE_URI} -f D ${TEST_BUILD_RESULT_DIR} - rc=$? - set -e - if [ $rc -ne 0 ]; then - echo "Build failed" - echo "TEST FAILED" - rc=158 - exit $rc - fi - - rc=159 - test_output_check - - rc=160 - test_cache_check -} - -#### Fake build and repopulate cache because cache is ivalidated (TC6) #### -build_tc6 () { - echo "TEST - $0 - FAKE BUILD AND REPOPULATE DUE TO CACHE INVALIDATION" - test_invalidate_cache - set +e - $0 -c ${TEST_BUILD_CACHE_URI} -f D ${TEST_BUILD_RESULT_DIR} - rc=$? - set -e - if [ $rc -ne 0 ]; then - echo "Build failed" - echo "TEST FAILED" - rc=161 - exit $rc - fi - - rc=162 - test_output_check - - rc=163 - test_cache_check -} - -#### Real build and populate cache (TC7) #### -build_tc7 () { - echo "TEST - $0 - FULL BUILD AND POPULATE CACHE" - test_prepare - BUILD_START=$(date +%s) - set +e - $0 -c ${TEST_BUILD_CACHE_URI} -f Pi ${TEST_BUILD_RESULT_DIR} - rc=$? - set -e - if [ $rc -ne 0 ]; then - echo "Build failed" - echo "TEST FAILED" - rc=164 - exit $rc - fi - - BUILD_TIME=$(($(date +%s)-${BUILD_START})) - rc=165 - test_output_check - rc=166 - test_cache_check -} - -#### Real build using cache from TC7 (TC8) #### -build_tc8 () { - echo "TEST - $0 - INCREMENTAL BUILD AND USING CACHE" - BUILD_START=$(date +%s) - set +e - $0 -c ${TEST_BUILD_CACHE_URI} -f i ${TEST_BUILD_RESULT_DIR} - rc=$? - set -e - if [ $rc -ne 0 ]; then - echo "Build failed" - echo "TEST FAILED" - rc=167 - exit $rc - fi - BUILD_TIME=$(($(date +%s)-${BUILD_START})) - test_output_check - rc=168 - test_cache_check - rc=169 -} - -TEST_BUILD_ARTIFACT_DIR=${BUILD_BASE}/test -TEST_BUILD_CACHE_URI_DIR=${TEST_BUILD_ARTIFACT_DIR} -TEST_BUILD_CACHE_URI=file://${TEST_BUILD_CACHE_URI_DIR} -TEST_BUILD_RESULT_DIR=${BUILD_BASE}/tmp/output - -TEST_START=$(date +%s) -# Run test case 1: Always succeed integration test -echo ========================================================================== -echo ========================= RUNNING TEST CASE 1 ============================ -echo ========================================================================== -test_prepare -build_tc1 - -# Run test case 2: Always fail integration test -echo ========================================================================== -echo ========================= RUNNING TEST CASE 2 ============================ -echo ========================================================================== -test_prepare -build_tc2 - -# Run test case 3: Fake build and populate a build cach -echo ========================================================================== -echo ========================= RUNNING TEST CASE 3 ============================ -echo ========================================================================== -test_prepare -build_tc3 - -# Run test case 4: Fake build using the build cache produced in TC 3 -echo ========================================================================== -echo ========================= RUNNING TEST CASE 4 ============================ -echo ========================================================================== -build_tc4 - -# Run test case 5: Fake build and repopulate the build cache as there is no existing one -echo ========================================================================== -echo ========================= RUNNING TEST CASE 5 ============================ -echo ========================================================================== -test_prepare -build_tc5 - -# Run test case 6: Fake build and repopulate the build cache as there the current one is invalidated/outdated -echo ========================================================================== -echo ========================= RUNNING TEST CASE 6 ============================ -echo ========================================================================== -test_invalidate_cache -build_tc4 - -# Run test case 7: Build from upstream repos and populate the build cache -if [ ${FULL_INTEGRATION_TEST} -eq 1 ]; then -echo ========================================================================== -echo ========================= RUNNING TEST CASE 7 ============================ -echo ========================================================================== - test_prepare - build_tc7 - POPULATE_CACHE_TIME=${BUILD_TIME} -fi - -# Run test case 8: Build from cache produced in TC7 -if [ ${FULL_INTEGRATION_TEST} -eq 1 ]; then -echo ========================================================================== -echo ========================= RUNNING TEST CASE 8 ============================ -echo ========================================================================== - build_tc8 - PARTIAL_BUILD_TIME=${BUILD_TIME} -fi - -# Run test case 9: Build and repopulate the cache as the current one is invalidated/not up to date -if [ ${FULL_INTEGRATION_TEST} -eq 1 ]; then -echo ========================================================================== -echo ========================= RUNNING TEST CASE 9 ============================ -echo ========================================================================== - echo "TEST - $0 - REPOPULATING CACHE DUE TO CACHE INVALIDATION" - test_invalidate_cache - build_tc8 -fi - -TEST_TIME=$(($(date +%s)-${TEST_START})) - -rm -rf tmp -echo "All tests passed!, test duration was ${TEST_TIME} seconds" -if [ ${FULL_INTEGRATION_TEST} -eq 1 ]; then - echo "Full build took ${POPULATE_CACHE_TIME} second" - echo "Incremental build took ${PARTIAL_BUILD_TIME} seconds" -fi -rc=0 -exit $rc -} -# -# END of build script integration test -############################################################################ - -############################################################################ -# BEGIN of debug_make - a fake make initiated with -D for debugging -# -debug_make () { - make -C ${BUILD_BASE} clean - echo "This is a fake debug fuel .iso image" > ${BUILD_BASE}/fuel-6.0.1.iso - - echo "This is a fake debug versions file" > ${BUILD_BASE}/.versions - echo "cache.mk" $(md5sum ${BUILD_BASE}/cache.mk | cut -f1 -d " ") >> ${BUILD_BASE}/.versions - echo "config.mk" $(md5sum ${BUILD_BASE}/config.mk | cut -f1 -d " ") >> ${BUILD_BASE}/.versions - echo "fuel" $(git ls-remote $(make -C ${BUILD_BASE} get-fuel-repo | grep https) | awk '{print $(NF-1)}') >> ${BUILD_BASE}/.versions - echo "odl" $(git ls-remote $(make -C ${BUILD_BASE} get-odl-repo | grep https) | awk '{print $(NF-1)}') >> ${BUILD_BASE}/.versions - - rm -rf ${BUILD_BASE}/release - mkdir ${BUILD_BASE}/release - echo "This is a fake debug OPNFV .iso image" > ${BUILD_BASE}/release/fake-debug.iso - echo "This is a fake debug OPNFV .iso.txt message" > ${BUILD_BASE}/release/fake-debug.iso.txt - echo "This a fake debug odl build history" > ${BUILD_BASE}/opendaylight/.odl-build-history - echo "This a fake debug odl build log" > ${BUILD_BASE}/opendaylight/.odl-build.log -} -# -# END of debug_make -############################################################################ |