build: ISO refactor, use docker, enable cache

Previously, Armband Makefile used to call Fuel@OPNFV Makefile
target <iso>, instead of <all>, which resulted in bypassing
using Docker for ISO building completely.

Switch to <all> Makefile target, hence bringing back Docker
as a build container.

This change justifies moving make variables to a separate spec
file, armband.mk.

While at it, enable caching of builds.
This requires Jenkins bash scripts to be updated for the ISO
build job, done in [1].

JIRA: ARMBAND-56

[1] https://gerrit.opnfv.org/gerrit/#/c/18453/

CHANGES:
- all: release (was all: build)
- Common make targets now depend on submodules-init;
- clean-build and clean-docker make targets have been replaced by:
  make clean, make deepclean;

FIXME: These issues should be fixed later:
- (optional) trim local repo paths from gitinfo files
- only account for armband submodules in cache fingerprints;
- identify size mismatch culprit

Change-Id: I7d73409be44983fa21d4c09d2e07b87a33231e03
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>

1 files changed, 95 insertions, 94 deletions

diff --git a/Makefile b/Makefile
index df14ef68..519bce64 100644
--- a/Makefile
+++ b/Makefile
@@ -1,115 +1,116 @@
-root:=$(shell pwd)
+##############################################################################
+# Copyright (c) 2016 Cavium
+# Copyright (c) 2016 Enea AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+# NOTE: Armband patching consists of:
+# - clone upstream repositories to git submodules;
+# - checkout submodule commits where set in Fuel@OPNFV's config.mk
+#   (e.g. FUEL_ASTUTE_COMMIT=...);
+# - tag each submodule (before patching) with "armband-workbench-root";
+# - apply Armband patches for each submodule;
+# - tag each submodule (after patching) with "armband-workbench";
+# - pass updated repository info to Fuel@OPNFV build system
+#   (e.g. FUEL_ASTUTE_COMMIT=HEAD) via armband.mk;
+
+# NOTE: Long-term goals (Armband repo should merge with Fuel@OPNFV):
+# - all build related changes should affect Fuel@OPNFV, NOT Armband;
+# - Armband make/build system should only handle patching,
+#   including eventual build related patching of Fuel@OPNFV,
+#   and then invoke Fuel@OPNFV's build system;
+# - Fuel@OPNFV is made aware of an Armband type build by passing
+#   the "ARMBAND_BASE" env var;
+
+# Fist, inherit Fuel submodule commit references from Fuel@OPNFV
+# using "config.mk" as a make target that links to Fuel's config.mk.
+# Some values will be overriden at Fuel ISO build time by armband.mk.
 include config.mk
 
-all: build
+export ARMBAND_BASE  := $(shell pwd)
+export OPNFV_GIT_SHA := $(shell git rev-parse HEAD)
+export REVSTATE
+
+# Prepare for future directory re-layout when merging with Fuel@OPNFV
+PATCH_DIR  := ${ARMBAND_BASE}/patches
+SUBMOD_DIR := ${ARMBAND_BASE}/upstream
+FUEL_BASE  := ${SUBMOD_DIR}/fuel
+
+all: release
 
+# Use config.mk & clean_cache.sh from Fuel@OPNFV
 config.mk: submodules-init
-	@ln -s -f ${root}/upstream/fuel/build/config.mk ${root}/config.mk
+	@ln -sf ${FUEL_BASE}/build/config.mk ${ARMBAND_BASE}/config.mk
+	@ln -sf ${FUEL_BASE}/ci/clean_cache.sh ${ARMBAND_BASE}/ci/clean_cache.sh
 
-.PHONY: submodules-init submodules-clean
+# Fetch & update git submodules, checkout remote HEAD
+.PHONY: submodules-init
 submodules-init:
-	@if [ ! -d ${root}/upstream/fuel/build ]; then \
+	@if [ ! -d ${FUEL_BASE}/build ]; then \
 		git submodule -q init; \
 		git submodule -q sync; \
 		git submodule update --remote; \
 	fi
 
-# Cleans any changes made to submodules
-submodules-clean:
-	@git submodule -q foreach \
-		'git am --abort; \
-		git checkout -q armband-workbench-root; \
-		git branch -q -D armband-workbench; \
-		git tag -d armband-workbench-root; \
-		git reset -q --hard HEAD; git clean -xdff'
+# Clean any changes made to submodules, checkout Armband root commit
+.PHONY: submodules-clean
+submodules-clean: submodules-init
+	@git submodule -q foreach ' \
+		git am -q --abort 2>/dev/null; \
+		git checkout -q armband-workbench-root 2>/dev/null; \
+		git branch -q -D armband-workbench 2>/dev/null; \
+		git tag -d armband-workbench-root 2>/dev/null; \
+		git reset -q --hard HEAD; \
+		git clean -xdff'
 
-.PHONY: patches-export patches-import
 # Generate patches from submodules
-patches-export:
-	@git submodule -q foreach 'mkdir -p ${root}/patches/$$name'
-	@git submodule -q foreach 'git format-patch \
-		-o ${root}/patches/$$name -N armband-workbench-root \
-	    --no-signature'
-	@find ${root}/patches -name '*.patch' -exec sed -i -e '1d' {} \;
+.PHONY: patches-export
+patches-export: submodules-init
+	@git submodule -q foreach ' \
+		mkdir -p ${PATCH_DIR}/$$name; \
+		git format-patch --no-signature \
+			-o ${PATCH_DIR}/$$name -N armband-workbench-root'
+	@find ${PATCH_DIR} -name '*.patch' -exec sed -i -e '1d' {} \;
 
-# apply patches from patches/* to respective submodules
-patches-import:
-	@cd ${root}/upstream/fuel-agent && \
-		git checkout -q ${FUEL_AGENT_COMMIT}
-	@cd ${root}/upstream/fuel-astute && \
-		git checkout -q ${ASTUTE_COMMIT}
-	@cd ${root}/upstream/fuel-library && \
-		git checkout -q ${FUELLIB_COMMIT}
-	@cd ${root}/upstream/fuel-mirror && \
-		git checkout -q ${FUEL_MIRROR_COMMIT}
-	@cd ${root}/upstream/fuel-nailgun-agent && \
-		git checkout -q ${FUEL_NAILGUN_AGENT_COMMIT}
-	@cd ${root}/upstream/fuel-web && \
-		git checkout -q ${NAILGUN_COMMIT}
-	@git submodule -q foreach 'mkdir -p ${root}/patches/$$name'
-	@git submodule -q foreach 'git tag armband-workbench-root'
-	@git submodule -q foreach 'git checkout -q -b armband-workbench'
-	@git submodule -q foreach \
-		'if [ ! -z "$$(ls ${root}/patches/$$name/)" ]; then \
+# Apply patches from patches/* to respective submodules
+# For repos pinned in Fuel@OPNFV's config.mk, checkout pinned commit first
+.PHONY: patches-import
+patches-import: submodules-init
+	@git -C ${FUEL_BASE} checkout -q master
+	@git -C ${SUBMOD_DIR}/fuel-agent checkout -q ${FUEL_AGENT_COMMIT}
+	@git -C ${SUBMOD_DIR}/fuel-astute checkout -q ${ASTUTE_COMMIT}
+	@git -C ${SUBMOD_DIR}/fuel-library checkout -q ${FUELLIB_COMMIT}
+	@git -C ${SUBMOD_DIR}/fuel-mirror checkout -q ${FUEL_MIRROR_COMMIT}
+	@git -C ${SUBMOD_DIR}/fuel-nailgun-agent \
+		checkout -q ${FUEL_NAILGUN_AGENT_COMMIT}
+	@git -C ${SUBMOD_DIR}/fuel-web checkout -q ${NAILGUN_COMMIT}
+	@git submodule -q foreach ' \
+		mkdir -p ${PATCH_DIR}/$$name; \
+		git tag armband-workbench-root; \
+		git checkout -q -b armband-workbench; \
+		if [ ! -z "$$(ls ${PATCH_DIR}/$$name/)" ]; then \
 			echo "-- patching $$name"; \
-			git am ${root}/patches/$$name/*.patch; \
+			git am --whitespace=nowarn \
+				--committer-date-is-author-date \
+				${PATCH_DIR}/$$name/*.patch; \
 		fi'
-clean-docker:
-	@if [ -d ${root}/upstream/fuel/build ]; then \
-		sudo make -C ${root}/upstream/fuel/build deepclean; \
-	fi
-	@for container in $(shell sudo docker ps -a -q); do \
-		sudo docker rm -f -v $${container}; \
-	done
-	@for image in $(shell sudo docker images -q); do \
-		sudo docker rmi -f $${image}; \
-	done
-
-clean-build:
-	sudo rm -rf /tmp/fuel-main
-	git -C ${root}/upstream/fuel reset --hard HEAD
-	git -C ${root}/upstream/fuel clean -xdff
 
-release: export LC_ALL=en_US.UTF-8
-release: submodules-clean clean-docker clean-build submodules-init patches-import build
+# Pass down clean/deepclean/build to Fuel@OPNFV
+.PHONY: clean
+clean: submodules-init
+	$(MAKE) -e --no-print-directory -C ${FUEL_BASE}/build clean
 
-ifneq ($(REVSTATE),)
-    EXTRA_PARAMS="REVSTATE=$(REVSTATE)"
-endif
+.PHONY: deepclean
+deepclean: submodules-init
+	$(MAKE) -e --no-print-directory -C ${FUEL_BASE}/build deepclean
 
+.PHONY: build
 build:
-	cd ${root}/upstream/fuel/build && \
-		make \
-			BUILD_FUEL_PLUGINS="f_odlpluginbuild f_bgpvpn-pluginbuild" \
-			UBUNTU_ARCH="amd64 arm64" \
-			PRODNO="OPNFV_A_FUEL" \
-			OPNFV_GIT_SHA=$(shell git rev-parse HEAD) \
-			ASTUTE_REPO=${root}/upstream/fuel-astute \
-			ASTUTE_COMMIT=HEAD \
-			NAILGUN_REPO=${root}/upstream/fuel-web \
-			NAILGUN_COMMIT=HEAD \
-			FUEL_AGENT_REPO=${root}/upstream/fuel-agent \
-			FUEL_AGENT_COMMIT=HEAD \
-			FUEL_NAILGUN_AGENT_REPO=${root}/upstream/fuel-nailgun-agent \
-			FUEL_NAILGUN_AGENT_COMMIT=HEAD \
-			FUEL_MIRROR_REPO=${root}/upstream/fuel-mirror \
-			FUEL_MIRROR_COMMIT=HEAD \
-			FUELLIB_REPO=${root}/upstream/fuel-library \
-			FUELLIB_COMMIT=HEAD \
-			ODL_REPO=${root}/upstream/fuel-plugin-opendaylight \
-			ODL_BRANCH=armband-workbench \
-			ODL_CHANGE= \
-			OPNFV_QUAGGE_PACKAGING_REPO="https://github.com/alexandruavadanii/opnfv-quagga-packaging" \
-			OVS_NSH_DPDK_REPO=${root}/upstream/fuel-plugin-ovs \
-			OVS_NSH_DPDK_BRANCH=HEAD \
-			VSPERF_REPO=${root}/upstream/vswitchperf \
-			VSPERF_BRANCH=armband-workbench \
-			VSPERF_CHANGE= \
-			YARDSTICK_REPO=${root}/upstream/yardstick \
-			YARDSTICK_BRANCH=armband-workbench \
-			YARDSTICK_CHANGE= \
-			EXTRA_RPM_REPOS="armband,http://linux.enea.com/mos-repos/centos/mos9.0-centos7/armband/x86_64,10" \
-			MIRROR_MOS_UBUNTU=linux.enea.com \
-			$(EXTRA_PARAMS) \
-			iso 2>&1 | tee ${root}/build.log
+	$(MAKE) -e --no-print-directory -C ${FUEL_BASE}/build all
 
+.PHONY: release
+release: export LC_ALL=en_US.UTF-8
+release: submodules-clean patches-import build