From ec307077e542fd290701a871fedc7f58db37712c Mon Sep 17 00:00:00 2001 From: Alexandru Avadanii Date: Tue, 9 Aug 2016 18:40:22 +0200 Subject: build: ISO refactor, use docker, enable cache Previously, Armband Makefile used to call Fuel@OPNFV Makefile target , instead of , which resulted in bypassing using Docker for ISO building completely. Switch to Makefile target, hence bringing back Docker as a build container. This change justifies moving make variables to a separate spec file, armband.mk. While at it, enable caching of builds. This requires Jenkins bash scripts to be updated for the ISO build job, done in [1]. JIRA: ARMBAND-56 [1] https://gerrit.opnfv.org/gerrit/#/c/18453/ CHANGES: - all: release (was all: build) - Common make targets now depend on submodules-init; - clean-build and clean-docker make targets have been replaced by: make clean, make deepclean; FIXME: These issues should be fixed later: - (optional) trim local repo paths from gitinfo files - only account for armband submodules in cache fingerprints; - identify size mismatch culprit Change-Id: I7d73409be44983fa21d4c09d2e07b87a33231e03 Signed-off-by: Alexandru Avadanii --- Makefile | 189 ++++++++++++++++++++++++++++++++------------------------------- 1 file changed, 95 insertions(+), 94 deletions(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index df14ef68..519bce64 100644 --- a/Makefile +++ b/Makefile @@ -1,115 +1,116 @@ -root:=$(shell pwd) +############################################################################## +# Copyright (c) 2016 Cavium +# Copyright (c) 2016 Enea AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# NOTE: Armband patching consists of: +# - clone upstream repositories to git submodules; +# - checkout submodule commits where set in Fuel@OPNFV's config.mk +# (e.g. FUEL_ASTUTE_COMMIT=...); +# - tag each submodule (before patching) with "armband-workbench-root"; +# - apply Armband patches for each submodule; +# - tag each submodule (after patching) with "armband-workbench"; +# - pass updated repository info to Fuel@OPNFV build system +# (e.g. FUEL_ASTUTE_COMMIT=HEAD) via armband.mk; + +# NOTE: Long-term goals (Armband repo should merge with Fuel@OPNFV): +# - all build related changes should affect Fuel@OPNFV, NOT Armband; +# - Armband make/build system should only handle patching, +# including eventual build related patching of Fuel@OPNFV, +# and then invoke Fuel@OPNFV's build system; +# - Fuel@OPNFV is made aware of an Armband type build by passing +# the "ARMBAND_BASE" env var; + +# Fist, inherit Fuel submodule commit references from Fuel@OPNFV +# using "config.mk" as a make target that links to Fuel's config.mk. +# Some values will be overriden at Fuel ISO build time by armband.mk. include config.mk -all: build +export ARMBAND_BASE := $(shell pwd) +export OPNFV_GIT_SHA := $(shell git rev-parse HEAD) +export REVSTATE + +# Prepare for future directory re-layout when merging with Fuel@OPNFV +PATCH_DIR := ${ARMBAND_BASE}/patches +SUBMOD_DIR := ${ARMBAND_BASE}/upstream +FUEL_BASE := ${SUBMOD_DIR}/fuel + +all: release +# Use config.mk & clean_cache.sh from Fuel@OPNFV config.mk: submodules-init - @ln -s -f ${root}/upstream/fuel/build/config.mk ${root}/config.mk + @ln -sf ${FUEL_BASE}/build/config.mk ${ARMBAND_BASE}/config.mk + @ln -sf ${FUEL_BASE}/ci/clean_cache.sh ${ARMBAND_BASE}/ci/clean_cache.sh -.PHONY: submodules-init submodules-clean +# Fetch & update git submodules, checkout remote HEAD +.PHONY: submodules-init submodules-init: - @if [ ! -d ${root}/upstream/fuel/build ]; then \ + @if [ ! -d ${FUEL_BASE}/build ]; then \ git submodule -q init; \ git submodule -q sync; \ git submodule update --remote; \ fi -# Cleans any changes made to submodules -submodules-clean: - @git submodule -q foreach \ - 'git am --abort; \ - git checkout -q armband-workbench-root; \ - git branch -q -D armband-workbench; \ - git tag -d armband-workbench-root; \ - git reset -q --hard HEAD; git clean -xdff' +# Clean any changes made to submodules, checkout Armband root commit +.PHONY: submodules-clean +submodules-clean: submodules-init + @git submodule -q foreach ' \ + git am -q --abort 2>/dev/null; \ + git checkout -q armband-workbench-root 2>/dev/null; \ + git branch -q -D armband-workbench 2>/dev/null; \ + git tag -d armband-workbench-root 2>/dev/null; \ + git reset -q --hard HEAD; \ + git clean -xdff' -.PHONY: patches-export patches-import # Generate patches from submodules -patches-export: - @git submodule -q foreach 'mkdir -p ${root}/patches/$$name' - @git submodule -q foreach 'git format-patch \ - -o ${root}/patches/$$name -N armband-workbench-root \ - --no-signature' - @find ${root}/patches -name '*.patch' -exec sed -i -e '1d' {} \; +.PHONY: patches-export +patches-export: submodules-init + @git submodule -q foreach ' \ + mkdir -p ${PATCH_DIR}/$$name; \ + git format-patch --no-signature \ + -o ${PATCH_DIR}/$$name -N armband-workbench-root' + @find ${PATCH_DIR} -name '*.patch' -exec sed -i -e '1d' {} \; -# apply patches from patches/* to respective submodules -patches-import: - @cd ${root}/upstream/fuel-agent && \ - git checkout -q ${FUEL_AGENT_COMMIT} - @cd ${root}/upstream/fuel-astute && \ - git checkout -q ${ASTUTE_COMMIT} - @cd ${root}/upstream/fuel-library && \ - git checkout -q ${FUELLIB_COMMIT} - @cd ${root}/upstream/fuel-mirror && \ - git checkout -q ${FUEL_MIRROR_COMMIT} - @cd ${root}/upstream/fuel-nailgun-agent && \ - git checkout -q ${FUEL_NAILGUN_AGENT_COMMIT} - @cd ${root}/upstream/fuel-web && \ - git checkout -q ${NAILGUN_COMMIT} - @git submodule -q foreach 'mkdir -p ${root}/patches/$$name' - @git submodule -q foreach 'git tag armband-workbench-root' - @git submodule -q foreach 'git checkout -q -b armband-workbench' - @git submodule -q foreach \ - 'if [ ! -z "$$(ls ${root}/patches/$$name/)" ]; then \ +# Apply patches from patches/* to respective submodules +# For repos pinned in Fuel@OPNFV's config.mk, checkout pinned commit first +.PHONY: patches-import +patches-import: submodules-init + @git -C ${FUEL_BASE} checkout -q master + @git -C ${SUBMOD_DIR}/fuel-agent checkout -q ${FUEL_AGENT_COMMIT} + @git -C ${SUBMOD_DIR}/fuel-astute checkout -q ${ASTUTE_COMMIT} + @git -C ${SUBMOD_DIR}/fuel-library checkout -q ${FUELLIB_COMMIT} + @git -C ${SUBMOD_DIR}/fuel-mirror checkout -q ${FUEL_MIRROR_COMMIT} + @git -C ${SUBMOD_DIR}/fuel-nailgun-agent \ + checkout -q ${FUEL_NAILGUN_AGENT_COMMIT} + @git -C ${SUBMOD_DIR}/fuel-web checkout -q ${NAILGUN_COMMIT} + @git submodule -q foreach ' \ + mkdir -p ${PATCH_DIR}/$$name; \ + git tag armband-workbench-root; \ + git checkout -q -b armband-workbench; \ + if [ ! -z "$$(ls ${PATCH_DIR}/$$name/)" ]; then \ echo "-- patching $$name"; \ - git am ${root}/patches/$$name/*.patch; \ + git am --whitespace=nowarn \ + --committer-date-is-author-date \ + ${PATCH_DIR}/$$name/*.patch; \ fi' -clean-docker: - @if [ -d ${root}/upstream/fuel/build ]; then \ - sudo make -C ${root}/upstream/fuel/build deepclean; \ - fi - @for container in $(shell sudo docker ps -a -q); do \ - sudo docker rm -f -v $${container}; \ - done - @for image in $(shell sudo docker images -q); do \ - sudo docker rmi -f $${image}; \ - done - -clean-build: - sudo rm -rf /tmp/fuel-main - git -C ${root}/upstream/fuel reset --hard HEAD - git -C ${root}/upstream/fuel clean -xdff -release: export LC_ALL=en_US.UTF-8 -release: submodules-clean clean-docker clean-build submodules-init patches-import build +# Pass down clean/deepclean/build to Fuel@OPNFV +.PHONY: clean +clean: submodules-init + $(MAKE) -e --no-print-directory -C ${FUEL_BASE}/build clean -ifneq ($(REVSTATE),) - EXTRA_PARAMS="REVSTATE=$(REVSTATE)" -endif +.PHONY: deepclean +deepclean: submodules-init + $(MAKE) -e --no-print-directory -C ${FUEL_BASE}/build deepclean +.PHONY: build build: - cd ${root}/upstream/fuel/build && \ - make \ - BUILD_FUEL_PLUGINS="f_odlpluginbuild f_bgpvpn-pluginbuild" \ - UBUNTU_ARCH="amd64 arm64" \ - PRODNO="OPNFV_A_FUEL" \ - OPNFV_GIT_SHA=$(shell git rev-parse HEAD) \ - ASTUTE_REPO=${root}/upstream/fuel-astute \ - ASTUTE_COMMIT=HEAD \ - NAILGUN_REPO=${root}/upstream/fuel-web \ - NAILGUN_COMMIT=HEAD \ - FUEL_AGENT_REPO=${root}/upstream/fuel-agent \ - FUEL_AGENT_COMMIT=HEAD \ - FUEL_NAILGUN_AGENT_REPO=${root}/upstream/fuel-nailgun-agent \ - FUEL_NAILGUN_AGENT_COMMIT=HEAD \ - FUEL_MIRROR_REPO=${root}/upstream/fuel-mirror \ - FUEL_MIRROR_COMMIT=HEAD \ - FUELLIB_REPO=${root}/upstream/fuel-library \ - FUELLIB_COMMIT=HEAD \ - ODL_REPO=${root}/upstream/fuel-plugin-opendaylight \ - ODL_BRANCH=armband-workbench \ - ODL_CHANGE= \ - OPNFV_QUAGGE_PACKAGING_REPO="https://github.com/alexandruavadanii/opnfv-quagga-packaging" \ - OVS_NSH_DPDK_REPO=${root}/upstream/fuel-plugin-ovs \ - OVS_NSH_DPDK_BRANCH=HEAD \ - VSPERF_REPO=${root}/upstream/vswitchperf \ - VSPERF_BRANCH=armband-workbench \ - VSPERF_CHANGE= \ - YARDSTICK_REPO=${root}/upstream/yardstick \ - YARDSTICK_BRANCH=armband-workbench \ - YARDSTICK_CHANGE= \ - EXTRA_RPM_REPOS="armband,http://linux.enea.com/mos-repos/centos/mos9.0-centos7/armband/x86_64,10" \ - MIRROR_MOS_UBUNTU=linux.enea.com \ - $(EXTRA_PARAMS) \ - iso 2>&1 | tee ${root}/build.log + $(MAKE) -e --no-print-directory -C ${FUEL_BASE}/build all +.PHONY: release +release: export LC_ALL=en_US.UTF-8 +release: submodules-clean patches-import build -- cgit 1.2.3-korg