From 42f8585ebb8fffad19a89314659ab9129176c3e9 Mon Sep 17 00:00:00 2001 From: Alexandru Avadanii Date: Sat, 1 Oct 2016 13:18:11 +0200 Subject: build: Rework patch mechanism for Fuel submodules While refactoring the patching mechanism, take care of: - Sync submodule handling with Fuel@OPFNV; - build: Investigate/prepare for moving patches to Fuel@OPNFV; - build: Investigate divergent fuel-mirror; - ISO build: cacheid for Fuel comps should not depend on Armband git commit; CHANGE: Rename/shuffle patches while grouping them in "features", preparing for upstreaming them to Fuel@OPNFV and beyond. CHANGE: Allow linking patches for better representing the dependency between one patch and different features. e.g. 0001-Add-arch-to-nailgun-release-and-target-image.patch: - part of `multiarch-fuel`, because it extends Fuel; - part of `direct-kernel-boot`, as arch is required for that; - part of `cross-bootstrap`, target image is arch-dependent; NOTE: Patch links are not staged to Fuel@OPNFV, they only serve as markers that a specific patch is part of a feature. CHANGE: Kill all Fuel component submodules, now handled in Fuel@OPNFV: - fuel-agent - fuel-astute - fuel-library - fuel-mirror - fuel-nailgun-agent - fuel-web CHANGE: Move armband-fuel-config.mk to armband git root. FIXME: m1.micro-Increase-profile-RAM-size-to-128MB.patch is NOT part of `cross-bootstrap` feature, but patch context says so ... FIXME: 0001-Add-arm64-deb-repositories-setup.patch is broken at `make patches-export` by removing spaces at EOL. v2 -> v3: * Phony patch support (links to show a patch belongs to a feature); * Updated README.md v3 -> v7: * Re-export Fuel submodules & plugins patches (update patch context); * Update Cavium mail addresses (s/caviumnetworks.com/cavium.com/); * Ignore submodule changes; * Add armband git repo info to gitinfo_fuel.txt at build time; Implements: ARMBAND-136 Closes-bug: ARMBAND-95 Closes-bug: ARMBAND-93 Closes-bug: ARMBAND-92 Change-Id: I1a236d9f43b2e6dca22055911f696b43c22b5973 Signed-off-by: Alexandru Avadanii --- Makefile | 168 ++++++++++++++++++++++++++++++++++++++++----------------------- 1 file changed, 106 insertions(+), 62 deletions(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index d3bc979a..8300711d 100644 --- a/Makefile +++ b/Makefile @@ -8,108 +8,152 @@ ############################################################################## # NOTE: Armband patching consists of: -# - clone upstream repositories to git submodules; -# - checkout submodule commits where set in Fuel@OPNFV's config.mk -# (e.g. FUEL_ASTUTE_COMMIT=...); -# - tag each submodule (before patching) with "armband-workbench-root"; +# - clone upstream repositories to git submodules, tracking remotes; +# - tag each submodule (before patching) with "${A_OPNFV_TAG}-root"; # - apply Armband patches for each submodule; -# - tag each submodule (after patching) with "armband-workbench"; +# - tag each submodule (after patching) with "${OPNFV_TAG}"; +# - stage Fuel submodule patches by copying them to Fuel f_repos/patch dir; # - pass updated repository info to Fuel@OPNFV build system -# (e.g. FUEL_ASTUTE_COMMIT=HEAD) via armband.mk; +# (e.g. FUEL_PLUGIN_ODL_CHANGE="${OPNFV_TAG}") via armband-fuel-config.mk; # NOTE: Long-term goals (Armband repo should merge with Fuel@OPNFV): # - all build related changes should affect Fuel@OPNFV, NOT Armband; -# - Armband make/build system should only handle patching, -# including eventual build related patching of Fuel@OPNFV, -# and then invoke Fuel@OPNFV's build system; +# - Armband make/build system should only handle patching non-Fuel submodules, +# and then invoke Fuel@OPNFV's patch & build system; # - Fuel@OPNFV is made aware of an Armband type build by passing # the "ARMBAND_BASE" env var; -# Fist, inherit Fuel submodule commit references from Fuel@OPNFV -# using "config.mk" as a make target that links to Fuel's config.mk. -# Some values will be overriden at Fuel ISO build time by armband.mk. -include config.mk - export ARMBAND_BASE := $(shell pwd) export OPNFV_GIT_SHA := $(shell git rev-parse HEAD) export REVSTATE -# Prepare for future directory re-layout when merging with Fuel@OPNFV -PATCH_DIR := ${ARMBAND_BASE}/patches -SUBMOD_DIR := ${ARMBAND_BASE}/upstream -FUEL_BASE := ${SUBMOD_DIR}/fuel +include armband-fuel-config.mk all: release -# Use config.mk & clean_cache.sh from Fuel@OPNFV -config.mk: submodules-init - @ln -sf ${FUEL_BASE}/build/config.mk ${ARMBAND_BASE}/config.mk - @ln -sf ${FUEL_BASE}/ci/clean_cache.sh ${ARMBAND_BASE}/ci/clean_cache.sh - # Fetch & update git submodules, checkout remote HEAD .PHONY: submodules-init -submodules-init: - @if [ ! -d ${FUEL_BASE}/build ]; then \ - git submodule -q init; \ - git submodule -q sync; \ - git submodule update --remote; \ +submodules-init: .submodules-init + +.submodules-init: + @if [ -n "${ARMBAND_TRACK_REMOTES}" ]; then \ + git submodule update --init --remote 2>/dev/null; \ + else \ + git submodule update --init 2>/dev/null; \ fi + @ln -sf ${A_FUEL_BASE}/ci/clean_cache.sh ${ARMBAND_BASE}/ci/clean_cache.sh + @touch $@ # Clean any changes made to submodules, checkout Armband root commit .PHONY: submodules-clean -submodules-clean: submodules-init +submodules-clean: .submodules-init + @test ! -d ${F_REPOS_DIR} || $(MAKE) fuel-patches-clean @git submodule -q foreach ' \ git am -q --abort 2>/dev/null; \ - git checkout -q armband-workbench-root 2>/dev/null; \ - git branch -q -D armband-workbench 2>/dev/null; \ - git tag -d armband-workbench-root 2>/dev/null; \ + git checkout -q -f ${A_OPNFV_TAG}-root 2>/dev/null; \ + git branch -q -D opnfv-armband 2>/dev/null; \ + git tag | grep ${A_OPNFV_TAG} | xargs git tag -d > /dev/null 2>&1; \ git reset -q --hard HEAD; \ git clean -xdff' + @rm -f .submodules-patched # Generate patches from submodules .PHONY: patches-export -patches-export: submodules-init +patches-export: .submodules-init @git submodule -q foreach ' \ - mkdir -p ${PATCH_DIR}/$$name; \ - git format-patch --no-signature \ - -o ${PATCH_DIR}/$$name -N armband-workbench-root' - @find ${PATCH_DIR} -name '*.patch' -exec sed -i -e '1d' {} \; + SUB_DIR=${A_PATCH_DIR}/$$name; \ + git tag | awk "!/root/ && /${A_OPNFV_TAG}-fuel/" | while read A_TAG; do \ + SUB_FEATURE=`dirname $${A_TAG#${A_OPNFV_TAG}-fuel/}`; \ + echo "`tput setaf 2`== exporting $$name ($$A_TAG)`tput sgr0`"; \ + mkdir -p $$SUB_DIR/$${SUB_FEATURE} && \ + git format-patch --no-signature --ignore-space-at-eol \ + -o $$SUB_DIR/$$SUB_FEATURE -N $$A_TAG-root..$$A_TAG; \ + sed -i -e "1{/From: /!d}" -e "s/[[:space:]]*$$//" \ + $$SUB_DIR/$$SUB_FEATURE/*.patch; \ + done' # Apply patches from patches/* to respective submodules -# For repos pinned in Fuel@OPNFV's config.mk, checkout pinned commit first .PHONY: patches-import -patches-import: submodules-init - @cd ${SUBMOD_DIR}/fuel-agent && git checkout -q ${FUEL_AGENT_COMMIT} - @cd ${SUBMOD_DIR}/fuel-astute && git checkout -q ${ASTUTE_COMMIT} - @cd ${SUBMOD_DIR}/fuel-library && git checkout -q ${FUELLIB_COMMIT} - @cd ${SUBMOD_DIR}/fuel-mirror && git checkout -q ${FUEL_MIRROR_COMMIT} - @cd ${SUBMOD_DIR}/fuel-nailgun-agent && \ - git checkout -q ${FUEL_NAILGUN_AGENT_COMMIT} - @cd ${SUBMOD_DIR}/fuel-web && git checkout -q ${NAILGUN_COMMIT} +patches-import: .submodules-init .submodules-patched + +.submodules-patched: ${A_PATCHES} + @$(MAKE) submodules-clean @git submodule -q foreach ' \ - mkdir -p ${PATCH_DIR}/$$name; \ - git tag armband-workbench-root; \ - git checkout -q -b armband-workbench; \ - if [ ! -z "$$(ls ${PATCH_DIR}/$$name/)" ]; then \ - echo "-- patching $$name"; \ - git am --whitespace=nowarn \ - --committer-date-is-author-date \ - ${PATCH_DIR}/$$name/*.patch; \ - fi' + SUB_DIR=${A_PATCH_DIR}/$$name; mkdir -p $$SUB_DIR && \ + git tag ${A_OPNFV_TAG}-root && \ + git checkout -q -b opnfv-armband && \ + find $$SUB_DIR -type d | sort | while read p_dir; do \ + SUB_PATCHES=$$(ls $$p_dir/*.patch 2>/dev/null); \ + if [ -n "$$SUB_PATCHES" ]; then \ + SUB_FEATURE=$${p_dir#$$SUB_DIR} \ + SUB_TAG=${A_OPNFV_TAG}-fuel$$SUB_FEATURE/patch; \ + echo "`tput setaf 2`== patching $$name ($$SUB_TAG)`tput sgr0`";\ + git tag $$SUB_TAG-root && git am -3 --whitespace=nowarn \ + --committer-date-is-author-date $$SUB_PATCHES && \ + git tag $$SUB_TAG || exit 1; \ + fi \ + done && \ + git tag ${A_OPNFV_TAG}' + # Staging Fuel@OPNFV patches + @ls -d ${F_SUB_DIR}/* 2>/dev/null | while read p_sub_path; do \ + SUB_NAME=`basename $$p_sub_path`; \ + find ${A_PATCH_DIR}/$$SUB_NAME -name '*.patch' 2>/dev/null -exec sh -c '\ + A_PATCH={}; R_PATCH=$${A_PATCH#${A_PATCH_DIR}/}; \ + F_PATCH=${F_PATCH_DIR}/$${0}/armband/$${R_PATCH#$${0}/}; \ + if [ -f $$F_PATCH ]; then \ + echo "`tput setaf 3`* WARN: $$R_PATCH upstream.`tput sgr0`"; \ + else \ + if [ -h $$A_PATCH ]; then \ + echo "`tput setaf 3`* PHONY: $$R_PATCH`tput sgr0`"; \ + else \ + echo "`tput setaf 6`* Staging $$R_PATCH`tput sgr0`"; \ + mkdir -p `dirname $$F_PATCH` && cp $$A_PATCH $$F_PATCH; \ + fi; \ + fi' "$$SUB_NAME" \; || true ; \ + done + @touch $@ # Pass down clean/deepclean/build to Fuel@OPNFV .PHONY: clean -clean: submodules-init - $(MAKE) -e --no-print-directory -C ${FUEL_BASE}/build clean +clean: .submodules-init + $(MAKE) -e --no-print-directory -C ${F_BUILD_DIR} clean .PHONY: deepclean -deepclean: submodules-init - $(MAKE) -e --no-print-directory -C ${FUEL_BASE}/build deepclean +deepclean: clean + $(MAKE) -e --no-print-directory -C ${F_BUILD_DIR} deepclean + @git submodule deinit -f . + @rm -f .submodules* .PHONY: build -build: - $(MAKE) -e --no-print-directory -C ${FUEL_BASE}/build all +build: patches-import + $(MAKE) -e --no-print-directory -C ${F_BUILD_DIR} all .PHONY: release release: export LC_ALL=en_US.UTF-8 -release: submodules-clean patches-import build +release: build + +############################################################################## +# Fuel@OPNFV patch operations - to be used only during development +############################################################################## + +# Apply all Fuel@OPNFV patches, including Armband patches +.PHONY: fuel-patches-import +fuel-patches-import: .submodules-patched fuel-patches-clean + $(MAKE) -e -C ${F_REPOS_DIR} patches-import + +# Export Fuel@OPNFV patches, including Armband patches +.PHONY: fuel-patches-export +fuel-patches-export: .submodules-patched + $(MAKE) -e -C ${F_REPOS_DIR} patches-export + @ls -d ${F_PATCH_DIR}/* 2>/dev/null | while read p_sub_path; do \ + SUB_NAME=`basename $$p_sub_path`; \ + if [ -d $$p_sub_path/armband ]; then \ + echo "`tput setaf 6`* Pulling $$SUB_NAME patches.`tput sgr0`"; \ + cp -R $$p_sub_path/armband/* ${A_PATCH_DIR}/$$SUB_NAME && \ + rm -rf $$p_sub_path/armband; \ + fi \ + done + +.PHONY: fuel-patches-clean +fuel-patches-clean: + $(MAKE) -e -C ${F_REPOS_DIR} clean -- cgit 1.2.3-korg