salt.sh: Drop upstream clone in favor of local git

salt.sh currently clones the full Fuel@OPNFV git repo from upstream
public mirror, preventing us from testing locally edited or new
patches.

Instead, bring back git submodule handling from old f_repos, clone
and patch each submodule locally, then copy the whole parent repo
over to cfg01.

This is also a first step towards implementing offline deploy support.

NOTE: This adds new deploy prerequisite packages:
- git (for submodule clone/update);
- make (for submodule patching);
- rsync (for parent repo replication to cfg01);

NOTE: Parent repository is expected to be a git repo, in order to
work with git submodules.

While at it, perform some minor related changes:
- add deploy artifacts (ISOs, qcow2 files) to .gitignore, also used
  to filter-out such files during rsync to cfg01;
- remove obsolete Fuel patches (old f_repos mechanism);
- rename "reclass-system-salt-model" submodule;

Change-Id: I6210d80d41010b2802e4f1b31acf249a18db7963
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>

6 files changed, 289 insertions, 11 deletions

diff --git a/mcp/patches/Makefile b/mcp/patches/Makefile
new file mode 100644
index 000000000..6a176db8d
--- /dev/null
+++ b/mcp/patches/Makefile
@@ -0,0 +1,117 @@
+##############################################################################
+# Copyright (c) 2015,2016,2017 Ericsson AB, Enea AB and others.
+# stefan.k.berg@ericsson.com
+# jonas.bjurel@ericsson.com
+# Alexandru.Avadanii@enea.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+include config.mk
+
+SHELL = /bin/sh
+FPATCHES = $(shell find ${F_PATCH_DIR} -name '*.patch')
+
+# NOTE: Mechanism overview is presented in ./README.md.
+
+# Submodule consistent states:
+#   - NOT initialized (submodule trees are not populated at all);
+#   - initialized, bound to saved commits;
+#   - initialized, tracking remote origin (only for FUEL_TRACK_REMOTES);
+#   - patched (local patches are applied);
+
+# In order to keep things sort of separate, we should only pass up (to main
+# Makefile) the fully-patched repos, and gather any fingerprinting info here.
+
+# Fuel@OPNFV relies on upstream git repos (one per component) in 1 of 2 ways:
+#   - pinned down to tag objects (e.g. "9.0.1")
+#   - tracking upstream remote HEAD on a stable or master branch
+# FIXME(alav): Should we support mixed cases? (e.g. pin down only fuel-main)
+
+# To enable remote tracking, set the following var to any non-empty string.
+# Leaving this var empty will bind each git submodule to its saved commit.
+FUEL_TRACK_REMOTES ?= yes
+
+.PHONY: all
+all: release
+
+##############################################################################
+# git submodule operations - to be used stand-alone or from parent Makefile
+##############################################################################
+
+# Fetch & update git submodules, checkout remote HEAD or saved commit
+# Also gather fingerprints for parent gitinfo_fuel.txt.
+.PHONY: sub
+sub: .cachefuelinfo
+
+.cachefuelinfo:
+	@if [ -n "${FUEL_TRACK_REMOTES}" ]; then \
+		git submodule update --init --remote 2>/dev/null; \
+	else \
+		git submodule update --init 2>/dev/null; \
+	fi
+	@touch $@
+
+# Generate patches from submodules
+.PHONY: patches-export
+patches-export: sub
+	@git submodule -q foreach ' \
+		SUB_DIR=${F_PATCH_DIR}/$$name; \
+		git tag | awk "!/root/ && /${F_OPNFV_TAG}-fuel/" | while read F_TAG; do \
+			SUB_FEATURE=`dirname $${F_TAG#${F_OPNFV_TAG}-fuel/}`; \
+			echo "`tput setaf 2`-- exporting $$name ($$F_TAG)`tput sgr0`"; \
+			mkdir -p $$SUB_DIR/$${SUB_FEATURE} && \
+			git format-patch --no-signature --ignore-space-at-eol \
+				-o $$SUB_DIR/$$SUB_FEATURE -N $$F_TAG-root..$$F_TAG; \
+			sed -i -e "1{/From: /!d}" -e "s/[[:space:]]*$$//" \
+				$$SUB_DIR/$$SUB_FEATURE/*.patch; \
+		done'
+
+# Apply patches from patch/* to respective submodules
+# We rely on `make sub` and/or `make clean` to checkout correct base
+.PHONY: patches-import
+patches-import: sub .cachepatched
+
+.cachepatched: ${FPATCHES}
+	@$(MAKE) clean
+	@git submodule -q foreach ' \
+		SUB_DIR=${F_PATCH_DIR}/$$name; mkdir -p $$SUB_DIR && \
+		git tag ${F_OPNFV_TAG}-root && \
+		git checkout -q -b opnfv-fuel && \
+		find $$SUB_DIR -type d | sort | while read p_dir; do \
+			SUB_PATCHES=$$(ls $$p_dir/*.patch 2>/dev/null); \
+			if [ -n "$$SUB_PATCHES" ]; then \
+				SUB_FEATURE=$${p_dir#$$SUB_DIR}; \
+				SUB_TAG=${F_OPNFV_TAG}-fuel$$SUB_FEATURE/patch; \
+				echo "`tput setaf 2`-- patching $$name ($$SUB_TAG)`tput sgr0`";\
+				git tag $$SUB_TAG-root && \
+				git am -3 --whitespace=nowarn --patch-format=mbox \
+					--committer-date-is-author-date $$SUB_PATCHES && \
+				git tag $$SUB_TAG || exit 1; \
+			fi \
+		done && \
+		git tag ${F_OPNFV_TAG}'
+	@touch $@
+
+# Clean any changes made to submodules, checkout upstream Fuel root commit
+.PHONY: clean
+clean:
+	@cd ${F_GIT_ROOT} && git submodule -q foreach ' \
+		git am -q --abort > /dev/null 2>&1; \
+		git checkout -q -f ${F_OPNFV_TAG}-root > /dev/null 2>&1; \
+		git branch -q -D opnfv-fuel > /dev/null 2>&1; \
+		git tag | grep ${F_OPNFV_TAG} | xargs git tag -d > /dev/null 2>&1; \
+		git reset -q --hard HEAD; \
+		git clean -xdff'
+	@rm -f .cachepatched
+
+.PHONY: deepclean
+deepclean: clean
+	@git submodule deinit -f ${F_GIT_ROOT}
+	@rm -f .cache*
+
+.PHONY: release
+release: sub
+	$(MAKE) -f Makefile patches-import
diff --git a/mcp/patches/README.md b/mcp/patches/README.md
new file mode 100644
index 000000000..eab4d64e3
--- /dev/null
+++ b/mcp/patches/README.md
@@ -0,0 +1,103 @@
+Fuel@OPNFV submodule fetching and patching
+==========================================
+
+This directory holds submodule fetching/patching scripts, intended for
+working with upstream Fuel/MCP components (e.g.: reclass-system-salt-model) in
+developing/applying OPNFV patches (backports, custom fixes etc.).
+
+The scripts should be friendly to the following 2 use-cases:
+  - development work: easily cloning, binding repos to specific commits,
+    remote tracking, patch development etc.;
+  - to provide parent build scripts an easy method of tracking upstream
+    references and applying OPNFV patches on top;
+
+Also, we need to support at least the following modes of operations:
+  - submodule bind - each submodule patches will be based on the commit ID
+    saved in the .gitmodules config file;
+  - remote tracking - each submodule will sync with the upstream remote
+    and patches will be applied on top of <sub_remote>/<sub_branch>/HEAD;
+
+Workflow (development)
+----------------------
+The standard development workflow should look as follows:
+
+1. Decide whether remote tracking should be active or not:
+   NOTE: Setting the following var to any non-empty str enables remote track.
+   NOTE: Leaving unset will enable remote track for anything but stable branch.
+
+   $ export FUEL_TRACK_REMOTES=""
+
+2. All Fuel sub-projects are registered as submodules. To initialize them, call:
+   If remote tracking is active, upstream remote is queried and latest remote
+   branch HEAD is fetched. Otherwise, checkout commit IDs from .gitmodules.
+
+   $ make sub
+
+3. Apply patches from `patches/<sub-project>/*` to respective submodules via:
+
+   $ make patches-import
+
+   This will result in creation of:
+   - a tag called `${FUEL_MAIN_TAG}-opnfv-root` at the same commit as Fuel@OPNFV
+     upstream reference (bound to git submodule OR tracking remote HEAD);
+   - a new branch `opnfv-fuel` which will hold all the OPNFV patches,
+     each patch is applied on this new branch with `git-am`;
+   - a tag called `${FUEL_MAIN_TAG}-opnfv` at `opnfv-fuel/HEAD`;
+
+4. Modify sub-projects for whatever you need.
+   Commit your changes when you want them taken into account in the build.
+
+5. Re-create patches via:
+
+   $ make patches-export
+
+   Each commit on `opnfv-fuel` branch of each subproject will be
+   exported to `patches/subproject/` via `git format-patch`.
+
+   NOTE: Only commit (-f) submodules when you need to bump upstream ref.
+   NOTE: DO NOT commit patched submodules!
+
+6. Clean workbench branches and tags with:
+
+   $ make clean
+
+7. De-initialize submodules and force a clean clone with:
+
+   $ make deepclean
+
+Sub-project maintenance
+-----------------------
+1. Adding a new submodule
+   If you need to add another subproject, you can do it with `git submodule`.
+   Make sure that you specify branch (with `-b`), short name (with `--name`):
+
+   $ git submodule -b master add --name reclass-system-salt-model \
+     https://github.com/Mirantis/reclass-system-salt-model \
+     relative/path/to/submodule
+
+2. Working with remote tracking for upgrading Fuel components
+   Enable remote tracking as described above, which at `make sub` will update
+   ALL submodules (e.g. reclass-system-salt-model) to remote branch (set in
+   .gitmodules) HEAD.
+
+   * If upstream has NOT already tagged a new version, we can still work on
+     our patches, make sure they apply etc., then check for new upstream
+     changes (and that our patches still apply on top of them) by:
+
+   $ make deepclean patches-import
+
+   * If upstream has already tagged a new version we want to pick up, checkout
+     the new tag in each submodule:
+
+   $ git submodule foreach 'git checkout <newtag>'
+
+   * Once satisfied with the patch and submodule changes, commit them:
+     - enforce FUEL_TRACK_REMOTES to "yes" if you want to constatly use the
+       latest remote branch HEAD (as soon as upstream pushes a change on that
+       branch, our next build will automatically include it - risk of our
+       patches colliding with new upstream changes);
+     - stage patch changes if any;
+     - if submodule tags have been updated (relevant when remote tracking is
+       disabled, i.e. we have a stable upstream baseline), add submodules:
+
+   $ make deepclean sub && git add -f relative/path/to/submodule
diff --git a/mcp/patches/config.mk b/mcp/patches/config.mk
new file mode 100644
index 000000000..260cbf829
--- /dev/null
+++ b/mcp/patches/config.mk
@@ -0,0 +1,23 @@
+##############################################################################
+# Copyright (c) 2015,2016,2017 Ericsson AB, Enea AB and others.
+# stefan.k.berg@ericsson.com
+# jonas.bjurel@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+##############################################################################
+# Components pinning / remote tracking
+##############################################################################
+
+# git submodule & patch locations for Fuel components
+F_GIT_ROOT   := $(shell git rev-parse --show-toplevel)
+F_GIT_DIR    := $(shell git rev-parse --git-dir)
+F_PATCH_DIR  := $(shell pwd)
+F_OPNFV_TAG  := master-opnfv
+
+# for the patches applying purposes (empty git config in docker build container)
+export GIT_COMMITTER_NAME?=Fuel OPNFV
+export GIT_COMMITTER_EMAIL?=fuel@opnfv.org
diff --git a/mcp/patches/patches.list b/mcp/patches/patches.list
index 0a84657d2..0542433c7 100644
--- a/mcp/patches/patches.list
+++ b/mcp/patches/patches.list
@@ -1,2 +1 @@
 /usr/share/salt-formulas/env: 0002-opendaylight-formula-neutron.patch
-/srv/salt/reclass/classes/system: 0001-opendaylight-reclass-system.patch
diff --git a/mcp/patches/0001-opendaylight-reclass-system.patch b/mcp/patches/reclass-system-salt-model/0001-Bring-in-opendaylight-support.patch
index f8c986be6..15f23db34 100644
--- a/mcp/patches/0001-opendaylight-reclass-system.patch
+++ b/mcp/patches/reclass-system-salt-model/0001-Bring-in-opendaylight-support.patch
@@ -3,6 +3,18 @@ Date: Thu, 29 Jun 2017 12:22:42 +0400
 Subject: [PATCH] Bring in opendaylight support
 
 Change-Id: I3efec9a8b586a6c75b1c1635ad2a7024d73d9ad2
+---
+ neutron/control/opendaylight/cluster.yml              | 19 +++++++++++++++++++
+ neutron/control/opendaylight/single.yml               | 16 ++++++++++++++++
+ neutron/gateway/opendaylight/single.yml               |  8 ++++++++
+ opendaylight/server/single.yml                        |  2 ++
+ .../storage/system/opendaylight_control_single.yml    | 15 +++++++++++++++
+ 5 files changed, 60 insertions(+)
+ create mode 100644 neutron/control/opendaylight/cluster.yml
+ create mode 100644 neutron/control/opendaylight/single.yml
+ create mode 100644 neutron/gateway/opendaylight/single.yml
+ create mode 100644 opendaylight/server/single.yml
+ create mode 100644 reclass/storage/system/opendaylight_control_single.yml
 
 diff --git a/neutron/control/opendaylight/cluster.yml b/neutron/control/opendaylight/cluster.yml
 new file mode 100644
diff --git a/mcp/scripts/salt.sh b/mcp/scripts/salt.sh
index 4c6cbf782..3d0fb9117 100755
--- a/mcp/scripts/salt.sh
+++ b/mcp/scripts/salt.sh
@@ -1,34 +1,58 @@
 #!/bin/bash
+##############################################################################
+# Copyright (c) 2017 Mirantis Inc., Enea AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
 #
 # Deploy Salt Master
 #
 
+F_GIT_ROOT=$(git rev-parse --show-toplevel)
+SALT_MASTER_USER=${SALT_MASTER_USER:-ubuntu}
+SSH_SALT="${SALT_MASTER_USER}@${SALT_MASTER}"
+OPNFV_TMP_DIR="/home/${SALT_MASTER_USER}/fuel"
+OPNFV_FUEL_DIR="/root/fuel"
+
+# patch reclass-system-salt-model locally before copying it over
+make -C "${F_GIT_ROOT}/mcp/patches" patches-import
+
+# push to cfg01 current git repo first (including submodules), at ~ubuntu/fuel
+# later we move it to ~root/fuel and delete the temporary clone
+rsync -Erl --delete -e "ssh ${SSH_OPTS}" \
+  --exclude-from="${F_GIT_ROOT}/.gitignore" \
+  "${F_GIT_ROOT}/" "${SSH_SALT}:$(basename "${OPNFV_TMP_DIR}")/"
+
 # ssh to cfg01
 # shellcheck disable=SC2086,2087
-ssh ${SSH_OPTS} "ubuntu@${SALT_MASTER}" bash -s << SALT_INSTALL_END
+ssh ${SSH_OPTS} "${SSH_SALT}" bash -s << SALT_INSTALL_END
   sudo -i
 
   echo -n 'Checking out cloud-init has finished running ...'
   while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo -n '.'; sleep 1; done
   echo ' done'
 
-  apt-get install -y git curl subversion
+  DEBIAN_FRONTEND=noninteractive apt-get install -y git curl subversion
 
-  svn export --force https://github.com/salt-formulas/salt-formulas/trunk/deploy/scripts /srv/salt/scripts
-  git clone --depth=1 --recurse-submodules https://git.opnfv.org/fuel
-  ln -s /root/fuel/mcp/reclass /srv/salt/reclass
+  svn export --force \
+    https://github.com/salt-formulas/salt-formulas/trunk/deploy/scripts /srv/salt/scripts
+  mv ${OPNFV_TMP_DIR} ${OPNFV_FUEL_DIR} && chown -R root.root ${OPNFV_FUEL_DIR}
+  ln -s ${OPNFV_FUEL_DIR}/mcp/reclass /srv/salt/reclass
 
   mkdir -p /usr/share/salt-formulas/reclass
-  cp -r /root/fuel/mcp/metadata/service /usr/share/salt-formulas/reclass
-  cd /srv/salt/reclass/classes/service && ln -s /usr/share/salt-formulas/reclass/service/opendaylight
-  cd /root/fuel/mcp/patches && ./patch.sh patches.list reclass
+  cp -r ${OPNFV_FUEL_DIR}/mcp/metadata/service /usr/share/salt-formulas/reclass
+  cd /srv/salt/reclass/classes/service && \
+    ln -s /usr/share/salt-formulas/reclass/service/opendaylight
+  cd ${OPNFV_FUEL_DIR}/mcp/patches && ./patch.sh patches.list reclass
 
   cd /srv/salt/scripts
   MASTER_HOSTNAME=cfg01.${CLUSTER_DOMAIN} DISTRIB_REVISION=nightly ./salt-master-init.sh
   salt-key -Ay
 
-  cp -r /root/fuel/mcp/salt-formulas/* /usr/share/salt-formulas/env
-  cd /root/fuel/mcp/patches && ./patch.sh patches.list formulas
+  cp -r ${OPNFV_FUEL_DIR}/mcp/salt-formulas/* /usr/share/salt-formulas/env
+  cd ${OPNFV_FUEL_DIR}/mcp/patches && ./patch.sh patches.list formulas
 
   salt-call state.apply salt
   salt '*' state.apply salt | fgrep -q 'No response' && salt '*' state.apply salt