aboutsummaryrefslogtreecommitdiffstats
path: root/mcp
diff options
context:
space:
mode:
authorAlexandru Avadanii <Alexandru.Avadanii@enea.com>2017-07-29 22:49:26 +0200
committerAlexandru Avadanii <Alexandru.Avadanii@enea.com>2017-08-01 00:16:27 +0200
commit8d7128a62cd2f9b3df1d59bf4aa4fa7a387b1679 (patch)
tree9d6dba2a15d15b7f82668d23373a3ddd28ce9aa9 /mcp
parent9dfd08262dd3cf833c84cd1c7c1b05c25e09862b (diff)
salt.sh: Drop upstream clone in favor of local git
salt.sh currently clones the full Fuel@OPNFV git repo from upstream public mirror, preventing us from testing locally edited or new patches. Instead, bring back git submodule handling from old f_repos, clone and patch each submodule locally, then copy the whole parent repo over to cfg01. This is also a first step towards implementing offline deploy support. NOTE: This adds new deploy prerequisite packages: - git (for submodule clone/update); - make (for submodule patching); - rsync (for parent repo replication to cfg01); NOTE: Parent repository is expected to be a git repo, in order to work with git submodules. While at it, perform some minor related changes: - add deploy artifacts (ISOs, qcow2 files) to .gitignore, also used to filter-out such files during rsync to cfg01; - remove obsolete Fuel patches (old f_repos mechanism); - rename "reclass-system-salt-model" submodule; Change-Id: I6210d80d41010b2802e4f1b31acf249a18db7963 Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
Diffstat (limited to 'mcp')
-rw-r--r--mcp/patches/Makefile117
-rw-r--r--mcp/patches/README.md103
-rw-r--r--mcp/patches/config.mk23
-rw-r--r--mcp/patches/patches.list1
-rw-r--r--mcp/patches/reclass-system-salt-model/0001-Bring-in-opendaylight-support.patch (renamed from mcp/patches/0001-opendaylight-reclass-system.patch)12
-rwxr-xr-xmcp/scripts/salt.sh44
6 files changed, 289 insertions, 11 deletions
diff --git a/mcp/patches/Makefile b/mcp/patches/Makefile
new file mode 100644
index 000000000..6a176db8d
--- /dev/null
+++ b/mcp/patches/Makefile
@@ -0,0 +1,117 @@
+##############################################################################
+# Copyright (c) 2015,2016,2017 Ericsson AB, Enea AB and others.
+# stefan.k.berg@ericsson.com
+# jonas.bjurel@ericsson.com
+# Alexandru.Avadanii@enea.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+include config.mk
+
+SHELL = /bin/sh
+FPATCHES = $(shell find ${F_PATCH_DIR} -name '*.patch')
+
+# NOTE: Mechanism overview is presented in ./README.md.
+
+# Submodule consistent states:
+# - NOT initialized (submodule trees are not populated at all);
+# - initialized, bound to saved commits;
+# - initialized, tracking remote origin (only for FUEL_TRACK_REMOTES);
+# - patched (local patches are applied);
+
+# In order to keep things sort of separate, we should only pass up (to main
+# Makefile) the fully-patched repos, and gather any fingerprinting info here.
+
+# Fuel@OPNFV relies on upstream git repos (one per component) in 1 of 2 ways:
+# - pinned down to tag objects (e.g. "9.0.1")
+# - tracking upstream remote HEAD on a stable or master branch
+# FIXME(alav): Should we support mixed cases? (e.g. pin down only fuel-main)
+
+# To enable remote tracking, set the following var to any non-empty string.
+# Leaving this var empty will bind each git submodule to its saved commit.
+FUEL_TRACK_REMOTES ?= yes
+
+.PHONY: all
+all: release
+
+##############################################################################
+# git submodule operations - to be used stand-alone or from parent Makefile
+##############################################################################
+
+# Fetch & update git submodules, checkout remote HEAD or saved commit
+# Also gather fingerprints for parent gitinfo_fuel.txt.
+.PHONY: sub
+sub: .cachefuelinfo
+
+.cachefuelinfo:
+ @if [ -n "${FUEL_TRACK_REMOTES}" ]; then \
+ git submodule update --init --remote 2>/dev/null; \
+ else \
+ git submodule update --init 2>/dev/null; \
+ fi
+ @touch $@
+
+# Generate patches from submodules
+.PHONY: patches-export
+patches-export: sub
+ @git submodule -q foreach ' \
+ SUB_DIR=${F_PATCH_DIR}/$$name; \
+ git tag | awk "!/root/ && /${F_OPNFV_TAG}-fuel/" | while read F_TAG; do \
+ SUB_FEATURE=`dirname $${F_TAG#${F_OPNFV_TAG}-fuel/}`; \
+ echo "`tput setaf 2`-- exporting $$name ($$F_TAG)`tput sgr0`"; \
+ mkdir -p $$SUB_DIR/$${SUB_FEATURE} && \
+ git format-patch --no-signature --ignore-space-at-eol \
+ -o $$SUB_DIR/$$SUB_FEATURE -N $$F_TAG-root..$$F_TAG; \
+ sed -i -e "1{/From: /!d}" -e "s/[[:space:]]*$$//" \
+ $$SUB_DIR/$$SUB_FEATURE/*.patch; \
+ done'
+
+# Apply patches from patch/* to respective submodules
+# We rely on `make sub` and/or `make clean` to checkout correct base
+.PHONY: patches-import
+patches-import: sub .cachepatched
+
+.cachepatched: ${FPATCHES}
+ @$(MAKE) clean
+ @git submodule -q foreach ' \
+ SUB_DIR=${F_PATCH_DIR}/$$name; mkdir -p $$SUB_DIR && \
+ git tag ${F_OPNFV_TAG}-root && \
+ git checkout -q -b opnfv-fuel && \
+ find $$SUB_DIR -type d | sort | while read p_dir; do \
+ SUB_PATCHES=$$(ls $$p_dir/*.patch 2>/dev/null); \
+ if [ -n "$$SUB_PATCHES" ]; then \
+ SUB_FEATURE=$${p_dir#$$SUB_DIR}; \
+ SUB_TAG=${F_OPNFV_TAG}-fuel$$SUB_FEATURE/patch; \
+ echo "`tput setaf 2`-- patching $$name ($$SUB_TAG)`tput sgr0`";\
+ git tag $$SUB_TAG-root && \
+ git am -3 --whitespace=nowarn --patch-format=mbox \
+ --committer-date-is-author-date $$SUB_PATCHES && \
+ git tag $$SUB_TAG || exit 1; \
+ fi \
+ done && \
+ git tag ${F_OPNFV_TAG}'
+ @touch $@
+
+# Clean any changes made to submodules, checkout upstream Fuel root commit
+.PHONY: clean
+clean:
+ @cd ${F_GIT_ROOT} && git submodule -q foreach ' \
+ git am -q --abort > /dev/null 2>&1; \
+ git checkout -q -f ${F_OPNFV_TAG}-root > /dev/null 2>&1; \
+ git branch -q -D opnfv-fuel > /dev/null 2>&1; \
+ git tag | grep ${F_OPNFV_TAG} | xargs git tag -d > /dev/null 2>&1; \
+ git reset -q --hard HEAD; \
+ git clean -xdff'
+ @rm -f .cachepatched
+
+.PHONY: deepclean
+deepclean: clean
+ @git submodule deinit -f ${F_GIT_ROOT}
+ @rm -f .cache*
+
+.PHONY: release
+release: sub
+ $(MAKE) -f Makefile patches-import
diff --git a/mcp/patches/README.md b/mcp/patches/README.md
new file mode 100644
index 000000000..eab4d64e3
--- /dev/null
+++ b/mcp/patches/README.md
@@ -0,0 +1,103 @@
+Fuel@OPNFV submodule fetching and patching
+==========================================
+
+This directory holds submodule fetching/patching scripts, intended for
+working with upstream Fuel/MCP components (e.g.: reclass-system-salt-model) in
+developing/applying OPNFV patches (backports, custom fixes etc.).
+
+The scripts should be friendly to the following 2 use-cases:
+ - development work: easily cloning, binding repos to specific commits,
+ remote tracking, patch development etc.;
+ - to provide parent build scripts an easy method of tracking upstream
+ references and applying OPNFV patches on top;
+
+Also, we need to support at least the following modes of operations:
+ - submodule bind - each submodule patches will be based on the commit ID
+ saved in the .gitmodules config file;
+ - remote tracking - each submodule will sync with the upstream remote
+ and patches will be applied on top of <sub_remote>/<sub_branch>/HEAD;
+
+Workflow (development)
+----------------------
+The standard development workflow should look as follows:
+
+1. Decide whether remote tracking should be active or not:
+ NOTE: Setting the following var to any non-empty str enables remote track.
+ NOTE: Leaving unset will enable remote track for anything but stable branch.
+
+ $ export FUEL_TRACK_REMOTES=""
+
+2. All Fuel sub-projects are registered as submodules. To initialize them, call:
+ If remote tracking is active, upstream remote is queried and latest remote
+ branch HEAD is fetched. Otherwise, checkout commit IDs from .gitmodules.
+
+ $ make sub
+
+3. Apply patches from `patches/<sub-project>/*` to respective submodules via:
+
+ $ make patches-import
+
+ This will result in creation of:
+ - a tag called `${FUEL_MAIN_TAG}-opnfv-root` at the same commit as Fuel@OPNFV
+ upstream reference (bound to git submodule OR tracking remote HEAD);
+ - a new branch `opnfv-fuel` which will hold all the OPNFV patches,
+ each patch is applied on this new branch with `git-am`;
+ - a tag called `${FUEL_MAIN_TAG}-opnfv` at `opnfv-fuel/HEAD`;
+
+4. Modify sub-projects for whatever you need.
+ Commit your changes when you want them taken into account in the build.
+
+5. Re-create patches via:
+
+ $ make patches-export
+
+ Each commit on `opnfv-fuel` branch of each subproject will be
+ exported to `patches/subproject/` via `git format-patch`.
+
+ NOTE: Only commit (-f) submodules when you need to bump upstream ref.
+ NOTE: DO NOT commit patched submodules!
+
+6. Clean workbench branches and tags with:
+
+ $ make clean
+
+7. De-initialize submodules and force a clean clone with:
+
+ $ make deepclean
+
+Sub-project maintenance
+-----------------------
+1. Adding a new submodule
+ If you need to add another subproject, you can do it with `git submodule`.
+ Make sure that you specify branch (with `-b`), short name (with `--name`):
+
+ $ git submodule -b master add --name reclass-system-salt-model \
+ https://github.com/Mirantis/reclass-system-salt-model \
+ relative/path/to/submodule
+
+2. Working with remote tracking for upgrading Fuel components
+ Enable remote tracking as described above, which at `make sub` will update
+ ALL submodules (e.g. reclass-system-salt-model) to remote branch (set in
+ .gitmodules) HEAD.
+
+ * If upstream has NOT already tagged a new version, we can still work on
+ our patches, make sure they apply etc., then check for new upstream
+ changes (and that our patches still apply on top of them) by:
+
+ $ make deepclean patches-import
+
+ * If upstream has already tagged a new version we want to pick up, checkout
+ the new tag in each submodule:
+
+ $ git submodule foreach 'git checkout <newtag>'
+
+ * Once satisfied with the patch and submodule changes, commit them:
+ - enforce FUEL_TRACK_REMOTES to "yes" if you want to constatly use the
+ latest remote branch HEAD (as soon as upstream pushes a change on that
+ branch, our next build will automatically include it - risk of our
+ patches colliding with new upstream changes);
+ - stage patch changes if any;
+ - if submodule tags have been updated (relevant when remote tracking is
+ disabled, i.e. we have a stable upstream baseline), add submodules:
+
+ $ make deepclean sub && git add -f relative/path/to/submodule
diff --git a/mcp/patches/config.mk b/mcp/patches/config.mk
new file mode 100644
index 000000000..260cbf829
--- /dev/null
+++ b/mcp/patches/config.mk
@@ -0,0 +1,23 @@
+##############################################################################
+# Copyright (c) 2015,2016,2017 Ericsson AB, Enea AB and others.
+# stefan.k.berg@ericsson.com
+# jonas.bjurel@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+##############################################################################
+# Components pinning / remote tracking
+##############################################################################
+
+# git submodule & patch locations for Fuel components
+F_GIT_ROOT := $(shell git rev-parse --show-toplevel)
+F_GIT_DIR := $(shell git rev-parse --git-dir)
+F_PATCH_DIR := $(shell pwd)
+F_OPNFV_TAG := master-opnfv
+
+# for the patches applying purposes (empty git config in docker build container)
+export GIT_COMMITTER_NAME?=Fuel OPNFV
+export GIT_COMMITTER_EMAIL?=fuel@opnfv.org
diff --git a/mcp/patches/patches.list b/mcp/patches/patches.list
index 0a84657d2..0542433c7 100644
--- a/mcp/patches/patches.list
+++ b/mcp/patches/patches.list
@@ -1,2 +1 @@
/usr/share/salt-formulas/env: 0002-opendaylight-formula-neutron.patch
-/srv/salt/reclass/classes/system: 0001-opendaylight-reclass-system.patch
diff --git a/mcp/patches/0001-opendaylight-reclass-system.patch b/mcp/patches/reclass-system-salt-model/0001-Bring-in-opendaylight-support.patch
index f8c986be6..15f23db34 100644
--- a/mcp/patches/0001-opendaylight-reclass-system.patch
+++ b/mcp/patches/reclass-system-salt-model/0001-Bring-in-opendaylight-support.patch
@@ -3,6 +3,18 @@ Date: Thu, 29 Jun 2017 12:22:42 +0400
Subject: [PATCH] Bring in opendaylight support
Change-Id: I3efec9a8b586a6c75b1c1635ad2a7024d73d9ad2
+---
+ neutron/control/opendaylight/cluster.yml | 19 +++++++++++++++++++
+ neutron/control/opendaylight/single.yml | 16 ++++++++++++++++
+ neutron/gateway/opendaylight/single.yml | 8 ++++++++
+ opendaylight/server/single.yml | 2 ++
+ .../storage/system/opendaylight_control_single.yml | 15 +++++++++++++++
+ 5 files changed, 60 insertions(+)
+ create mode 100644 neutron/control/opendaylight/cluster.yml
+ create mode 100644 neutron/control/opendaylight/single.yml
+ create mode 100644 neutron/gateway/opendaylight/single.yml
+ create mode 100644 opendaylight/server/single.yml
+ create mode 100644 reclass/storage/system/opendaylight_control_single.yml
diff --git a/neutron/control/opendaylight/cluster.yml b/neutron/control/opendaylight/cluster.yml
new file mode 100644
diff --git a/mcp/scripts/salt.sh b/mcp/scripts/salt.sh
index 4c6cbf782..3d0fb9117 100755
--- a/mcp/scripts/salt.sh
+++ b/mcp/scripts/salt.sh
@@ -1,34 +1,58 @@
#!/bin/bash
+##############################################################################
+# Copyright (c) 2017 Mirantis Inc., Enea AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
#
# Deploy Salt Master
#
+F_GIT_ROOT=$(git rev-parse --show-toplevel)
+SALT_MASTER_USER=${SALT_MASTER_USER:-ubuntu}
+SSH_SALT="${SALT_MASTER_USER}@${SALT_MASTER}"
+OPNFV_TMP_DIR="/home/${SALT_MASTER_USER}/fuel"
+OPNFV_FUEL_DIR="/root/fuel"
+
+# patch reclass-system-salt-model locally before copying it over
+make -C "${F_GIT_ROOT}/mcp/patches" patches-import
+
+# push to cfg01 current git repo first (including submodules), at ~ubuntu/fuel
+# later we move it to ~root/fuel and delete the temporary clone
+rsync -Erl --delete -e "ssh ${SSH_OPTS}" \
+ --exclude-from="${F_GIT_ROOT}/.gitignore" \
+ "${F_GIT_ROOT}/" "${SSH_SALT}:$(basename "${OPNFV_TMP_DIR}")/"
+
# ssh to cfg01
# shellcheck disable=SC2086,2087
-ssh ${SSH_OPTS} "ubuntu@${SALT_MASTER}" bash -s << SALT_INSTALL_END
+ssh ${SSH_OPTS} "${SSH_SALT}" bash -s << SALT_INSTALL_END
sudo -i
echo -n 'Checking out cloud-init has finished running ...'
while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo -n '.'; sleep 1; done
echo ' done'
- apt-get install -y git curl subversion
+ DEBIAN_FRONTEND=noninteractive apt-get install -y git curl subversion
- svn export --force https://github.com/salt-formulas/salt-formulas/trunk/deploy/scripts /srv/salt/scripts
- git clone --depth=1 --recurse-submodules https://git.opnfv.org/fuel
- ln -s /root/fuel/mcp/reclass /srv/salt/reclass
+ svn export --force \
+ https://github.com/salt-formulas/salt-formulas/trunk/deploy/scripts /srv/salt/scripts
+ mv ${OPNFV_TMP_DIR} ${OPNFV_FUEL_DIR} && chown -R root.root ${OPNFV_FUEL_DIR}
+ ln -s ${OPNFV_FUEL_DIR}/mcp/reclass /srv/salt/reclass
mkdir -p /usr/share/salt-formulas/reclass
- cp -r /root/fuel/mcp/metadata/service /usr/share/salt-formulas/reclass
- cd /srv/salt/reclass/classes/service && ln -s /usr/share/salt-formulas/reclass/service/opendaylight
- cd /root/fuel/mcp/patches && ./patch.sh patches.list reclass
+ cp -r ${OPNFV_FUEL_DIR}/mcp/metadata/service /usr/share/salt-formulas/reclass
+ cd /srv/salt/reclass/classes/service && \
+ ln -s /usr/share/salt-formulas/reclass/service/opendaylight
+ cd ${OPNFV_FUEL_DIR}/mcp/patches && ./patch.sh patches.list reclass
cd /srv/salt/scripts
MASTER_HOSTNAME=cfg01.${CLUSTER_DOMAIN} DISTRIB_REVISION=nightly ./salt-master-init.sh
salt-key -Ay
- cp -r /root/fuel/mcp/salt-formulas/* /usr/share/salt-formulas/env
- cd /root/fuel/mcp/patches && ./patch.sh patches.list formulas
+ cp -r ${OPNFV_FUEL_DIR}/mcp/salt-formulas/* /usr/share/salt-formulas/env
+ cd ${OPNFV_FUEL_DIR}/mcp/patches && ./patch.sh patches.list formulas
salt-call state.apply salt
salt '*' state.apply salt | fgrep -q 'No response' && salt '*' state.apply salt