From 8d7128a62cd2f9b3df1d59bf4aa4fa7a387b1679 Mon Sep 17 00:00:00 2001 From: Alexandru Avadanii Date: Sat, 29 Jul 2017 22:49:26 +0200 Subject: salt.sh: Drop upstream clone in favor of local git salt.sh currently clones the full Fuel@OPNFV git repo from upstream public mirror, preventing us from testing locally edited or new patches. Instead, bring back git submodule handling from old f_repos, clone and patch each submodule locally, then copy the whole parent repo over to cfg01. This is also a first step towards implementing offline deploy support. NOTE: This adds new deploy prerequisite packages: - git (for submodule clone/update); - make (for submodule patching); - rsync (for parent repo replication to cfg01); NOTE: Parent repository is expected to be a git repo, in order to work with git submodules. While at it, perform some minor related changes: - add deploy artifacts (ISOs, qcow2 files) to .gitignore, also used to filter-out such files during rsync to cfg01; - remove obsolete Fuel patches (old f_repos mechanism); - rename "reclass-system-salt-model" submodule; Change-Id: I6210d80d41010b2802e4f1b31acf249a18db7963 Signed-off-by: Alexandru Avadanii --- mcp/patches/0001-opendaylight-reclass-system.patch | 94 ----------------- mcp/patches/Makefile | 117 +++++++++++++++++++++ mcp/patches/README.md | 103 ++++++++++++++++++ mcp/patches/config.mk | 23 ++++ mcp/patches/patches.list | 1 - .../0001-Bring-in-opendaylight-support.patch | 106 +++++++++++++++++++ mcp/scripts/salt.sh | 44 ++++++-- 7 files changed, 383 insertions(+), 105 deletions(-) delete mode 100644 mcp/patches/0001-opendaylight-reclass-system.patch create mode 100644 mcp/patches/Makefile create mode 100644 mcp/patches/README.md create mode 100644 mcp/patches/config.mk create mode 100644 mcp/patches/reclass-system-salt-model/0001-Bring-in-opendaylight-support.patch (limited to 'mcp') diff --git a/mcp/patches/0001-opendaylight-reclass-system.patch b/mcp/patches/0001-opendaylight-reclass-system.patch deleted file mode 100644 index f8c986be6..000000000 --- a/mcp/patches/0001-opendaylight-reclass-system.patch +++ /dev/null @@ -1,94 +0,0 @@ -From: Michael Polenchuk -Date: Thu, 29 Jun 2017 12:22:42 +0400 -Subject: [PATCH] Bring in opendaylight support - -Change-Id: I3efec9a8b586a6c75b1c1635ad2a7024d73d9ad2 - -diff --git a/neutron/control/opendaylight/cluster.yml b/neutron/control/opendaylight/cluster.yml -new file mode 100644 -index 0000000..4133650 ---- /dev/null -+++ b/neutron/control/opendaylight/cluster.yml -@@ -0,0 +1,19 @@ -+classes: -+- service.keepalived.cluster.single -+- service.haproxy.proxy.single -+- service.neutron.control.cluster -+- system.haproxy.proxy.listen.openstack.neutron -+- system.neutron.control.openvswitch.cluster -+parameters: -+ neutron: -+ server: -+ backend: -+ engine: opendaylight -+ host: ${_param:opendaylight_service_host} -+ rest_api_port: 8282 -+ user: admin -+ password: admin -+ ovsdb_connection: tcp:127.0.0.1:6639 -+ mechanism: -+ ovs: -+ driver: opendaylight_v2 -diff --git a/neutron/control/opendaylight/single.yml b/neutron/control/opendaylight/single.yml -new file mode 100644 -index 0000000..19af585 ---- /dev/null -+++ b/neutron/control/opendaylight/single.yml -@@ -0,0 +1,16 @@ -+classes: -+- service.neutron.control.single -+- system.neutron.control.openvswitch.single -+parameters: -+ neutron: -+ server: -+ backend: -+ engine: opendaylight -+ host: ${_param:opendaylight_service_host} -+ rest_api_port: 8282 -+ user: admin -+ password: admin -+ ovsdb_connection: tcp:127.0.0.1:6639 -+ mechanism: -+ ovs: -+ driver: opendaylight_v2 -diff --git a/neutron/gateway/opendaylight/single.yml b/neutron/gateway/opendaylight/single.yml -new file mode 100644 -index 0000000..29c8d95 ---- /dev/null -+++ b/neutron/gateway/opendaylight/single.yml -@@ -0,0 +1,8 @@ -+classes: -+- service.neutron.gateway.single -+parameters: -+ neutron: -+ gateway: -+ backend: -+ engine: opendaylight -+ ovsdb_connection: tcp:127.0.0.1:6639 -diff --git a/opendaylight/server/single.yml b/opendaylight/server/single.yml -new file mode 100644 -index 0000000..3d11872 ---- /dev/null -+++ b/opendaylight/server/single.yml -@@ -0,0 +1,2 @@ -+classes: -+- service.opendaylight.server.single -diff --git a/reclass/storage/system/opendaylight_control_single.yml b/reclass/storage/system/opendaylight_control_single.yml -new file mode 100644 -index 0000000..ad809f5 ---- /dev/null -+++ b/reclass/storage/system/opendaylight_control_single.yml -@@ -0,0 +1,13 @@ -+parameters: -+ _param: -+ opendaylight_control_hostname: odl01 -+ reclass: -+ storage: -+ node: -+ opendaylight_control_node01: -+ name: ${_param:opendaylight_control_hostname} -+ domain: ${_param:cluster_domain} -+ params: -+ salt_master_host: ${_param:reclass_config_master} -+ linux_system_codename: ${_param:linux_system_codename} -+ single_address: ${_param:opendaylight_service_host} diff --git a/mcp/patches/Makefile b/mcp/patches/Makefile new file mode 100644 index 000000000..6a176db8d --- /dev/null +++ b/mcp/patches/Makefile @@ -0,0 +1,117 @@ +############################################################################## +# Copyright (c) 2015,2016,2017 Ericsson AB, Enea AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# Alexandru.Avadanii@enea.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +include config.mk + +SHELL = /bin/sh +FPATCHES = $(shell find ${F_PATCH_DIR} -name '*.patch') + +# NOTE: Mechanism overview is presented in ./README.md. + +# Submodule consistent states: +# - NOT initialized (submodule trees are not populated at all); +# - initialized, bound to saved commits; +# - initialized, tracking remote origin (only for FUEL_TRACK_REMOTES); +# - patched (local patches are applied); + +# In order to keep things sort of separate, we should only pass up (to main +# Makefile) the fully-patched repos, and gather any fingerprinting info here. + +# Fuel@OPNFV relies on upstream git repos (one per component) in 1 of 2 ways: +# - pinned down to tag objects (e.g. "9.0.1") +# - tracking upstream remote HEAD on a stable or master branch +# FIXME(alav): Should we support mixed cases? (e.g. pin down only fuel-main) + +# To enable remote tracking, set the following var to any non-empty string. +# Leaving this var empty will bind each git submodule to its saved commit. +FUEL_TRACK_REMOTES ?= yes + +.PHONY: all +all: release + +############################################################################## +# git submodule operations - to be used stand-alone or from parent Makefile +############################################################################## + +# Fetch & update git submodules, checkout remote HEAD or saved commit +# Also gather fingerprints for parent gitinfo_fuel.txt. +.PHONY: sub +sub: .cachefuelinfo + +.cachefuelinfo: + @if [ -n "${FUEL_TRACK_REMOTES}" ]; then \ + git submodule update --init --remote 2>/dev/null; \ + else \ + git submodule update --init 2>/dev/null; \ + fi + @touch $@ + +# Generate patches from submodules +.PHONY: patches-export +patches-export: sub + @git submodule -q foreach ' \ + SUB_DIR=${F_PATCH_DIR}/$$name; \ + git tag | awk "!/root/ && /${F_OPNFV_TAG}-fuel/" | while read F_TAG; do \ + SUB_FEATURE=`dirname $${F_TAG#${F_OPNFV_TAG}-fuel/}`; \ + echo "`tput setaf 2`-- exporting $$name ($$F_TAG)`tput sgr0`"; \ + mkdir -p $$SUB_DIR/$${SUB_FEATURE} && \ + git format-patch --no-signature --ignore-space-at-eol \ + -o $$SUB_DIR/$$SUB_FEATURE -N $$F_TAG-root..$$F_TAG; \ + sed -i -e "1{/From: /!d}" -e "s/[[:space:]]*$$//" \ + $$SUB_DIR/$$SUB_FEATURE/*.patch; \ + done' + +# Apply patches from patch/* to respective submodules +# We rely on `make sub` and/or `make clean` to checkout correct base +.PHONY: patches-import +patches-import: sub .cachepatched + +.cachepatched: ${FPATCHES} + @$(MAKE) clean + @git submodule -q foreach ' \ + SUB_DIR=${F_PATCH_DIR}/$$name; mkdir -p $$SUB_DIR && \ + git tag ${F_OPNFV_TAG}-root && \ + git checkout -q -b opnfv-fuel && \ + find $$SUB_DIR -type d | sort | while read p_dir; do \ + SUB_PATCHES=$$(ls $$p_dir/*.patch 2>/dev/null); \ + if [ -n "$$SUB_PATCHES" ]; then \ + SUB_FEATURE=$${p_dir#$$SUB_DIR}; \ + SUB_TAG=${F_OPNFV_TAG}-fuel$$SUB_FEATURE/patch; \ + echo "`tput setaf 2`-- patching $$name ($$SUB_TAG)`tput sgr0`";\ + git tag $$SUB_TAG-root && \ + git am -3 --whitespace=nowarn --patch-format=mbox \ + --committer-date-is-author-date $$SUB_PATCHES && \ + git tag $$SUB_TAG || exit 1; \ + fi \ + done && \ + git tag ${F_OPNFV_TAG}' + @touch $@ + +# Clean any changes made to submodules, checkout upstream Fuel root commit +.PHONY: clean +clean: + @cd ${F_GIT_ROOT} && git submodule -q foreach ' \ + git am -q --abort > /dev/null 2>&1; \ + git checkout -q -f ${F_OPNFV_TAG}-root > /dev/null 2>&1; \ + git branch -q -D opnfv-fuel > /dev/null 2>&1; \ + git tag | grep ${F_OPNFV_TAG} | xargs git tag -d > /dev/null 2>&1; \ + git reset -q --hard HEAD; \ + git clean -xdff' + @rm -f .cachepatched + +.PHONY: deepclean +deepclean: clean + @git submodule deinit -f ${F_GIT_ROOT} + @rm -f .cache* + +.PHONY: release +release: sub + $(MAKE) -f Makefile patches-import diff --git a/mcp/patches/README.md b/mcp/patches/README.md new file mode 100644 index 000000000..eab4d64e3 --- /dev/null +++ b/mcp/patches/README.md @@ -0,0 +1,103 @@ +Fuel@OPNFV submodule fetching and patching +========================================== + +This directory holds submodule fetching/patching scripts, intended for +working with upstream Fuel/MCP components (e.g.: reclass-system-salt-model) in +developing/applying OPNFV patches (backports, custom fixes etc.). + +The scripts should be friendly to the following 2 use-cases: + - development work: easily cloning, binding repos to specific commits, + remote tracking, patch development etc.; + - to provide parent build scripts an easy method of tracking upstream + references and applying OPNFV patches on top; + +Also, we need to support at least the following modes of operations: + - submodule bind - each submodule patches will be based on the commit ID + saved in the .gitmodules config file; + - remote tracking - each submodule will sync with the upstream remote + and patches will be applied on top of //HEAD; + +Workflow (development) +---------------------- +The standard development workflow should look as follows: + +1. Decide whether remote tracking should be active or not: + NOTE: Setting the following var to any non-empty str enables remote track. + NOTE: Leaving unset will enable remote track for anything but stable branch. + + $ export FUEL_TRACK_REMOTES="" + +2. All Fuel sub-projects are registered as submodules. To initialize them, call: + If remote tracking is active, upstream remote is queried and latest remote + branch HEAD is fetched. Otherwise, checkout commit IDs from .gitmodules. + + $ make sub + +3. Apply patches from `patches//*` to respective submodules via: + + $ make patches-import + + This will result in creation of: + - a tag called `${FUEL_MAIN_TAG}-opnfv-root` at the same commit as Fuel@OPNFV + upstream reference (bound to git submodule OR tracking remote HEAD); + - a new branch `opnfv-fuel` which will hold all the OPNFV patches, + each patch is applied on this new branch with `git-am`; + - a tag called `${FUEL_MAIN_TAG}-opnfv` at `opnfv-fuel/HEAD`; + +4. Modify sub-projects for whatever you need. + Commit your changes when you want them taken into account in the build. + +5. Re-create patches via: + + $ make patches-export + + Each commit on `opnfv-fuel` branch of each subproject will be + exported to `patches/subproject/` via `git format-patch`. + + NOTE: Only commit (-f) submodules when you need to bump upstream ref. + NOTE: DO NOT commit patched submodules! + +6. Clean workbench branches and tags with: + + $ make clean + +7. De-initialize submodules and force a clean clone with: + + $ make deepclean + +Sub-project maintenance +----------------------- +1. Adding a new submodule + If you need to add another subproject, you can do it with `git submodule`. + Make sure that you specify branch (with `-b`), short name (with `--name`): + + $ git submodule -b master add --name reclass-system-salt-model \ + https://github.com/Mirantis/reclass-system-salt-model \ + relative/path/to/submodule + +2. Working with remote tracking for upgrading Fuel components + Enable remote tracking as described above, which at `make sub` will update + ALL submodules (e.g. reclass-system-salt-model) to remote branch (set in + .gitmodules) HEAD. + + * If upstream has NOT already tagged a new version, we can still work on + our patches, make sure they apply etc., then check for new upstream + changes (and that our patches still apply on top of them) by: + + $ make deepclean patches-import + + * If upstream has already tagged a new version we want to pick up, checkout + the new tag in each submodule: + + $ git submodule foreach 'git checkout ' + + * Once satisfied with the patch and submodule changes, commit them: + - enforce FUEL_TRACK_REMOTES to "yes" if you want to constatly use the + latest remote branch HEAD (as soon as upstream pushes a change on that + branch, our next build will automatically include it - risk of our + patches colliding with new upstream changes); + - stage patch changes if any; + - if submodule tags have been updated (relevant when remote tracking is + disabled, i.e. we have a stable upstream baseline), add submodules: + + $ make deepclean sub && git add -f relative/path/to/submodule diff --git a/mcp/patches/config.mk b/mcp/patches/config.mk new file mode 100644 index 000000000..260cbf829 --- /dev/null +++ b/mcp/patches/config.mk @@ -0,0 +1,23 @@ +############################################################################## +# Copyright (c) 2015,2016,2017 Ericsson AB, Enea AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +############################################################################## +# Components pinning / remote tracking +############################################################################## + +# git submodule & patch locations for Fuel components +F_GIT_ROOT := $(shell git rev-parse --show-toplevel) +F_GIT_DIR := $(shell git rev-parse --git-dir) +F_PATCH_DIR := $(shell pwd) +F_OPNFV_TAG := master-opnfv + +# for the patches applying purposes (empty git config in docker build container) +export GIT_COMMITTER_NAME?=Fuel OPNFV +export GIT_COMMITTER_EMAIL?=fuel@opnfv.org diff --git a/mcp/patches/patches.list b/mcp/patches/patches.list index 0a84657d2..0542433c7 100644 --- a/mcp/patches/patches.list +++ b/mcp/patches/patches.list @@ -1,2 +1 @@ /usr/share/salt-formulas/env: 0002-opendaylight-formula-neutron.patch -/srv/salt/reclass/classes/system: 0001-opendaylight-reclass-system.patch diff --git a/mcp/patches/reclass-system-salt-model/0001-Bring-in-opendaylight-support.patch b/mcp/patches/reclass-system-salt-model/0001-Bring-in-opendaylight-support.patch new file mode 100644 index 000000000..15f23db34 --- /dev/null +++ b/mcp/patches/reclass-system-salt-model/0001-Bring-in-opendaylight-support.patch @@ -0,0 +1,106 @@ +From: Michael Polenchuk +Date: Thu, 29 Jun 2017 12:22:42 +0400 +Subject: [PATCH] Bring in opendaylight support + +Change-Id: I3efec9a8b586a6c75b1c1635ad2a7024d73d9ad2 +--- + neutron/control/opendaylight/cluster.yml | 19 +++++++++++++++++++ + neutron/control/opendaylight/single.yml | 16 ++++++++++++++++ + neutron/gateway/opendaylight/single.yml | 8 ++++++++ + opendaylight/server/single.yml | 2 ++ + .../storage/system/opendaylight_control_single.yml | 15 +++++++++++++++ + 5 files changed, 60 insertions(+) + create mode 100644 neutron/control/opendaylight/cluster.yml + create mode 100644 neutron/control/opendaylight/single.yml + create mode 100644 neutron/gateway/opendaylight/single.yml + create mode 100644 opendaylight/server/single.yml + create mode 100644 reclass/storage/system/opendaylight_control_single.yml + +diff --git a/neutron/control/opendaylight/cluster.yml b/neutron/control/opendaylight/cluster.yml +new file mode 100644 +index 0000000..4133650 +--- /dev/null ++++ b/neutron/control/opendaylight/cluster.yml +@@ -0,0 +1,19 @@ ++classes: ++- service.keepalived.cluster.single ++- service.haproxy.proxy.single ++- service.neutron.control.cluster ++- system.haproxy.proxy.listen.openstack.neutron ++- system.neutron.control.openvswitch.cluster ++parameters: ++ neutron: ++ server: ++ backend: ++ engine: opendaylight ++ host: ${_param:opendaylight_service_host} ++ rest_api_port: 8282 ++ user: admin ++ password: admin ++ ovsdb_connection: tcp:127.0.0.1:6639 ++ mechanism: ++ ovs: ++ driver: opendaylight_v2 +diff --git a/neutron/control/opendaylight/single.yml b/neutron/control/opendaylight/single.yml +new file mode 100644 +index 0000000..19af585 +--- /dev/null ++++ b/neutron/control/opendaylight/single.yml +@@ -0,0 +1,16 @@ ++classes: ++- service.neutron.control.single ++- system.neutron.control.openvswitch.single ++parameters: ++ neutron: ++ server: ++ backend: ++ engine: opendaylight ++ host: ${_param:opendaylight_service_host} ++ rest_api_port: 8282 ++ user: admin ++ password: admin ++ ovsdb_connection: tcp:127.0.0.1:6639 ++ mechanism: ++ ovs: ++ driver: opendaylight_v2 +diff --git a/neutron/gateway/opendaylight/single.yml b/neutron/gateway/opendaylight/single.yml +new file mode 100644 +index 0000000..29c8d95 +--- /dev/null ++++ b/neutron/gateway/opendaylight/single.yml +@@ -0,0 +1,8 @@ ++classes: ++- service.neutron.gateway.single ++parameters: ++ neutron: ++ gateway: ++ backend: ++ engine: opendaylight ++ ovsdb_connection: tcp:127.0.0.1:6639 +diff --git a/opendaylight/server/single.yml b/opendaylight/server/single.yml +new file mode 100644 +index 0000000..3d11872 +--- /dev/null ++++ b/opendaylight/server/single.yml +@@ -0,0 +1,2 @@ ++classes: ++- service.opendaylight.server.single +diff --git a/reclass/storage/system/opendaylight_control_single.yml b/reclass/storage/system/opendaylight_control_single.yml +new file mode 100644 +index 0000000..ad809f5 +--- /dev/null ++++ b/reclass/storage/system/opendaylight_control_single.yml +@@ -0,0 +1,13 @@ ++parameters: ++ _param: ++ opendaylight_control_hostname: odl01 ++ reclass: ++ storage: ++ node: ++ opendaylight_control_node01: ++ name: ${_param:opendaylight_control_hostname} ++ domain: ${_param:cluster_domain} ++ params: ++ salt_master_host: ${_param:reclass_config_master} ++ linux_system_codename: ${_param:linux_system_codename} ++ single_address: ${_param:opendaylight_service_host} diff --git a/mcp/scripts/salt.sh b/mcp/scripts/salt.sh index 4c6cbf782..3d0fb9117 100755 --- a/mcp/scripts/salt.sh +++ b/mcp/scripts/salt.sh @@ -1,34 +1,58 @@ #!/bin/bash +############################################################################## +# Copyright (c) 2017 Mirantis Inc., Enea AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## # # Deploy Salt Master # +F_GIT_ROOT=$(git rev-parse --show-toplevel) +SALT_MASTER_USER=${SALT_MASTER_USER:-ubuntu} +SSH_SALT="${SALT_MASTER_USER}@${SALT_MASTER}" +OPNFV_TMP_DIR="/home/${SALT_MASTER_USER}/fuel" +OPNFV_FUEL_DIR="/root/fuel" + +# patch reclass-system-salt-model locally before copying it over +make -C "${F_GIT_ROOT}/mcp/patches" patches-import + +# push to cfg01 current git repo first (including submodules), at ~ubuntu/fuel +# later we move it to ~root/fuel and delete the temporary clone +rsync -Erl --delete -e "ssh ${SSH_OPTS}" \ + --exclude-from="${F_GIT_ROOT}/.gitignore" \ + "${F_GIT_ROOT}/" "${SSH_SALT}:$(basename "${OPNFV_TMP_DIR}")/" + # ssh to cfg01 # shellcheck disable=SC2086,2087 -ssh ${SSH_OPTS} "ubuntu@${SALT_MASTER}" bash -s << SALT_INSTALL_END +ssh ${SSH_OPTS} "${SSH_SALT}" bash -s << SALT_INSTALL_END sudo -i echo -n 'Checking out cloud-init has finished running ...' while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo -n '.'; sleep 1; done echo ' done' - apt-get install -y git curl subversion + DEBIAN_FRONTEND=noninteractive apt-get install -y git curl subversion - svn export --force https://github.com/salt-formulas/salt-formulas/trunk/deploy/scripts /srv/salt/scripts - git clone --depth=1 --recurse-submodules https://git.opnfv.org/fuel - ln -s /root/fuel/mcp/reclass /srv/salt/reclass + svn export --force \ + https://github.com/salt-formulas/salt-formulas/trunk/deploy/scripts /srv/salt/scripts + mv ${OPNFV_TMP_DIR} ${OPNFV_FUEL_DIR} && chown -R root.root ${OPNFV_FUEL_DIR} + ln -s ${OPNFV_FUEL_DIR}/mcp/reclass /srv/salt/reclass mkdir -p /usr/share/salt-formulas/reclass - cp -r /root/fuel/mcp/metadata/service /usr/share/salt-formulas/reclass - cd /srv/salt/reclass/classes/service && ln -s /usr/share/salt-formulas/reclass/service/opendaylight - cd /root/fuel/mcp/patches && ./patch.sh patches.list reclass + cp -r ${OPNFV_FUEL_DIR}/mcp/metadata/service /usr/share/salt-formulas/reclass + cd /srv/salt/reclass/classes/service && \ + ln -s /usr/share/salt-formulas/reclass/service/opendaylight + cd ${OPNFV_FUEL_DIR}/mcp/patches && ./patch.sh patches.list reclass cd /srv/salt/scripts MASTER_HOSTNAME=cfg01.${CLUSTER_DOMAIN} DISTRIB_REVISION=nightly ./salt-master-init.sh salt-key -Ay - cp -r /root/fuel/mcp/salt-formulas/* /usr/share/salt-formulas/env - cd /root/fuel/mcp/patches && ./patch.sh patches.list formulas + cp -r ${OPNFV_FUEL_DIR}/mcp/salt-formulas/* /usr/share/salt-formulas/env + cd ${OPNFV_FUEL_DIR}/mcp/patches && ./patch.sh patches.list formulas salt-call state.apply salt salt '*' state.apply salt | fgrep -q 'No response' && salt '*' state.apply salt -- cgit 1.2.3-korg