From 8d0731eecbb4c7f78e23c8a7e93004da7593afbe Mon Sep 17 00:00:00 2001 From: Alexandru Avadanii Date: Thu, 24 Aug 2017 17:10:33 +0200 Subject: p/fuel: Rebase after MaaS retry got upstream Change-Id: I321fcb4aa77ebcb4595a9468c2257fef8b9a8ed6 Signed-off-by: Alexandru Avadanii --- .../0018-MaaS-commissioning-deployment-retry.patch | 168 --------------------- 1 file changed, 168 deletions(-) delete mode 100644 patches/opnfv-fuel/0018-MaaS-commissioning-deployment-retry.patch (limited to 'patches/opnfv-fuel/0018-MaaS-commissioning-deployment-retry.patch') diff --git a/patches/opnfv-fuel/0018-MaaS-commissioning-deployment-retry.patch b/patches/opnfv-fuel/0018-MaaS-commissioning-deployment-retry.patch deleted file mode 100644 index 3a26d9b0..00000000 --- a/patches/opnfv-fuel/0018-MaaS-commissioning-deployment-retry.patch +++ /dev/null @@ -1,168 +0,0 @@ -From: Alexandru Avadanii -Date: Wed, 23 Aug 2017 04:23:26 +0200 -Subject: [PATCH] MaaS: commissioning/deployment retry - -While at it, parametrize max attempt number in maas state's "wait_for", -and reduce retries count for certain simpler tasks. - -Change-Id: I3ac2877719cdd32613bcf41186ebbb9f3f3aee93 -Signed-off-by: Alexandru Avadanii ---- - mcp/config/states/maas | 63 ++++++++++++++++------ - mcp/salt-formulas/maas/machines/delete.sls | 12 +++++ - .../maas/machines/mark_broken_fixed.sls | 12 +++++ - 3 files changed, 70 insertions(+), 17 deletions(-) - create mode 100644 mcp/salt-formulas/maas/machines/delete.sls - create mode 100644 mcp/salt-formulas/maas/machines/mark_broken_fixed.sls - -diff --git a/mcp/config/states/maas b/mcp/config/states/maas -index 0cf4f68..080bbf5 100755 ---- a/mcp/config/states/maas -+++ b/mcp/config/states/maas -@@ -1,17 +1,47 @@ - #!/bin/bash - function wait_for() { -+ local total_attempts=$1; shift - local cmdstr=$@ -- local total_attempts=360 - local sleep_time=10 -- local attempt=1 - echo "[NOTE] Waiting for cmd to return success: ${cmdstr}" -- while ((attempt <= total_attempts)); do -+ for attempt in $(seq "${total_attempts}"); do - eval "${cmdstr}" && break || true - echo -n '.'; sleep "${sleep_time}" -- ((attempt+=1)) - done - } - -+# Wait for MaaS commissioning/deploy to finish, retry on failure -+function maas_fixup() { -+ local statuscmd="salt 'mas01*' --out yaml state.apply maas.machines.status" -+ wait_for 180 "${statuscmd} | tee /dev/stderr | " \ -+ "grep -Eq '((Deployed|Ready): 5|status:Failed|status:Allocated)'" -+ local statusout=$(eval "${statuscmd}") -+ -+ local fcnodes=$(echo "${statusout}" | \ -+ grep -Po '(?<=system_id:)(.*)(?=,status:Failed commissioning)') -+ for node_system_id in ${fcnodes}; do -+ salt -C 'mas01*' state.apply maas.machines.delete \ -+ pillar="{'system_id': '${node_system_id}'}" -+ done -+ if [ -n "${fcnodes}" ]; then -+ salt -C 'mas01*' state.apply maas.machines -+ return 1 -+ fi -+ -+ local fdnodes=$(echo "${statusout}" | \ -+ grep -Po '(?<=system_id:)(.*)(?=,status:(Failed deployment|Allocated))') -+ for node_system_id in ${fdnodes}; do -+ salt -C 'mas01*' state.apply maas.machines.mark_broken_fixed \ -+ pillar="{'system_id': '${node_system_id}'}" -+ done -+ if [ -n "${fdnodes}" ]; then -+ salt -C 'mas01*' state.apply maas.machines.deploy -+ return 1 -+ fi -+ -+ return 0 -+} -+ - # MaaS rack/region controller, node commissioning - salt -C 'mas01*' cmd.run "add-apt-repository ppa:maas/stable" - -@@ -22,20 +52,18 @@ salt -C 'mas01*' state.apply maas.cluster - salt -C 'cfg01*' cmd.run \ - "route add -net 192.168.11.0/24 gw ${MAAS_IP:-192.168.10.2}" - --wait_for "salt -C 'mas01*' state.apply maas.region" -+wait_for 10 "salt -C 'mas01*' state.apply maas.region" - - salt -C 'mas01*' state.apply maas.machines --# TODO: relax cond, as this is not re-entrant (e.g. nodes already deployed) --wait_for "salt 'mas01*' --out yaml state.apply maas.machines.status | " \ -- "tee /dev/stderr | fgrep -q 'Ready: 5'" -+wait_for 10 maas_fixup - - # cleanup outdated salt keys --salt-key --out yaml | awk '!/^(minions|- cfg01|- mas01)/ {print $2}' | xargs -I{} salt-key -yd {} -+salt-key --out yaml | awk '!/^(minions|- cfg01|- mas01)/ {print $2}' | \ -+ xargs -I{} salt-key -yd {} - - # MaaS node deployment - salt -C 'mas01*' state.apply maas.machines.deploy --wait_for "salt 'mas01*' --out yaml state.apply maas.machines.status | " \ -- "tee /dev/stderr | fgrep -q 'Deployed: 5'" -+wait_for 10 maas_fixup - - salt -C 'mas01*' pillar.item\ - maas:region:admin:username \ -@@ -48,7 +76,7 @@ salt -C 'kvm*' pkg.install bridge-utils - salt -C 'kvm*' state.apply linux.network - salt -C 'kvm*' state.apply armband.bootstrap_script_arm64 - salt -C 'kvm*' system.reboot --wait_for "! salt '*' test.ping | tee /dev/stderr | fgrep -q 'Not connected'" -+wait_for 90 "! salt '*' test.ping | tee /dev/stderr | fgrep -q 'Not connected'" - - salt -C '* and not cfg01* and not mas01*' state.apply linux,ntp - -@@ -59,7 +87,8 @@ salt -C '* and not cfg01* and not mas01*' state.apply salt - salt -C 'kvm*' saltutil.sync_all - salt -C 'kvm*' state.sls salt.control - --vcp_nodes=$(salt --out yaml 'kvm01*' pillar.get salt:control:cluster:internal:node | awk '/\s+\w+:$/ {gsub(/:$/, "*"); print $1}') -+vcp_nodes=$(salt --out yaml 'kvm01*' pillar.get salt:control:cluster:internal:node | \ -+ awk '/\s+\w+:$/ {gsub(/:$/, "*"); print $1}') - - # Check all vcp nodes are available - rc=1 -@@ -71,9 +100,9 @@ while [ $rc -ne 0 ]; do - sleep 5 - done - --wait_for "salt -C '* and not cfg01* and not mas01*' ssh.set_auth_key ${SUDO_USER} \ -+wait_for 10 "salt -C '* and not cfg01* and not mas01*' ssh.set_auth_key ${SUDO_USER} \ - $(awk 'NR==1{print $2}' $(eval echo ~${SUDO_USER}/.ssh/authorized_keys))" - --wait_for "salt -C '* and not cfg01* and not mas01*' saltutil.sync_all" --salt -C '* and not cfg01* and not mas01*' state.apply salt --wait_for "salt -C '* and not cfg01* and not mas01*' state.apply linux,ntp" -+wait_for 10 "salt -C '* and not cfg01* and not mas01*' saltutil.sync_all" -+wait_for 10 "salt -C '* and not cfg01* and not mas01*' state.apply salt" -+wait_for 10 "salt -C '* and not cfg01* and not mas01*' state.apply linux,ntp" -diff --git a/mcp/salt-formulas/maas/machines/delete.sls b/mcp/salt-formulas/maas/machines/delete.sls -new file mode 100644 -index 0000000..306dbca ---- /dev/null -+++ b/mcp/salt-formulas/maas/machines/delete.sls -@@ -0,0 +1,12 @@ -+{%- from "maas/map.jinja" import region with context %} -+ -+maas_login_admin: -+ cmd.run: -+ - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials" -+ -+# TODO: implement delete_machine via _modules/maas.py -+delete_machine: -+ cmd.run: -+ - name: "maas login {{ region.admin.username }} http://{{ region.bind.host }}:5240/MAAS/api/2.0 - < /var/lib/maas/.maas_credentials && maas opnfv machine delete {{ pillar['system_id'] }}" -+ - require: -+ - cmd: maas_login_admin -diff --git a/mcp/salt-formulas/maas/machines/mark_broken_fixed.sls b/mcp/salt-formulas/maas/machines/mark_broken_fixed.sls -new file mode 100644 -index 0000000..874718f ---- /dev/null -+++ b/mcp/salt-formulas/maas/machines/mark_broken_fixed.sls -@@ -0,0 +1,12 @@ -+{%- from "maas/map.jinja" import region with context %} -+ -+maas_login_admin: -+ cmd.run: -+ - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials" -+ -+# TODO: implement mark_broken_fixed_machine via _modules/maas.py -+mark_broken_fixed_machine: -+ cmd.run: -+ - name: "maas login {{ region.admin.username }} http://{{ region.bind.host }}:5240/MAAS/api/2.0 - < /var/lib/maas/.maas_credentials && maas opnfv machine mark-broken {{ pillar['system_id'] }} && maas opnfv machine mark-fixed {{ pillar['system_id'] }}" -+ - require: -+ - cmd: maas_login_admin -- cgit 1.2.3-korg