From 257855082abd78ac81407b8b11f8325fab649e8c Mon Sep 17 00:00:00 2001 From: Michele Baldessari Date: Fri, 24 Jun 2016 21:07:27 +0200 Subject: Disable stonith temporarily during upgrades It is best if we disable stonith if a cluster has it configured and on, before we call "pcs cluster stop --all", because should a service fail to stop for whatever reason, pacemaker will fence the node where it happened. This is something that we unlikely want during an upgrade as it will make things worse. Once the cluster is stopped we can reenable stonith (if it was enabled to start with) in the CIB while the cluster is shut down. Closes-Bug: #1596065 Change-Id: I38dcacfabc44539aab1f7da85168fe44a1b43a51 --- extraconfig/tasks/major_upgrade_controller_pacemaker_1.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'extraconfig/tasks') diff --git a/extraconfig/tasks/major_upgrade_controller_pacemaker_1.sh b/extraconfig/tasks/major_upgrade_controller_pacemaker_1.sh index f5399222..72bb81f5 100755 --- a/extraconfig/tasks/major_upgrade_controller_pacemaker_1.sh +++ b/extraconfig/tasks/major_upgrade_controller_pacemaker_1.sh @@ -9,6 +9,14 @@ if pcs status 2>&1 | grep -E '(cluster is not currently running)|(OFFLINE:)'; th exit 1 fi + +# We want to disable fencing during the cluster --stop as it might fence +# nodes where a service fails to stop, which could be fatal during an upgrade +# procedure. So we remember the stonith state. If it was enabled we reenable it +# at the end of this script +STONITH_STATE=$(pcs property show stonith-enabled | grep "stonith-enabled" | awk '{ print $2 }') +pcs property set stonith-enabled=false + if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname)" ]; then pcs resource disable httpd check_resource httpd stopped 1800 @@ -49,6 +57,12 @@ done yum -y install python-zaqarclient # needed for os-collect-config yum -y -q update + +# Let's reset the stonith back to true if it was true, before starting the cluster +if [ $STONITH_STATE == "true" ]; then + pcs -f /var/lib/pacemaker/cib/cib.xml property set stonith-enabled=true +fi + # Pin messages sent to compute nodes to kilo, these will be upgraded later crudini --set /etc/nova/nova.conf upgrade_levels compute "$upgrade_level_nova_compute" # https://bugzilla.redhat.com/show_bug.cgi?id=1284047 -- cgit 1.2.3-korg