diff options
Diffstat (limited to 'extraconfig')
| -rwxr-xr-x | extraconfig/tasks/pacemaker_resource_restart.sh | 5 | ||||
| -rwxr-xr-x | extraconfig/tasks/yum_update.sh | 16 |
2 files changed, 16 insertions, 5 deletions
diff --git a/extraconfig/tasks/pacemaker_resource_restart.sh b/extraconfig/tasks/pacemaker_resource_restart.sh index 4e8b20fd..12201097 100755 --- a/extraconfig/tasks/pacemaker_resource_restart.sh +++ b/extraconfig/tasks/pacemaker_resource_restart.sh @@ -26,6 +26,7 @@ function check_resource { sleep $check_interval else echo "$service has $state" + timeout -k 10 $timeout crm_resource --wait return fi done @@ -50,7 +51,7 @@ if [ "$pacemaker_status" = "active" -a \ pcs resource disable httpd check_resource httpd stopped 300 pcs resource disable openstack-keystone - check_resource openstack-keystone stopped 1200 + check_resource openstack-keystone stopped 1800 if pcs status | grep haproxy-clone; then pcs resource restart haproxy-clone @@ -62,7 +63,7 @@ if [ "$pacemaker_status" = "active" -a \ pcs resource restart galera-master pcs resource enable openstack-keystone - check_resource openstack-keystone started 300 + check_resource openstack-keystone started 1800 pcs resource enable httpd check_resource httpd started 800 diff --git a/extraconfig/tasks/yum_update.sh b/extraconfig/tasks/yum_update.sh index e32369e1..c6313d9d 100755 --- a/extraconfig/tasks/yum_update.sh +++ b/extraconfig/tasks/yum_update.sh @@ -24,6 +24,7 @@ update_identifier=${update_identifier//[^a-zA-Z0-9-_]/} # seconds to wait for this node to rejoin the cluster after update cluster_start_timeout=600 galera_sync_timeout=360 +cluster_settle_timeout=1800 timestamp_file="$timestamp_dir/$update_identifier" if [[ -a "$timestamp_file" ]]; then @@ -122,13 +123,16 @@ openstack-nova-scheduler" echo "Setting resource start/stop timeouts" for service in $SERVICES; do - pcs -f $pacemaker_dumpfile resource update $service op start timeout=100s op stop timeout=100s + pcs -f $pacemaker_dumpfile resource update $service op start timeout=200s op stop timeout=200s done # mongod start timeout is higher, setting only stop timeout - pcs -f $pacemaker_dumpfile resource update mongod op stop timeout=100s + pcs -f $pacemaker_dumpfile resource update mongod op start timeout=370s op stop timeout=200s echo "Applying new Pacemaker config" - pcs cluster cib-push $pacemaker_dumpfile + if ! pcs cluster cib-push $pacemaker_dumpfile; then + echo "ERROR failed to apply new pacemaker config" + exit 1 + fi echo "Pacemaker running, stopping cluster node and doing full package update" node_count=$(pcs status xml | grep -o "<nodes_configured.*/>" | grep -o 'number="[0-9]*"' | grep -o "[0-9]*") @@ -188,6 +192,12 @@ if [[ "$pacemaker_status" == "active" ]] ; then fi done + echo "Waiting for pacemaker cluster to settle" + if ! timeout -k 10 $cluster_settle_timeout crm_resource --wait; then + echo "ERROR timed out while waiting for the cluster to settle" + exit 1 + fi + pcs status else |