diff options
author | 2016-11-10 19:00:08 +0000 | |
---|---|---|
committer | 2016-11-10 19:00:08 +0000 | |
commit | 9b4a981f955ea4a187c92e24c980f45c42f9a02a (patch) | |
tree | b2e2d3c9f836823a2e89a7224fcedc14fb513810 /extraconfig/tasks/major_upgrade_controller_pacemaker_3.sh | |
parent | 64a277323c580b16196b1d1805120ba35de1690c (diff) | |
parent | dde12b075ff51d4def4f49e635dd390a7f1f2cac (diff) |
Merge "Fix race during major-upgrade-pacemaker step"
Diffstat (limited to 'extraconfig/tasks/major_upgrade_controller_pacemaker_3.sh')
-rwxr-xr-x | extraconfig/tasks/major_upgrade_controller_pacemaker_3.sh | 69 |
1 files changed, 60 insertions, 9 deletions
diff --git a/extraconfig/tasks/major_upgrade_controller_pacemaker_3.sh b/extraconfig/tasks/major_upgrade_controller_pacemaker_3.sh index d2cb9553..37061512 100755 --- a/extraconfig/tasks/major_upgrade_controller_pacemaker_3.sh +++ b/extraconfig/tasks/major_upgrade_controller_pacemaker_3.sh @@ -2,16 +2,67 @@ set -eu -start_or_enable_service rabbitmq -check_resource rabbitmq started 600 +cluster_form_timeout=600 +cluster_settle_timeout=1800 +galera_sync_timeout=600 + +if [[ -n $(is_bootstrap_node) ]]; then + pcs cluster start --all + + tstart=$(date +%s) + while pcs status 2>&1 | grep -E '(cluster is not currently running)|(OFFLINE:)'; do + sleep 5 + tnow=$(date +%s) + if (( tnow-tstart > cluster_form_timeout )) ; then + echo_error "ERROR: timed out forming the cluster" + exit 1 + fi + done + + if ! timeout -k 10 $cluster_settle_timeout crm_resource --wait; then + echo_error "ERROR: timed out waiting for cluster to finish transition" + exit 1 + fi + + for vip in $(pcs resource show | grep ocf::heartbeat:IPaddr2 | grep Stopped | awk '{ print $1 }'); do + pcs resource enable $vip + check_resource_pacemaker $vip started 60 + done +fi + +start_or_enable_service galera +check_resource galera started 600 start_or_enable_service redis check_resource redis started 600 -start_or_enable_service openstack-cinder-volume -check_resource openstack-cinder-volume started 600 +# We need mongod which is now a systemd service up and running before calling +# ceilometer-dbsync. There is still a race here: mongod might not be up on all nodes +# so ceilometer-dbsync will fail a couple of times before that. As it retries indefinitely +# we should be good. +# Due to LP Bug https://bugs.launchpad.net/tripleo/+bug/1627254 am using systemctl directly atm +systemctl start mongod +check_resource mongod started 600 -# start httpd so keystone is available for gnocchi -# upgrade to run. -systemctl start httpd +if [[ -n $(is_bootstrap_node) ]]; then + tstart=$(date +%s) + while ! clustercheck; do + sleep 5 + tnow=$(date +%s) + if (( tnow-tstart > galera_sync_timeout )) ; then + echo_error "ERROR galera sync timed out" + exit 1 + fi + done -# Swift isn't controled by pacemaker -systemctl_swift start + # Run all the db syncs + # TODO: check if this can be triggered in puppet and removed from here + ceilometer-dbsync --config-file=/etc/ceilometer/ceilometer.conf + cinder-manage db sync + glance-manage --config-file=/etc/glance/glance-registry.conf db_sync + heat-manage --config-file /etc/heat/heat.conf db_sync + keystone-manage db_sync + neutron-db-manage --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/plugin.ini upgrade head + nova-manage db sync + nova-manage api_db sync + nova-manage db online_data_migrations + sahara-db-manage --config-file /etc/sahara/sahara.conf upgrade head +fi |