From dde12b075ff51d4def4f49e635dd390a7f1f2cac Mon Sep 17 00:00:00 2001 From: Michele Baldessari Date: Wed, 9 Nov 2016 09:05:08 +0100 Subject: Fix race during major-upgrade-pacemaker step Currently when we call the major-upgrade step we do the following: """ ... if [[ -n $(is_bootstrap_node) ]]; then check_clean_cluster fi ... if [[ -n $(is_bootstrap_node) ]]; then migrate_full_to_ng_ha fi ... for service in $(services_to_migrate); do manage_systemd_service stop "${service%%-clone}" ... done """ The problem with the above code is that it is open to the following race condition: 1. Code gets run first on a non-bootstrap controller node so we start stopping a bunch of services 2. Pacemaker notices will notice that services are down and will mark the service as stopped 3. Code gets run on the bootstrap node (controller-0) and the check_clean_cluster function will fail and exit 4. Eventually also the script on the non-bootstrap controller node will timeout and exit because the cluster never shut down (it never actually started the shutdown because we failed at 3) Let's make sure we first only call the HA NG migration step as a separate heat step. Only afterwards we start shutting down the systemd services on all nodes. We also need to move the STONITH_STATE variable into a file because it is being used across two different scripts (1 and 2) and we need to store that state. Co-Authored-By: Athlan-Guyot Sofer Closes-Bug: #1640407 Change-Id: Ifb9b9e633fcc77604cca2590071656f4b2275c60 --- extraconfig/tasks/major_upgrade_check.sh | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) (limited to 'extraconfig/tasks/major_upgrade_check.sh') diff --git a/extraconfig/tasks/major_upgrade_check.sh b/extraconfig/tasks/major_upgrade_check.sh index b65f6915..8bdff5e7 100755 --- a/extraconfig/tasks/major_upgrade_check.sh +++ b/extraconfig/tasks/major_upgrade_check.sh @@ -18,14 +18,8 @@ check_pcsd() fi } -check_disk_for_mysql_dump() +mysql_need_update() { - # Where to backup current database if mysql need to be upgraded - MYSQL_BACKUP_DIR=/var/tmp/mysql_upgrade_osp - MYSQL_TEMP_UPGRADE_BACKUP_DIR=/var/lib/mysql-temp-upgrade-backup - # Spare disk ratio for extra safety - MYSQL_BACKUP_SIZE_RATIO=1.2 - # Shall we upgrade mysql data directory during the stack upgrade? if [ "$mariadb_do_major_upgrade" = "auto" ]; then ret=$(is_mysql_upgrade_needed) @@ -40,6 +34,17 @@ check_disk_for_mysql_dump() else DO_MYSQL_UPGRADE=1 fi +} + +check_disk_for_mysql_dump() +{ + # Where to backup current database if mysql need to be upgraded + MYSQL_BACKUP_DIR=/var/tmp/mysql_upgrade_osp + MYSQL_TEMP_UPGRADE_BACKUP_DIR=/var/lib/mysql-temp-upgrade-backup + # Spare disk ratio for extra safety + MYSQL_BACKUP_SIZE_RATIO=1.2 + + mysql_need_update if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname)" ]; then if [ $DO_MYSQL_UPGRADE -eq 1 ]; then -- cgit 1.2.3-korg