M/N upgrade fix galera restart.

We have to recreate the /var/lib/mysql directory on all controller node,
not just the boostrap node for the cluster to be able to restart.

Adding a warning on the fact that those script are local and know
nothing about the good upgrade state of the other nodes.

Closes-Bug: 1612642
Change-Id: I48e2812d7df80bbf2db53a8b71dc434d4209a160

1 files changed, 16 insertions, 11 deletions

diff --git a/extraconfig/tasks/major_upgrade_controller_pacemaker_1.sh b/extraconfig/tasks/major_upgrade_controller_pacemaker_1.sh
index d67d5a1a..0ef1f776 100755
--- a/extraconfig/tasks/major_upgrade_controller_pacemaker_1.sh
+++ b/extraconfig/tasks/major_upgrade_controller_pacemaker_1.sh
@@ -155,17 +155,19 @@ wsrep_on = ON
 wsrep_cluster_address = gcomm://localhost
 EOF
 
-if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname)" ]; then
-    if [ $DO_MYSQL_UPGRADE -eq 1 ]; then
-        # Scripts run via heat have no HOME variable set and this confuses
-        # mysqladmin
-        export HOME=/root
-        mkdir /var/lib/mysql || /bin/true
-        chown mysql:mysql /var/lib/mysql
-        chmod 0755 /var/lib/mysql
-        restorecon -R /var/lib/mysql/
-        mysql_install_db --datadir=/var/lib/mysql --user=mysql
-        chown -R mysql:mysql /var/lib/mysql/
+if [ $DO_MYSQL_UPGRADE -eq 1 ]; then
+    # Scripts run via heat have no HOME variable set and this confuses
+    # mysqladmin
+    export HOME=/root
+
+    mkdir /var/lib/mysql || /bin/true
+    chown mysql:mysql /var/lib/mysql
+    chmod 0755 /var/lib/mysql
+    restorecon -R /var/lib/mysql/
+    mysql_install_db --datadir=/var/lib/mysql --user=mysql
+    chown -R mysql:mysql /var/lib/mysql/
+
+    if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname)" ]; then
         mysqld_safe --wsrep-new-cluster &
         # We have a populated /root/.my.cnf with root/password here so
         # we need to temporarily rename it because the newly created
@@ -182,6 +184,9 @@ fi
 
 # If we reached here without error we can safely blow away the origin
 # mysql dir from every controller
+
+# TODO: What if the upgrade fails on the bootstrap node, but not on
+# this controller.  Data may be lost.
 if [ $DO_MYSQL_UPGRADE -eq 1 ]; then
     rm -r $MYSQL_TEMP_UPGRADE_BACKUP_DIR
 fi