aboutsummaryrefslogtreecommitdiffstats
path: root/extraconfig/tasks
diff options
context:
space:
mode:
Diffstat (limited to 'extraconfig/tasks')
-rwxr-xr-xextraconfig/tasks/major_upgrade_ceph_mon.sh63
-rwxr-xr-xextraconfig/tasks/major_upgrade_controller_pacemaker_1.sh29
-rw-r--r--extraconfig/tasks/major_upgrade_pacemaker.yaml18
3 files changed, 98 insertions, 12 deletions
diff --git a/extraconfig/tasks/major_upgrade_ceph_mon.sh b/extraconfig/tasks/major_upgrade_ceph_mon.sh
new file mode 100755
index 00000000..38befbbf
--- /dev/null
+++ b/extraconfig/tasks/major_upgrade_ceph_mon.sh
@@ -0,0 +1,63 @@
+#!/bin/bash
+set -eu
+set -o pipefail
+
+echo INFO: starting $(basename "$0")
+
+# Exit if not running
+if ! pidof ceph-mon; then
+ echo INFO: ceph-mon is not running, skipping
+ exit 0
+fi
+
+# Exit if not Hammer
+INSTALLED_VERSION=$(ceph --version | awk '{print $3}')
+if ! [[ "$INSTALLED_VERSION" =~ ^0\.94.* ]]; then
+ echo INFO: version of Ceph installed is not 0.94, skipping
+ exit 0
+fi
+
+MON_PID=$(pidof ceph-mon)
+MON_ID=$(hostname -s)
+
+# Stop daemon using Hammer sysvinit script
+service ceph stop mon.${MON_ID}
+
+# Ensure it's stopped
+timeout 60 bash -c "while kill -0 ${MON_PID} 2> /dev/null; do
+ sleep 2;
+done"
+
+# Update to Jewel
+yum -y -q update ceph-mon
+
+# Restart/Exit if not on Jewel, only in that case we need the changes
+UPDATED_VERSION=$(ceph --version | awk '{print $3}')
+if [[ "$UPDATED_VERSION" =~ ^0\.94.* ]]; then
+ echo WARNING: Ceph was not upgraded, restarting daemons
+ service ceph start mon.${MON_ID}
+elif [[ "$UPDATED_VERSION" =~ ^10\.2.* ]]; then
+ echo INFO: Ceph was upgraded to Jewel
+
+ # RPM could own some of these but we can't take risks on the pre-existing files
+ for d in /var/lib/ceph/mon /var/log/ceph /var/run/ceph /etc/ceph; do
+ chown -R ceph:ceph $d
+ done
+
+ # Replay udev events with newer rules
+ udevadm trigger
+
+ # Enable systemd unit
+ systemctl enable ceph-mon.target
+ systemctl enable ceph-mon@${MON_ID}
+ systemctl start ceph-mon@${MON_ID}
+
+ # Wait for daemon to be back in the quorum
+ timeout 300 bash -c "until (ceph quorum_status | jq .quorum_names | grep -sq ${MON_ID}); do
+ echo Waiting for mon.${MON_ID} to re-join quorum;
+ sleep 10;
+ done"
+else
+ echo ERROR: Ceph was upgraded to an unknown release, daemon is stopped, need manual intervention
+ exit 1
+fi
diff --git a/extraconfig/tasks/major_upgrade_controller_pacemaker_1.sh b/extraconfig/tasks/major_upgrade_controller_pacemaker_1.sh
index 2ee473ce..0b702630 100755
--- a/extraconfig/tasks/major_upgrade_controller_pacemaker_1.sh
+++ b/extraconfig/tasks/major_upgrade_controller_pacemaker_1.sh
@@ -18,7 +18,7 @@ STONITH_STATE=$(pcs property show stonith-enabled | grep "stonith-enabled" | awk
pcs property set stonith-enabled=false
# If for some reason rpm-python are missing we want to error out early enough
-if [ ! rpm -q rpm-python &> /dev/null ]; then
+if ! rpm -q rpm-python &> /dev/null; then
echo_error "ERROR: upgrade cannot start without rpm-python installed"
exit 1
fi
@@ -155,17 +155,19 @@ wsrep_on = ON
wsrep_cluster_address = gcomm://localhost
EOF
-if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname)" ]; then
- if [ $DO_MYSQL_UPGRADE -eq 1 ]; then
- # Scripts run via heat have no HOME variable set and this confuses
- # mysqladmin
- export HOME=/root
- mkdir /var/lib/mysql || /bin/true
- chown mysql:mysql /var/lib/mysql
- chmod 0755 /var/lib/mysql
- restorecon -R /var/lib/mysql/
- mysql_install_db --datadir=/var/lib/mysql --user=mysql
- chown -R mysql:mysql /var/lib/mysql/
+if [ $DO_MYSQL_UPGRADE -eq 1 ]; then
+ # Scripts run via heat have no HOME variable set and this confuses
+ # mysqladmin
+ export HOME=/root
+
+ mkdir /var/lib/mysql || /bin/true
+ chown mysql:mysql /var/lib/mysql
+ chmod 0755 /var/lib/mysql
+ restorecon -R /var/lib/mysql/
+ mysql_install_db --datadir=/var/lib/mysql --user=mysql
+ chown -R mysql:mysql /var/lib/mysql/
+
+ if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname)" ]; then
mysqld_safe --wsrep-new-cluster &
# We have a populated /root/.my.cnf with root/password here so
# we need to temporarily rename it because the newly created
@@ -182,6 +184,9 @@ fi
# If we reached here without error we can safely blow away the origin
# mysql dir from every controller
+
+# TODO: What if the upgrade fails on the bootstrap node, but not on
+# this controller. Data may be lost.
if [ $DO_MYSQL_UPGRADE -eq 1 ]; then
rm -r $MYSQL_TEMP_UPGRADE_BACKUP_DIR
fi
diff --git a/extraconfig/tasks/major_upgrade_pacemaker.yaml b/extraconfig/tasks/major_upgrade_pacemaker.yaml
index c70a954f..c2e14880 100644
--- a/extraconfig/tasks/major_upgrade_pacemaker.yaml
+++ b/extraconfig/tasks/major_upgrade_pacemaker.yaml
@@ -32,6 +32,23 @@ resources:
# map_merge with input_values instead of feeding params into scripts
# via str_replace on bash snippets
+ CephMonUpgradeConfig:
+ type: OS::Heat::SoftwareConfig
+ properties:
+ group: script
+ config: {get_file: major_upgrade_ceph_mon.sh}
+
+ CephMonUpgradeDeployment:
+ type: OS::Heat::SoftwareDeploymentGroup
+ properties:
+ servers: {get_param: controller_servers}
+ config: {get_resource: CephMonUpgradeConfig}
+ input_values: {get_param: input_values}
+ batch_create:
+ max_batch_size: 1
+ rolling_update:
+ max_batch_size: 1
+
ControllerPacemakerUpgradeConfig_Step1:
type: OS::Heat::SoftwareConfig
properties:
@@ -57,6 +74,7 @@ resources:
ControllerPacemakerUpgradeDeployment_Step1:
type: OS::Heat::SoftwareDeploymentGroup
+ depends_on: CephMonUpgradeDeployment
properties:
servers: {get_param: controller_servers}
config: {get_resource: ControllerPacemakerUpgradeConfig_Step1}