Merge "Make the openvswitch 2.4->2.5 upgrade more robust"

10 files changed, 57 insertions, 80 deletions

diff --git a/extraconfig/tasks/major_upgrade_block_storage.sh b/extraconfig/tasks/major_upgrade_block_storage.sh
index 39861826..64c4457e 100644
--- a/extraconfig/tasks/major_upgrade_block_storage.sh
+++ b/extraconfig/tasks/major_upgrade_block_storage.sh
@@ -5,18 +5,7 @@
 set -eu
 
 # Special-case OVS for https://bugs.launchpad.net/tripleo/+bug/1635205
-if [[ -n $(rpm -q --scripts openvswitch | awk '/postuninstall/,/*/' | grep "systemctl.*try-restart") ]]; then
-    echo "Manual upgrade of openvswitch - restart in postun detected"
-    mkdir OVS_UPGRADE || true
-    pushd OVS_UPGRADE
-    echo "Attempting to downloading latest openvswitch with yumdownloader"
-    yumdownloader --resolve openvswitch
-    echo "Updating openvswitch with nopostun option"
-    rpm -U --replacepkgs --nopostun ./*.rpm
-    popd
-else
-    echo "Skipping manual upgrade of openvswitch - no restart in postun detected"
-fi
+special_case_ovs_upgrade_if_needed
 
 yum -y install python-zaqarclient  # needed for os-collect-config
 yum -y -q update
diff --git a/extraconfig/tasks/major_upgrade_ceph_storage.sh b/extraconfig/tasks/major_upgrade_ceph_storage.sh
index d84cad45..59b7c20e 100644
--- a/extraconfig/tasks/major_upgrade_ceph_storage.sh
+++ b/extraconfig/tasks/major_upgrade_ceph_storage.sh
@@ -8,7 +8,9 @@ set -o pipefail
 
 UPGRADE_SCRIPT=/root/tripleo_upgrade_node.sh
 
-cat > $UPGRADE_SCRIPT << 'ENDOFCAT'
+declare -f special_case_ovs_upgrade_if_needed > $UPGRADE_SCRIPT
+# use >> here so we don't lose the declaration we added above
+cat >> $UPGRADE_SCRIPT << 'ENDOFCAT'
 #!/bin/bash
 ### DO NOT MODIFY THIS FILE
 ### This file is automatically delivered to the ceph-storage nodes as part of the
@@ -49,19 +51,7 @@ timeout 60 bash -c "while kill -0 ${OSD_PIDS} 2> /dev/null; do
   sleep 2;
 done"
 
-# Special-case OVS for https://bugs.launchpad.net/tripleo/+bug/1635205
-if [[ -n $(rpm -q --scripts openvswitch | awk '/postuninstall/,/*/' | grep "systemctl.*try-restart") ]]; then
-    echo "Manual upgrade of openvswitch - restart in postun detected"
-    mkdir OVS_UPGRADE || true
-    pushd OVS_UPGRADE
-    echo "Attempting to downloading latest openvswitch with yumdownloader"
-    yumdownloader --resolve openvswitch
-    echo "Updating openvswitch with nopostun option"
-    rpm -U --replacepkgs --nopostun ./*.rpm
-    popd
-else
-    echo "Skipping manual upgrade of openvswitch - no restart in postun detected"
-fi
+special_case_ovs_upgrade_if_needed
 
 # Update (Ceph to Jewel)
 yum -y install python-zaqarclient  # needed for os-collect-config
diff --git a/extraconfig/tasks/major_upgrade_compute.sh b/extraconfig/tasks/major_upgrade_compute.sh
index b0d42806..7a3e1073 100644
--- a/extraconfig/tasks/major_upgrade_compute.sh
+++ b/extraconfig/tasks/major_upgrade_compute.sh
@@ -18,19 +18,8 @@ set -eu
 
 crudini  --set /etc/nova/nova.conf upgrade_levels compute $upgrade_level_nova_compute
 
-# Special-case OVS for https://bugs.launchpad.net/tripleo/+bug/1635205
-if [[ -n \$(rpm -q --scripts openvswitch | awk '/postuninstall/,/*/' | grep "systemctl.*try-restart") ]]; then
-    echo "Manual upgrade of openvswitch - restart in postun detected"
-    mkdir OVS_UPGRADE || true
-    pushd OVS_UPGRADE
-    echo "Attempting to downloading latest openvswitch with yumdownloader"
-    yumdownloader --resolve openvswitch
-    echo "Updating openvswitch with nopostun option"
-    rpm -U --replacepkgs --nopostun ./*.rpm
-    popd
-else
-    echo "Skipping manual upgrade of openvswitch - no restart in postun detected"
-fi
+$(declare -f special_case_ovs_upgrade_if_needed)
+special_case_ovs_upgrade_if_needed
 
 yum -y install python-zaqarclient  # needed for os-collect-config
 yum -y update
diff --git a/extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh b/extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh
index 7cc6735f..6bfe1239 100755
--- a/extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh
+++ b/extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh
@@ -100,18 +100,7 @@ if [ $DO_MYSQL_UPGRADE -eq 1 ]; then
 fi
 
 # Special-case OVS for https://bugs.launchpad.net/tripleo/+bug/1635205
-if [[ -n $(rpm -q --scripts openvswitch | awk '/postuninstall/,/*/' | grep "systemctl.*try-restart") ]]; then
-    echo "Manual upgrade of openvswitch - restart in postun detected"
-    mkdir OVS_UPGRADE || true
-    pushd OVS_UPGRADE
-    echo "Attempting to downloading latest openvswitch with yumdownloader"
-    yumdownloader --resolve openvswitch
-    echo "Updating openvswitch with nopostun option"
-    rpm -U --replacepkgs --nopostun ./*.rpm
-    popd
-else
-    echo "Skipping manual upgrade of openvswitch - no restart in postun detected"
-fi
+special_case_ovs_upgrade_if_needed
 
 yum -y install python-zaqarclient  # needed for os-collect-config
 yum -y -q update
diff --git a/extraconfig/tasks/major_upgrade_object_storage.sh b/extraconfig/tasks/major_upgrade_object_storage.sh
index 2667bb16..d9d1b4d5 100644
--- a/extraconfig/tasks/major_upgrade_object_storage.sh
+++ b/extraconfig/tasks/major_upgrade_object_storage.sh
@@ -23,19 +23,8 @@ function systemctl_swift {
     done
 }
 
-# Special-case OVS for https://bugs.launchpad.net/tripleo/+bug/1635205
-if [[ -n \$(rpm -q --scripts openvswitch | awk '/postuninstall/,/*/' | grep "systemctl.*try-restart") ]]; then
-    echo "Manual upgrade of openvswitch - restart in postun detected"
-    mkdir OVS_UPGRADE || true
-    pushd OVS_UPGRADE
-    echo "Attempting to downloading latest openvswitch with yumdownloader"
-    yumdownloader --resolve openvswitch
-    echo "Updating openvswitch with nopostun option"
-    rpm -U --replacepkgs --nopostun ./*.rpm
-    popd
-else
-    echo "Skipping manual upgrade of openvswitch - no restart in postun detected"
-fi
+$(declare -f special_case_ovs_upgrade_if_needed)
+special_case_ovs_upgrade_if_needed
 
 systemctl_swift stop
 
diff --git a/extraconfig/tasks/major_upgrade_pacemaker.yaml b/extraconfig/tasks/major_upgrade_pacemaker.yaml
index b0418a56..a175a423 100644
--- a/extraconfig/tasks/major_upgrade_pacemaker.yaml
+++ b/extraconfig/tasks/major_upgrade_pacemaker.yaml
@@ -97,7 +97,11 @@ resources:
     depends_on: ControllerPacemakerUpgradeDeployment_Step1
     properties:
       group: script
-      config: {get_file: major_upgrade_block_storage.sh}
+      config:
+        list_join:
+        - ''
+        - - get_file: pacemaker_common_functions.sh
+          - get_file: major_upgrade_block_storage.sh
 
   BlockStorageUpgradeDeployment:
     type: OS::Heat::SoftwareDeploymentGroup
diff --git a/extraconfig/tasks/major_upgrade_pacemaker_init.j2.yaml b/extraconfig/tasks/major_upgrade_pacemaker_init.j2.yaml
index f6aa3066..8e9cbdb4 100644
--- a/extraconfig/tasks/major_upgrade_pacemaker_init.j2.yaml
+++ b/extraconfig/tasks/major_upgrade_pacemaker_init.j2.yaml
@@ -54,19 +54,28 @@ resources:
                 upgrade_level_nova_compute='UPGRADE_LEVEL_NOVA_COMPUTE'
               params:
                 UPGRADE_LEVEL_NOVA_COMPUTE: {get_param: UpgradeLevelNovaCompute}
+          - get_file: pacemaker_common_functions.sh
           - get_file: major_upgrade_compute.sh
 
   ObjectStorageDeliverUpgradeScriptConfig:
     type: OS::Heat::SoftwareConfig
     properties:
       group: script
-      config: {get_file: major_upgrade_object_storage.sh}
+      config:
+        list_join:
+        - ''
+        - - get_file: pacemaker_common_functions.sh
+          - get_file: major_upgrade_object_storage.sh
 
   CephStorageDeliverUpgradeScriptConfig:
     type: OS::Heat::SoftwareConfig
     properties:
       group: script
-      config: {get_file: major_upgrade_ceph_storage.sh}
+      config:
+        list_join:
+        - ''
+        - - get_file: pacemaker_common_functions.sh
+          - get_file: major_upgrade_ceph_storage.sh
 
 {% for role in roles %}
   UpgradeInit{{role.name}}Deployment:
diff --git a/extraconfig/tasks/pacemaker_common_functions.sh b/extraconfig/tasks/pacemaker_common_functions.sh
index 2c7dfc35..aae4a2de 100755
--- a/extraconfig/tasks/pacemaker_common_functions.sh
+++ b/extraconfig/tasks/pacemaker_common_functions.sh
@@ -297,3 +297,27 @@ function systemctl_swift {
         manage_systemd_service $action $service
     done
 }
+
+# Special-case OVS for https://bugs.launchpad.net/tripleo/+bug/1635205
+function special_case_ovs_upgrade_if_needed {
+    if [[ -n $(rpm -q --scripts openvswitch | awk '/postuninstall/,/*/' | grep "systemctl.*try-restart") ]]; then
+        echo "Manual upgrade of openvswitch - restart in postun detected"
+        rm -rf OVS_UPGRADE
+        mkdir OVS_UPGRADE && pushd OVS_UPGRADE
+        echo "Attempting to downloading latest openvswitch with yumdownloader"
+        yumdownloader --resolve openvswitch
+        for pkg in $(ls -1 *.rpm);  do
+            if rpm -U --test $pkg 2>&1 | grep "already installed" ; then
+                echo "Looks like newer version of $pkg is already installed, skipping"
+            else
+                echo "Updating $pkg with nopostun option"
+                rpm -U --replacepkgs --nopostun $pkg
+            fi
+        done
+        popd
+    else
+        echo "Skipping manual upgrade of openvswitch - no restart in postun detected"
+    fi
+
+}
+
diff --git a/extraconfig/tasks/yum_update.sh b/extraconfig/tasks/yum_update.sh
index 8a88ee64..74af7b02 100755
--- a/extraconfig/tasks/yum_update.sh
+++ b/extraconfig/tasks/yum_update.sh
@@ -63,18 +63,7 @@ if [[ "$pacemaker_status" == "active" && \
 fi
 
 # Special-case OVS for https://bugs.launchpad.net/tripleo/+bug/1635205
-if [[ -n $(rpm -q --scripts openvswitch | awk '/postuninstall/,/*/' | grep "systemctl.*try-restart") ]]; then
-    echo "Manual upgrade of openvswitch - restart in postun detected"
-    mkdir OVS_UPGRADE || true
-    pushd OVS_UPGRADE
-    echo "Attempting to downloading latest openvswitch with yumdownloader"
-    yumdownloader --resolve openvswitch
-    echo "Updating openvswitch with nopostun option"
-    rpm -U --replacepkgs --nopostun ./*.rpm
-    popd
-else
-    echo "Skipping manual upgrade of openvswitch - no restart in postun detected"
-fi
+special_case_ovs_upgrade_if_needed
 
 if [[ "$pacemaker_status" == "active" ]] ; then
     echo "Pacemaker running, stopping cluster node and doing full package update"
diff --git a/extraconfig/tasks/yum_update.yaml b/extraconfig/tasks/yum_update.yaml
index d313ca9f..f2de5acf 100644
--- a/extraconfig/tasks/yum_update.yaml
+++ b/extraconfig/tasks/yum_update.yaml
@@ -9,7 +9,12 @@ resources:
     type: OS::Heat::SoftwareConfig
     properties:
       group: script
-      config: {get_file: yum_update.sh}
+      config:
+        list_join:
+        - ''
+        - - get_file: pacemaker_common_functions.sh
+          - get_file: yum_update.sh
+
       inputs:
       - name: update_identifier
         description: yum will only run for previously unused values of update_identifier