12 files changed, 181 insertions, 24 deletions
diff --git a/extraconfig/tasks/major_upgrade_block_storage.sh b/extraconfig/tasks/major_upgrade_block_storage.sh
index 07666245..39861826 100644
--- a/extraconfig/tasks/major_upgrade_block_storage.sh
+++ b/extraconfig/tasks/major_upgrade_block_storage.sh
@@ -4,5 +4,19 @@
 #
 set -eu
 
+# Special-case OVS for https://bugs.launchpad.net/tripleo/+bug/1635205
+if [[ -n $(rpm -q --scripts openvswitch | awk '/postuninstall/,/*/' | grep "systemctl.*try-restart") ]]; then
+    echo "Manual upgrade of openvswitch - restart in postun detected"
+    mkdir OVS_UPGRADE || true
+    pushd OVS_UPGRADE
+    echo "Attempting to downloading latest openvswitch with yumdownloader"
+    yumdownloader --resolve openvswitch
+    echo "Updating openvswitch with nopostun option"
+    rpm -U --replacepkgs --nopostun ./*.rpm
+    popd
+else
+    echo "Skipping manual upgrade of openvswitch - no restart in postun detected"
+fi
+
 yum -y install python-zaqarclient  # needed for os-collect-config
 yum -y -q update
diff --git a/extraconfig/tasks/major_upgrade_ceph_storage.sh b/extraconfig/tasks/major_upgrade_ceph_storage.sh
index 56b54e22..d84cad45 100644
--- a/extraconfig/tasks/major_upgrade_ceph_storage.sh
+++ b/extraconfig/tasks/major_upgrade_ceph_storage.sh
@@ -49,6 +49,20 @@ timeout 60 bash -c "while kill -0 ${OSD_PIDS} 2> /dev/null; do
   sleep 2;
 done"
 
+# Special-case OVS for https://bugs.launchpad.net/tripleo/+bug/1635205
+if [[ -n $(rpm -q --scripts openvswitch | awk '/postuninstall/,/*/' | grep "systemctl.*try-restart") ]]; then
+    echo "Manual upgrade of openvswitch - restart in postun detected"
+    mkdir OVS_UPGRADE || true
+    pushd OVS_UPGRADE
+    echo "Attempting to downloading latest openvswitch with yumdownloader"
+    yumdownloader --resolve openvswitch
+    echo "Updating openvswitch with nopostun option"
+    rpm -U --replacepkgs --nopostun ./*.rpm
+    popd
+else
+    echo "Skipping manual upgrade of openvswitch - no restart in postun detected"
+fi
+
 # Update (Ceph to Jewel)
 yum -y install python-zaqarclient  # needed for os-collect-config
 yum -y update
diff --git a/extraconfig/tasks/major_upgrade_compute.sh b/extraconfig/tasks/major_upgrade_compute.sh
index a1df695f..f5105a1a 100644
--- a/extraconfig/tasks/major_upgrade_compute.sh
+++ b/extraconfig/tasks/major_upgrade_compute.sh
@@ -18,6 +18,20 @@ set -eu
 
 crudini  --set /etc/nova/nova.conf upgrade_levels compute $upgrade_level_nova_compute
 
+# Special-case OVS for https://bugs.launchpad.net/tripleo/+bug/1635205
+if [[ -n \$(rpm -q --scripts openvswitch | awk '/postuninstall/,/*/' | grep "systemctl.*try-restart") ]]; then
+    echo "Manual upgrade of openvswitch - restart in postun detected"
+    mkdir OVS_UPGRADE || true
+    pushd OVS_UPGRADE
+    echo "Attempting to downloading latest openvswitch with yumdownloader"
+    yumdownloader --resolve openvswitch
+    echo "Updating openvswitch with nopostun option"
+    rpm -U --replacepkgs --nopostun ./*.rpm
+    popd
+else
+    echo "Skipping manual upgrade of openvswitch - no restart in postun detected"
+fi
+
 yum -y install python-zaqarclient  # needed for os-collect-config
 yum -y update
 
diff --git a/extraconfig/tasks/major_upgrade_controller_pacemaker_1.sh b/extraconfig/tasks/major_upgrade_controller_pacemaker_1.sh
index 23074fcb..fbdbc30b 100755
--- a/extraconfig/tasks/major_upgrade_controller_pacemaker_1.sh
+++ b/extraconfig/tasks/major_upgrade_controller_pacemaker_1.sh
@@ -17,8 +17,10 @@ check_disk_for_mysql_dump
 # nodes where a service fails to stop, which could be fatal during an upgrade
 # procedure. So we remember the stonith state. If it was enabled we reenable it
 # at the end of this script
-STONITH_STATE=$(pcs property show stonith-enabled | grep "stonith-enabled" | awk '{ print $2 }')
-pcs property set stonith-enabled=false
+if [[ -n $(is_bootstrap_node) ]]; then
+    STONITH_STATE=$(pcs property show stonith-enabled | grep "stonith-enabled" | awk '{ print $2 }')
+    pcs property set stonith-enabled=false
+fi
 
 # Migrate to HA NG and fix up rabbitmq queues
 # We fix up the rabbitmq ha queues after the migration because it will
@@ -120,6 +122,20 @@ if [ $DO_MYSQL_UPGRADE -eq 1 ]; then
     mv /var/lib/mysql $MYSQL_TEMP_UPGRADE_BACKUP_DIR
 fi
 
+# Special-case OVS for https://bugs.launchpad.net/tripleo/+bug/1635205
+if [[ -n $(rpm -q --scripts openvswitch | awk '/postuninstall/,/*/' | grep "systemctl.*try-restart") ]]; then
+    echo "Manual upgrade of openvswitch - restart in postun detected"
+    mkdir OVS_UPGRADE || true
+    pushd OVS_UPGRADE
+    echo "Attempting to downloading latest openvswitch with yumdownloader"
+    yumdownloader --resolve openvswitch
+    echo "Updating openvswitch with nopostun option"
+    rpm -U --replacepkgs --nopostun ./*.rpm
+    popd
+else
+    echo "Skipping manual upgrade of openvswitch - no restart in postun detected"
+fi
+
 yum -y install python-zaqarclient  # needed for os-collect-config
 yum -y -q update
 
@@ -170,8 +186,10 @@ if [ $DO_MYSQL_UPGRADE -eq 1 ]; then
 fi
 
 # Let's reset the stonith back to true if it was true, before starting the cluster
-if [ $STONITH_STATE == "true" ]; then
-    pcs -f /var/lib/pacemaker/cib/cib.xml property set stonith-enabled=true
+if [[ -n $(is_bootstrap_node) ]]; then
+    if [ $STONITH_STATE == "true" ]; then
+        pcs -f /var/lib/pacemaker/cib/cib.xml property set stonith-enabled=true
+    fi
 fi
 
 # Pin messages sent to compute nodes to kilo, these will be upgraded later
diff --git a/extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh b/extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh
index b3a0098c..37061512 100755
--- a/extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh
+++ b/extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh
@@ -64,6 +64,5 @@ if [[ -n $(is_bootstrap_node) ]]; then
     nova-manage db sync
     nova-manage api_db sync
     nova-manage db online_data_migrations
-    gnocchi-upgrade
     sahara-db-manage --config-file /etc/sahara/sahara.conf upgrade head
 fi
diff --git a/extraconfig/tasks/major_upgrade_controller_pacemaker_3.sh b/extraconfig/tasks/major_upgrade_controller_pacemaker_3.sh
index b653c7c7..d2cb9553 100755
--- a/extraconfig/tasks/major_upgrade_controller_pacemaker_3.sh
+++ b/extraconfig/tasks/major_upgrade_controller_pacemaker_3.sh
@@ -9,18 +9,9 @@ check_resource redis started 600
 start_or_enable_service openstack-cinder-volume
 check_resource openstack-cinder-volume started 600
 
+# start httpd so keystone is available for gnocchi
+# upgrade to run.
+systemctl start httpd
 
 # Swift isn't controled by pacemaker
 systemctl_swift start
-
-# We need to start the systemd services we explicitely stopped at step _1.sh
-# FIXME: Should we let puppet during the convergence step do the service enabling or
-# should we add it here?
-services=$(services_to_migrate)
-if [[ ${keep_sahara_services_on_upgrade} =~ [Ff]alse ]] ; then
-    services=${services%%openstack-sahara*}
-fi
-for service in $services; do
-    manage_systemd_service start "${service%%-clone}"
-    check_resource_systemd "${service%%-clone}" started 600
-done
diff --git a/extraconfig/tasks/major_upgrade_controller_pacemaker_4.sh b/extraconfig/tasks/major_upgrade_controller_pacemaker_4.sh
new file mode 100755
index 00000000..fa95f1f8
--- /dev/null
+++ b/extraconfig/tasks/major_upgrade_controller_pacemaker_4.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -eu
+
+if [[ -n $(is_bootstrap_node) ]]; then
+  # run gnocchi upgrade
+  gnocchi-upgrade
+fi
diff --git a/extraconfig/tasks/major_upgrade_controller_pacemaker_5.sh b/extraconfig/tasks/major_upgrade_controller_pacemaker_5.sh
new file mode 100755
index 00000000..d569084d
--- /dev/null
+++ b/extraconfig/tasks/major_upgrade_controller_pacemaker_5.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -eu
+
+# We need to start the systemd services we explicitely stopped at step _1.sh
+# FIXME: Should we let puppet during the convergence step do the service enabling or
+# should we add it here?
+services=$(services_to_migrate)
+if [[ ${keep_sahara_services_on_upgrade} =~ [Ff]alse ]] ; then
+    services=${services%%openstack-sahara*}
+fi
+for service in $services; do
+    manage_systemd_service start "${service%%-clone}"
+    check_resource_systemd "${service%%-clone}" started 600
+done
diff --git a/extraconfig/tasks/major_upgrade_object_storage.sh b/extraconfig/tasks/major_upgrade_object_storage.sh
index f82457ce..2667bb16 100644
--- a/extraconfig/tasks/major_upgrade_object_storage.sh
+++ b/extraconfig/tasks/major_upgrade_object_storage.sh
@@ -23,6 +23,19 @@ function systemctl_swift {
     done
 }
 
+# Special-case OVS for https://bugs.launchpad.net/tripleo/+bug/1635205
+if [[ -n \$(rpm -q --scripts openvswitch | awk '/postuninstall/,/*/' | grep "systemctl.*try-restart") ]]; then
+    echo "Manual upgrade of openvswitch - restart in postun detected"
+    mkdir OVS_UPGRADE || true
+    pushd OVS_UPGRADE
+    echo "Attempting to downloading latest openvswitch with yumdownloader"
+    yumdownloader --resolve openvswitch
+    echo "Updating openvswitch with nopostun option"
+    rpm -U --replacepkgs --nopostun ./*.rpm
+    popd
+else
+    echo "Skipping manual upgrade of openvswitch - no restart in postun detected"
+fi
 
 systemctl_swift stop
 
diff --git a/extraconfig/tasks/major_upgrade_pacemaker.yaml b/extraconfig/tasks/major_upgrade_pacemaker.yaml
index 7c78d5ad..e13aada3 100644
--- a/extraconfig/tasks/major_upgrade_pacemaker.yaml
+++ b/extraconfig/tasks/major_upgrade_pacemaker.yaml
@@ -132,6 +132,44 @@ resources:
       config:
         list_join:
         - ''
+        - - get_file: pacemaker_common_functions.sh
+          - get_file: major_upgrade_pacemaker_migrations.sh
+          - get_file: major_upgrade_controller_pacemaker_3.sh
+
+  ControllerPacemakerUpgradeDeployment_Step3:
+    type: OS::Heat::SoftwareDeploymentGroup
+    depends_on: ControllerPacemakerUpgradeDeployment_Step2
+    properties:
+      servers:  {get_param: [servers, Controller]}
+      config: {get_resource: ControllerPacemakerUpgradeConfig_Step3}
+      input_values: {get_param: input_values}
+
+  ControllerPacemakerUpgradeConfig_Step4:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      config:
+        list_join:
+        - ''
+        - - get_file: pacemaker_common_functions.sh
+          - get_file: major_upgrade_pacemaker_migrations.sh
+          - get_file: major_upgrade_controller_pacemaker_4.sh
+
+  ControllerPacemakerUpgradeDeployment_Step4:
+    type: OS::Heat::SoftwareDeploymentGroup
+    depends_on: ControllerPacemakerUpgradeDeployment_Step3
+    properties:
+      servers:  {get_param: [servers, Controller]}
+      config: {get_resource: ControllerPacemakerUpgradeConfig_Step4}
+      input_values: {get_param: input_values}
+
+  ControllerPacemakerUpgradeConfig_Step5:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      config:
+        list_join:
+        - ''
         - - str_replace:
               template: |
                 #!/bin/bash
@@ -140,13 +178,12 @@ resources:
                 KEEP_SAHARA_SERVICES_ON_UPGRADE: {get_param: KeepSaharaServicesOnUpgrade}
           - get_file: pacemaker_common_functions.sh
           - get_file: major_upgrade_pacemaker_migrations.sh
-          - get_file: major_upgrade_controller_pacemaker_3.sh
+          - get_file: major_upgrade_controller_pacemaker_5.sh
 
-  ControllerPacemakerUpgradeDeployment_Step3:
+  ControllerPacemakerUpgradeDeployment_Step5:
     type: OS::Heat::SoftwareDeploymentGroup
-    depends_on: ControllerPacemakerUpgradeDeployment_Step2
+    depends_on: ControllerPacemakerUpgradeDeployment_Step4
     properties:
       servers:  {get_param: [servers, Controller]}
-      config: {get_resource: ControllerPacemakerUpgradeConfig_Step3}
+      config: {get_resource: ControllerPacemakerUpgradeConfig_Step5}
       input_values: {get_param: input_values}
-
diff --git a/extraconfig/tasks/pacemaker_common_functions.sh b/extraconfig/tasks/pacemaker_common_functions.sh
index 4f17b69a..2c7dfc35 100755
--- a/extraconfig/tasks/pacemaker_common_functions.sh
+++ b/extraconfig/tasks/pacemaker_common_functions.sh
@@ -284,7 +284,7 @@ function systemctl_swift {
             services=$(systemctl | grep openstack-swift- | grep running | awk '{print $1}')
             ;;
         start)
-            enable_swift_storage=$(hiera -c /etc/puppet/hiera.yaml 'enable_swift_storage')
+            enable_swift_storage=$(hiera -c /etc/puppet/hiera.yaml tripleo::profile::base::swift::storage::enable_swift_storage)
             if [[ $enable_swift_storage != "true" ]]; then
                 services=( openstack-swift-proxy )
             fi
diff --git a/extraconfig/tasks/yum_update.sh b/extraconfig/tasks/yum_update.sh
index b045e5ea..4612f197 100755
--- a/extraconfig/tasks/yum_update.sh
+++ b/extraconfig/tasks/yum_update.sh
@@ -44,6 +44,25 @@ fi
 
 pacemaker_status=$(systemctl is-active pacemaker)
 
+# Fix the redis/rabbit resource start/stop timeouts. See https://bugs.launchpad.net/tripleo/+bug/1633455
+# and https://bugs.launchpad.net/tripleo/+bug/1634851
+if [[ "$pacemaker_status" == "active" && \
+      "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname)" ]] ; then
+    if pcs resource show rabbitmq | grep -E "start.*timeout=100"; then
+        pcs resource update rabbitmq op start timeout=200s
+    fi
+    if pcs resource show rabbitmq | grep -E "stop.*timeout=90"; then
+        pcs resource update rabbitmq op stop timeout=200s
+    fi
+    if pcs resource show redis | grep -E "start.*timeout=120"; then
+        pcs resource update redis op start timeout=200s
+    fi
+    if pcs resource show redis | grep -E "stop.*timeout=120"; then
+        pcs resource update redis op stop timeout=200s
+    fi
+fi
+
+
 if [[ "$pacemaker_status" == "active" ]] ; then
     echo "Pacemaker running, stopping cluster node and doing full package update"
     node_count=$(pcs status xml | grep -o "<nodes_configured.*/>" | grep -o 'number="[0-9]*"' | grep -o "[0-9]*")
@@ -54,13 +73,28 @@ if [[ "$pacemaker_status" == "active" ]] ; then
         pcs cluster stop
     fi
 else
-    echo "Upgrading openstack-puppet-modules"
+    echo "Upgrading openstack-puppet-modules and its dependencies"
     yum -q -y update openstack-puppet-modules
+    yum deplist openstack-puppet-modules | awk '/dependency/{print $2}' | xargs yum -q -y update
     echo "Upgrading other packages is handled by config management tooling"
     echo -n "true" > $heat_outputs_path.update_managed_packages
     exit 0
 fi
 
+# Special-case OVS for https://bugs.launchpad.net/tripleo/+bug/1635205
+if [[ -n $(rpm -q --scripts openvswitch | awk '/postuninstall/,/*/' | grep "systemctl.*try-restart") ]]; then
+    echo "Manual upgrade of openvswitch - restart in postun detected"
+    mkdir OVS_UPGRADE || true
+    pushd OVS_UPGRADE
+    echo "Attempting to downloading latest openvswitch with yumdownloader"
+    yumdownloader --resolve openvswitch
+    echo "Updating openvswitch with nopostun option"
+    rpm -U --replacepkgs --nopostun ./*.rpm
+    popd
+else
+    echo "Skipping manual upgrade of openvswitch - no restart in postun detected"
+fi
+
 command=${command:-update}
 full_command="yum -q -y $command $command_arguments"
 echo "Running: $full_command"