Rework gnocchi-upgrade to run in a separate upgrade step

gnocchi when configured with swift will require keystone
to be available to authenticate to migrate to v3. At this
step keystone is not available and gnocchi upgrade fails
with auth error. Instead start apache in step 3, start
apache first and then run gnocchi upgrade in a separate
step and let upgrade happen here.

Closes-Bug: #1634897

Change-Id: I22d02528420e4456f84b80905a7b3a80653fa7b0

5 files changed, 68 insertions, 18 deletions

diff --git a/extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh b/extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh
index b3a0098c..37061512 100755
--- a/extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh
+++ b/extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh
@@ -64,6 +64,5 @@ if [[ -n $(is_bootstrap_node) ]]; then
     nova-manage db sync
     nova-manage api_db sync
     nova-manage db online_data_migrations
-    gnocchi-upgrade
     sahara-db-manage --config-file /etc/sahara/sahara.conf upgrade head
 fi
diff --git a/extraconfig/tasks/major_upgrade_controller_pacemaker_3.sh b/extraconfig/tasks/major_upgrade_controller_pacemaker_3.sh
index b653c7c7..d2cb9553 100755
--- a/extraconfig/tasks/major_upgrade_controller_pacemaker_3.sh
+++ b/extraconfig/tasks/major_upgrade_controller_pacemaker_3.sh
@@ -9,18 +9,9 @@ check_resource redis started 600
 start_or_enable_service openstack-cinder-volume
 check_resource openstack-cinder-volume started 600
 
+# start httpd so keystone is available for gnocchi
+# upgrade to run.
+systemctl start httpd
 
 # Swift isn't controled by pacemaker
 systemctl_swift start
-
-# We need to start the systemd services we explicitely stopped at step _1.sh
-# FIXME: Should we let puppet during the convergence step do the service enabling or
-# should we add it here?
-services=$(services_to_migrate)
-if [[ ${keep_sahara_services_on_upgrade} =~ [Ff]alse ]] ; then
-    services=${services%%openstack-sahara*}
-fi
-for service in $services; do
-    manage_systemd_service start "${service%%-clone}"
-    check_resource_systemd "${service%%-clone}" started 600
-done
diff --git a/extraconfig/tasks/major_upgrade_controller_pacemaker_4.sh b/extraconfig/tasks/major_upgrade_controller_pacemaker_4.sh
new file mode 100755
index 00000000..fa95f1f8
--- /dev/null
+++ b/extraconfig/tasks/major_upgrade_controller_pacemaker_4.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -eu
+
+if [[ -n $(is_bootstrap_node) ]]; then
+  # run gnocchi upgrade
+  gnocchi-upgrade
+fi
diff --git a/extraconfig/tasks/major_upgrade_controller_pacemaker_5.sh b/extraconfig/tasks/major_upgrade_controller_pacemaker_5.sh
new file mode 100755
index 00000000..d569084d
--- /dev/null
+++ b/extraconfig/tasks/major_upgrade_controller_pacemaker_5.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -eu
+
+# We need to start the systemd services we explicitely stopped at step _1.sh
+# FIXME: Should we let puppet during the convergence step do the service enabling or
+# should we add it here?
+services=$(services_to_migrate)
+if [[ ${keep_sahara_services_on_upgrade} =~ [Ff]alse ]] ; then
+    services=${services%%openstack-sahara*}
+fi
+for service in $services; do
+    manage_systemd_service start "${service%%-clone}"
+    check_resource_systemd "${service%%-clone}" started 600
+done
diff --git a/extraconfig/tasks/major_upgrade_pacemaker.yaml b/extraconfig/tasks/major_upgrade_pacemaker.yaml
index 7c78d5ad..e13aada3 100644
--- a/extraconfig/tasks/major_upgrade_pacemaker.yaml
+++ b/extraconfig/tasks/major_upgrade_pacemaker.yaml
@@ -132,6 +132,44 @@ resources:
       config:
         list_join:
         - ''
+        - - get_file: pacemaker_common_functions.sh
+          - get_file: major_upgrade_pacemaker_migrations.sh
+          - get_file: major_upgrade_controller_pacemaker_3.sh
+
+  ControllerPacemakerUpgradeDeployment_Step3:
+    type: OS::Heat::SoftwareDeploymentGroup
+    depends_on: ControllerPacemakerUpgradeDeployment_Step2
+    properties:
+      servers:  {get_param: [servers, Controller]}
+      config: {get_resource: ControllerPacemakerUpgradeConfig_Step3}
+      input_values: {get_param: input_values}
+
+  ControllerPacemakerUpgradeConfig_Step4:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      config:
+        list_join:
+        - ''
+        - - get_file: pacemaker_common_functions.sh
+          - get_file: major_upgrade_pacemaker_migrations.sh
+          - get_file: major_upgrade_controller_pacemaker_4.sh
+
+  ControllerPacemakerUpgradeDeployment_Step4:
+    type: OS::Heat::SoftwareDeploymentGroup
+    depends_on: ControllerPacemakerUpgradeDeployment_Step3
+    properties:
+      servers:  {get_param: [servers, Controller]}
+      config: {get_resource: ControllerPacemakerUpgradeConfig_Step4}
+      input_values: {get_param: input_values}
+
+  ControllerPacemakerUpgradeConfig_Step5:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      config:
+        list_join:
+        - ''
         - - str_replace:
               template: |
                 #!/bin/bash
@@ -140,13 +178,12 @@ resources:
                 KEEP_SAHARA_SERVICES_ON_UPGRADE: {get_param: KeepSaharaServicesOnUpgrade}
           - get_file: pacemaker_common_functions.sh
           - get_file: major_upgrade_pacemaker_migrations.sh
-          - get_file: major_upgrade_controller_pacemaker_3.sh
+          - get_file: major_upgrade_controller_pacemaker_5.sh
 
-  ControllerPacemakerUpgradeDeployment_Step3:
+  ControllerPacemakerUpgradeDeployment_Step5:
     type: OS::Heat::SoftwareDeploymentGroup
-    depends_on: ControllerPacemakerUpgradeDeployment_Step2
+    depends_on: ControllerPacemakerUpgradeDeployment_Step4
     properties:
       servers:  {get_param: [servers, Controller]}
-      config: {get_resource: ControllerPacemakerUpgradeConfig_Step3}
+      config: {get_resource: ControllerPacemakerUpgradeConfig_Step5}
       input_values: {get_param: input_values}
-