Move UpgradeInitCommand to role templates

We can't run this during the upgrade steps, because there are things
which need to happen before any role configuration happens, e.g
installing the new hiera heat-config hook, which must be done before
e.g "ControllerDeployment" runs or the stack update hangs.

Partially-Implements: blueprint overcloud-upgrades-per-service
Change-Id: I365b57513590662c3f78a33dc625747f457c48c5

7 files changed, 169 insertions, 37 deletions

diff --git a/puppet/blockstorage-role.yaml b/puppet/blockstorage-role.yaml
index 00f586cd..7d1f8d8f 100644
--- a/puppet/blockstorage-role.yaml
+++ b/puppet/blockstorage-role.yaml
@@ -106,6 +106,12 @@ parameters:
     type: string
     description: Command which will be run whenever configuration data changes
     default: os-refresh-config --timeout 14400
+  UpgradeInitCommand:
+    type: string
+    description: |
+      Command or script snippet to run on all overcloud nodes to
+      initialize the upgrade process. E.g. a repository switch.
+    default: ''
 
 resources:
   BlockStorage:
@@ -327,9 +333,30 @@ resources:
       server: {get_resource: BlockStorage}
       actions: {get_param: NetworkDeploymentActions}
 
+  BlockStorageUpgradeInitConfig:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      config:
+        list_join:
+        - ''
+        - - "#!/bin/bash\n\n"
+          - "if [[ -f /etc/resolv.conf.save ]] ; then rm /etc/resolv.conf.save; fi\n\n"
+          - get_param: UpgradeInitCommand
+
+  # Note we may be able to make this conditional on UpgradeInitCommandNotEmpty
+  # but https://bugs.launchpad.net/heat/+bug/1649900 needs fixing first
+  BlockStorageUpgradeInitDeployment:
+    type: OS::Heat::SoftwareDeployment
+    depends_on: NetworkDeployment
+    properties:
+      name: BlockStorageUpgradeInitDeployment
+      server: {get_resource: BlockStorage}
+      config: {get_resource: BlockStorageUpgradeInitConfig}
+
   BlockStorageDeployment:
     type: OS::Heat::StructuredDeployment
-    depends_on: NetworkDeployment
+    depends_on: BlockStorageUpgradeInitDeployment
     properties:
       name: BlockStorageDeployment
       server: {get_resource: BlockStorage}
diff --git a/puppet/cephstorage-role.yaml b/puppet/cephstorage-role.yaml
index 82cc0134..2c46bf1a 100644
--- a/puppet/cephstorage-role.yaml
+++ b/puppet/cephstorage-role.yaml
@@ -112,6 +112,12 @@ parameters:
     type: string
     description: Command which will be run whenever configuration data changes
     default: os-refresh-config --timeout 14400
+  UpgradeInitCommand:
+    type: string
+    description: |
+      Command or script snippet to run on all overcloud nodes to
+      initialize the upgrade process. E.g. a repository switch.
+    default: ''
 
 resources:
   CephStorage:
@@ -333,9 +339,30 @@ resources:
       server: {get_resource: CephStorage}
       actions: {get_param: NetworkDeploymentActions}
 
+  CephStorageUpgradeInitConfig:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      config:
+        list_join:
+        - ''
+        - - "#!/bin/bash\n\n"
+          - "if [[ -f /etc/resolv.conf.save ]] ; then rm /etc/resolv.conf.save; fi\n\n"
+          - get_param: UpgradeInitCommand
+
+  # Note we may be able to make this conditional on UpgradeInitCommandNotEmpty
+  # but https://bugs.launchpad.net/heat/+bug/1649900 needs fixing first
+  CephStorageUpgradeInitDeployment:
+    type: OS::Heat::SoftwareDeployment
+    depends_on: NetworkDeployment
+    properties:
+      name: CephStorageUpgradeInitDeployment
+      server: {get_resource: CephStorage}
+      config: {get_resource: CephStorageUpgradeInitConfig}
+
   CephStorageDeployment:
     type: OS::Heat::StructuredDeployment
-    depends_on: NetworkDeployment
+    depends_on: CephStorageUpgradeInitDeployment
     properties:
       name: CephStorageDeployment
       config: {get_resource: CephStorageConfig}
diff --git a/puppet/compute-role.yaml b/puppet/compute-role.yaml
index 30a64835..0a2598c1 100644
--- a/puppet/compute-role.yaml
+++ b/puppet/compute-role.yaml
@@ -124,6 +124,12 @@ parameters:
     type: string
     description: Command which will be run whenever configuration data changes
     default: os-refresh-config --timeout 14400
+  UpgradeInitCommand:
+    type: string
+    description: |
+      Command or script snippet to run on all overcloud nodes to
+      initialize the upgrade process. E.g. a repository switch.
+    default: ''
 
 resources:
 
@@ -350,6 +356,27 @@ resources:
         bridge_name: {get_param: NeutronPhysicalBridge}
         interface_name: {get_param: NeutronPublicInterface}
 
+  NovaComputeUpgradeInitConfig:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      config:
+        list_join:
+        - ''
+        - - "#!/bin/bash\n\n"
+          - "if [[ -f /etc/resolv.conf.save ]] ; then rm /etc/resolv.conf.save; fi\n\n"
+          - get_param: UpgradeInitCommand
+
+  # Note we may be able to make this conditional on UpgradeInitCommandNotEmpty
+  # but https://bugs.launchpad.net/heat/+bug/1649900 needs fixing first
+  NovaComputeUpgradeInitDeployment:
+    type: OS::Heat::SoftwareDeployment
+    depends_on: NetworkDeployment
+    properties:
+      name: NovaComputeUpgradeInitDeployment
+      server: {get_resource: NovaCompute}
+      config: {get_resource: NovaComputeUpgradeInitConfig}
+
   NovaComputeConfig:
     type: OS::Heat::StructuredConfig
     properties:
@@ -395,7 +422,7 @@ resources:
 
   NovaComputeDeployment:
     type: OS::TripleO::SoftwareDeployment
-    depends_on: NetworkDeployment
+    depends_on: NovaComputeUpgradeInitDeployment
     properties:
       name: NovaComputeDeployment
       config: {get_resource: NovaComputeConfig}
diff --git a/puppet/controller-role.yaml b/puppet/controller-role.yaml
index a60cc942..5e03adcd 100644
--- a/puppet/controller-role.yaml
+++ b/puppet/controller-role.yaml
@@ -138,6 +138,12 @@ parameters:
     type: string
     description: Command which will be run whenever configuration data changes
     default: os-refresh-config --timeout 14400
+  UpgradeInitCommand:
+    type: string
+    description: |
+      Command or script snippet to run on all overcloud nodes to
+      initialize the upgrade process. E.g. a repository switch.
+    default: ''
 
 parameter_groups:
 - label: deprecated
@@ -384,10 +390,30 @@ resources:
       server: {get_resource: Controller}
       NodeIndex: {get_param: NodeIndex}
 
+  ControllerUpgradeInitConfig:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      config:
+        list_join:
+        - ''
+        - - "#!/bin/bash\n\n"
+          - "if [[ -f /etc/resolv.conf.save ]] ; then rm /etc/resolv.conf.save; fi\n\n"
+          - get_param: UpgradeInitCommand
+
+  # Note we may be able to make this conditional on UpgradeInitCommandNotEmpty
+  # but https://bugs.launchpad.net/heat/+bug/1649900 needs fixing first
+  ControllerUpgradeInitDeployment:
+    type: OS::Heat::SoftwareDeployment
+    depends_on: NetworkDeployment
+    properties:
+      name: ControllerUpgradeInitDeployment
+      server: {get_resource: Controller}
+      config: {get_resource: ControllerUpgradeInitConfig}
 
   ControllerDeployment:
     type: OS::TripleO::SoftwareDeployment
-    depends_on: NetworkDeployment
+    depends_on: ControllerUpgradeInitDeployment
     properties:
       name: ControllerDeployment
       config: {get_resource: ControllerConfig}
diff --git a/puppet/major_upgrade_steps.j2.yaml b/puppet/major_upgrade_steps.j2.yaml
index f8dad433..8d954c09 100644
--- a/puppet/major_upgrade_steps.j2.yaml
+++ b/puppet/major_upgrade_steps.j2.yaml
@@ -15,36 +15,8 @@ parameters:
       Setting to a previously unused value during stack-update will trigger
       the Upgrade resources to re-run on all roles.
 
-  UpgradeInitCommand:
-    type: string
-    description: |
-      Command or script snippet to run on all overcloud nodes to
-      initialize the upgrade process. E.g. a repository switch.
-    default: ''
-
 resources:
 
-  # For the UpgradeInit also rename /etc/resolv.conf.save for +bug/1567004
-  UpgradeInitConfig:
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: script
-      config:
-        list_join:
-        - ''
-        - - "#!/bin/bash\n\n"
-          - "if [[ -f /etc/resolv.conf.save ]] ; then rm /etc/resolv.conf.save; fi\n\n"
-          - get_param: UpgradeInitCommand
-
-{% for role in roles %}
-  {{role.name}}Upgrade_Init:
-    type: OS::Heat::StructuredDeploymentGroup
-    properties:
-      name: {{role.name}}Upgrade_Init
-      servers: {get_param: [servers, {{role.name}}]}
-      config: {get_resource: UpgradeInitConfig}
-{% endfor %}
-
 # Upgrade Steps for all roles
 # FIXME(shardy): would be nice to make the number of steps configurable
 {% for step in range(1, 8) %}
@@ -56,10 +28,8 @@ resources:
   # serialization, but the event output is easier to follow if we
   # do, and there should be minimal performance hit (creating the
   # config is cheap compared to the time to apply the deployment).
+  {% if step > 1 %}
     depends_on:
-  {% if step == 1 %}
-      - {{role.name}}Upgrade_Init
-  {% else %}
       {% for dep in roles %}
       - {{dep.name}}Upgrade_Step{{step -1}}
       {% endfor %}
diff --git a/puppet/objectstorage-role.yaml b/puppet/objectstorage-role.yaml
index 49b2704d..088a2e3d 100644
--- a/puppet/objectstorage-role.yaml
+++ b/puppet/objectstorage-role.yaml
@@ -106,6 +106,12 @@ parameters:
     type: string
     description: Command which will be run whenever configuration data changes
     default: os-refresh-config --timeout 14400
+  UpgradeInitCommand:
+    type: string
+    description: |
+      Command or script snippet to run on all overcloud nodes to
+      initialize the upgrade process. E.g. a repository switch.
+    default: ''
 
 resources:
 
@@ -327,6 +333,27 @@ resources:
       server: {get_resource: SwiftStorage}
       actions: {get_param: NetworkDeploymentActions}
 
+  SwiftStorageUpgradeInitConfig:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      config:
+        list_join:
+        - ''
+        - - "#!/bin/bash\n\n"
+          - "if [[ -f /etc/resolv.conf.save ]] ; then rm /etc/resolv.conf.save; fi\n\n"
+          - get_param: UpgradeInitCommand
+
+  # Note we may be able to make this conditional on UpgradeInitCommandNotEmpty
+  # but https://bugs.launchpad.net/heat/+bug/1649900 needs fixing first
+  SwiftStorageUpgradeInitDeployment:
+    type: OS::Heat::SoftwareDeployment
+    depends_on: NetworkDeployment
+    properties:
+      name: SwiftStorageUpgradeInitDeployment
+      server: {get_resource: SwiftStorage}
+      config: {get_resource: SwiftStorageUpgradeInitConfig}
+
   SwiftStorageHieraConfig:
     type: OS::Heat::StructuredConfig
     properties:
@@ -366,7 +393,7 @@ resources:
 
   SwiftStorageHieraDeploy:
     type: OS::Heat::StructuredDeployment
-    depends_on: NetworkDeployment
+    depends_on: SwiftStorageUpgradeInitDeployment
     properties:
       name: SwiftStorageHieraDeploy
       server: {get_resource: SwiftStorage}
diff --git a/puppet/role.role.j2.yaml b/puppet/role.role.j2.yaml
index d0eeed19..1f432773 100644
--- a/puppet/role.role.j2.yaml
+++ b/puppet/role.role.j2.yaml
@@ -124,6 +124,13 @@ parameters:
   LoggingGroups:
     type: comma_delimited_list
     default: []
+  UpgradeInitCommand:
+    type: string
+    description: |
+      Command or script snippet to run on all overcloud nodes to
+      initialize the upgrade process. E.g. a repository switch.
+    default: ''
+
 
 resources:
   {{role}}:
@@ -345,9 +352,30 @@ resources:
       server: {get_resource: {{role}}}
       actions: {get_param: NetworkDeploymentActions}
 
+  {{role}}UpgradeInitConfig:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      config:
+        list_join:
+        - ''
+        - - "#!/bin/bash\n\n"
+          - "if [[ -f /etc/resolv.conf.save ]] ; then rm /etc/resolv.conf.save; fi\n\n"
+          - get_param: UpgradeInitCommand
+
+  # Note we may be able to make this conditional on UpgradeInitCommandNotEmpty
+  # but https://bugs.launchpad.net/heat/+bug/1649900 needs fixing first
+  {{role}}UpgradeInitDeployment:
+    type: OS::Heat::SoftwareDeployment
+    depends_on: NetworkDeployment
+    properties:
+      name: {{role}}UpgradeInitDeployment
+      server: {get_resource: {{role}}}
+      config: {get_resource: {{role}}UpgradeInitConfig}
+
   {{role}}Deployment:
     type: OS::Heat::StructuredDeployment
-    depends_on: NetworkDeployment
+    depends_on: {{role}}UpgradeInitDeployment
     properties:
       name: {{role}}Deployment
       config: {get_resource: {{role}}Config}