Merge "Enable composable upgrades for docker service templates"

28 files changed, 451 insertions, 327 deletions

diff --git a/docker/docker-steps.j2 b/docker/docker-steps.j2
new file mode 100644
index 00000000..76232d14
--- /dev/null
+++ b/docker/docker-steps.j2
@@ -0,0 +1,325 @@
+# certain initialization steps (run in a container) will occur
+# on the first role listed in the roles file
+{% set primary_role_name = roles[0].name -%}
+
+heat_template_version: ocata
+
+description: >
+  Post-deploy configuration steps via puppet for all roles,
+  as defined in ../roles_data.yaml
+
+parameters:
+  servers:
+    type: json
+    description: Mapping of Role name e.g Controller to a list of servers
+  role_data:
+    type: json
+    description: Mapping of Role name e.g Controller to the per-role data
+  DeployIdentifier:
+    default: ''
+    type: string
+    description: >
+      Setting this to a unique value will re-run any deployment tasks which
+      perform configuration on a Heat stack-update.
+  EndpointMap:
+    default: {}
+    description: Mapping of service endpoint -> protocol. Typically set
+                 via parameter_defaults in the resource registry.
+    type: json
+
+resources:
+
+  # These utility tasks use docker-puppet.py to execute tasks via puppet
+  # We only execute these on the first node in the primary role
+  {{primary_role_name}}DockerPuppetTasks:
+    type: OS::Heat::Value
+    properties:
+      type: json
+      value:
+        yaql:
+          expression:
+            dict($.data.docker_puppet_tasks.where($1 != null).selectMany($.items()).groupBy($[0], $[1]))
+          data:
+            docker_puppet_tasks: {get_param: [role_data, {{primary_role_name}}, docker_puppet_tasks]}
+
+# BEGIN primary_role_name docker-puppet-tasks (run only on a single node)
+{% for step in range(1, 6) %}
+
+  {{primary_role_name}}DockerPuppetJsonConfig{{step}}:
+      type: OS::Heat::StructuredConfig
+      properties:
+        group: json-file
+        config:
+          /var/lib/docker-puppet/docker-puppet-tasks{{step}}.json:
+            {get_attr: [{{primary_role_name}}DockerPuppetTasks, value, 'step_{{step}}']}
+
+  {{primary_role_name}}DockerPuppetJsonDeployment{{step}}:
+    type: OS::Heat::SoftwareDeployment
+    properties:
+      server: {get_param: [servers, {{primary_role_name}}, '0']}
+      config: {get_resource: {{primary_role_name}}DockerPuppetJsonConfig{{step}}}
+
+  {{primary_role_name}}DockerPuppetTasksConfig{{step}}:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      config: {get_file: docker-puppet.py}
+      inputs:
+        - name: CONFIG
+        - name: NET_HOST
+        - name: NO_ARCHIVE
+        - name: STEP
+
+  {{primary_role_name}}DockerPuppetTasksDeployment{{step}}:
+    type: OS::Heat::SoftwareDeployment
+    depends_on:
+      {% for dep in roles %}
+      - {{dep.name}}Deployment_Step{{step}}
+      - {{dep.name}}ContainersDeployment_Step{{step}}
+      {% endfor %}
+      - {{primary_role_name}}DockerPuppetJsonDeployment{{step}}
+    properties:
+      name: {{primary_role_name}}DockerPuppetJsonDeployment{{step}}
+      server: {get_param: [servers, {{primary_role_name}}, '0']}
+      config: {get_resource: {{primary_role_name}}DockerPuppetTasksConfig{{step}}}
+      input_values:
+        CONFIG: /var/lib/docker-puppet/docker-puppet-tasks{{step}}.json
+        NET_HOST: 'true'
+        NO_ARCHIVE: 'true'
+        STEP: {{step}}
+
+{% endfor %}
+# END primary_role_name docker-puppet-tasks
+
+{% for role in roles %}
+  # Post deployment steps for all roles
+  # A single config is re-applied with an incrementing step number
+  # {{role.name}} Role steps
+  {{role.name}}ArtifactsConfig:
+    type: ../puppet/deploy-artifacts.yaml
+
+  {{role.name}}ArtifactsDeploy:
+    type: OS::Heat::StructuredDeploymentGroup
+    properties:
+      servers:  {get_param: [servers, {{role.name}}]}
+      config: {get_resource: {{role.name}}ArtifactsConfig}
+
+  {{role.name}}PreConfig:
+    type: OS::TripleO::Tasks::{{role.name}}PreConfig
+    properties:
+      servers: {get_param: [servers, {{role.name}}]}
+      input_values:
+        update_identifier: {get_param: DeployIdentifier}
+
+  {{role.name}}CreateConfigDir:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      config: {get_file: create-config-dir.sh}
+
+  {{role.name}}CreateConfigDirDeployment:
+    type: OS::Heat::SoftwareDeploymentGroup
+    properties:
+      servers: {get_param: [servers, {{role.name}}]}
+      config: {get_resource: {{role.name}}CreateConfigDir}
+
+  # this creates a JSON config file for our docker-puppet.py script
+  {{role.name}}GenPuppetConfig:
+    type: OS::Heat::StructuredConfig
+    properties:
+      group: json-file
+      config:
+        /var/lib/docker-puppet/docker-puppet.json:
+          {get_param: [role_data, {{role.name}}, puppet_config]}
+
+  {{role.name}}GenPuppetDeployment:
+    type: OS::Heat::SoftwareDeploymentGroup
+    properties:
+      servers: {get_param: [servers, {{role.name}}]}
+      config: {get_resource: {{role.name}}GenPuppetConfig}
+
+  {{role.name}}GenerateConfig:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      config: {get_file: docker-puppet.py}
+
+  {{role.name}}GenerateConfigDeployment:
+    type: OS::Heat::SoftwareDeploymentGroup
+    depends_on: [{{role.name}}GenPuppetDeployment, {{role.name}}ArtifactsDeploy, {{role.name}}CreateConfigDirDeployment]
+    properties:
+      name: {{role.name}}GenerateConfigDeployment
+      servers: {get_param: [servers, {{role.name}}]}
+      config: {get_resource: {{role.name}}GenerateConfig}
+
+  {{role.name}}PuppetStepConfig:
+    type: OS::Heat::Value
+    properties:
+      type: string
+      value:
+        yaql:
+          expression:
+            # select 'step_config' only from services that do not have a docker_image
+            $.data.service_names.zip($.data.step_config, $.data.docker_image).where($[2] = null).where($[1] != null).select($[1]).join("\n")
+          data:
+            service_names: {get_param: [role_data, {{role.name}}, service_names]}
+            step_config: {get_param: [role_data, {{role.name}}, step_config]}
+            docker_image: {get_param: [role_data, {{role.name}}, docker_image]}
+
+  {{role.name}}DockerConfig:
+    type: OS::Heat::Value
+    properties:
+      type: json
+      value:
+        yaql:
+          expression:
+            # select 'docker_config' only from services that have a docker_image
+            $.data.service_names.zip($.data.docker_config, $.data.docker_image).where($[2] != null).select($[1]).reduce($1.mergeWith($2), {})
+          data:
+            service_names: {get_param: [role_data, {{role.name}}, service_names]}
+            docker_config: {get_param: [role_data, {{role.name}}, docker_config]}
+            docker_image: {get_param: [role_data, {{role.name}}, docker_image]}
+
+  # Here we are dumping all the docker container startup configuration data
+  # so that we can have access to how they are started outside of heat
+  # and docker-cmd.  This lets us create command line tools to start and
+  # test these containers.
+  {{role.name}}DockerConfigJsonStartupData:
+    type: OS::Heat::StructuredConfig
+    properties:
+      group: json-file
+      config:
+        /var/lib/docker-container-startup-configs.json:
+          {get_attr: [{{role.name}}DockerConfig, value]}
+
+  {{role.name}}DockerConfigJsonStartupDataDeployment:
+    type: OS::Heat::SoftwareDeploymentGroup
+    properties:
+      config: {get_resource: {{role.name}}DockerConfigJsonStartupData}
+      servers: {get_param: [servers, {{role.name}}]}
+
+  {{role.name}}KollaJsonConfig:
+    type: OS::Heat::StructuredConfig
+    properties:
+      group: json-file
+      config:
+        {get_param: [role_data, {{role.name}}, kolla_config]}
+
+  {{role.name}}KollaJsonDeployment:
+    type: OS::Heat::SoftwareDeploymentGroup
+    properties:
+      name: {{role.name}}KollaJsonDeployment
+      config: {get_resource: {{role.name}}KollaJsonConfig}
+      servers: {get_param: [servers, {{role.name}}]}
+
+  # BEGIN BAREMETAL CONFIG STEPS
+
+  {% if role.name == 'Controller' %}
+  ControllerPrePuppet:
+    type: OS::TripleO::Tasks::ControllerPrePuppet
+    properties:
+      servers: {get_param: [servers, Controller]}
+      input_values:
+        update_identifier: {get_param: DeployIdentifier}
+  {% endif %}
+
+  {{role.name}}Config:
+    type: OS::TripleO::{{role.name}}Config
+    properties:
+      StepConfig: {get_attr: [{{role.name}}PuppetStepConfig, value]}
+
+  {% for step in range(1, 6) %}
+
+  {{role.name}}Deployment_Step{{step}}:
+    type: OS::Heat::StructuredDeploymentGroup
+  {% if step == 1 %}
+    depends_on: [{{role.name}}PreConfig, {{role.name}}ArtifactsDeploy]
+  {% else %}
+    depends_on:
+      {% for dep in roles %}
+      - {{dep.name}}Deployment_Step{{step -1}}
+      - {{dep.name}}ContainersDeployment_Step{{step -1}}
+      {% endfor %}
+      - {{primary_role_name}}DockerPuppetTasksDeployment{{step -1}}
+  {% endif %}
+    properties:
+      name: {{role.name}}Deployment_Step{{step}}
+      servers: {get_param: [servers, {{role.name}}]}
+      config: {get_resource: {{role.name}}Config}
+      input_values:
+        step: {{step}}
+        update_identifier: {get_param: DeployIdentifier}
+
+  {% endfor %}
+  # END BAREMETAL CONFIG STEPS
+
+  # BEGIN CONTAINER CONFIG STEPS
+  {% for step in range(1, 6) %}
+
+  {{role.name}}ContainersConfig_Step{{step}}:
+    type: OS::Heat::StructuredConfig
+    properties:
+      group: docker-cmd
+      config:
+        {get_attr: [{{role.name}}DockerConfig, value, step_{{step}}]}
+
+  {{role.name}}ContainersDeployment_Step{{step}}:
+    type: OS::Heat::StructuredDeploymentGroup
+  {% if step == 1 %}
+    depends_on:
+      - {{role.name}}PreConfig
+      - {{role.name}}KollaJsonDeployment
+      - {{role.name}}GenPuppetDeployment
+      - {{role.name}}GenerateConfigDeployment
+  {% else %}
+    depends_on:
+        {% for dep in roles %}
+        - {{dep.name}}ContainersDeployment_Step{{step -1}}
+        - {{dep.name}}Deployment_Step{{step}} # baremetal steps of the same level run first
+        - {{dep.name}}Deployment_Step{{step -1}}
+        {% endfor %}
+        - {{primary_role_name}}DockerPuppetTasksDeployment{{step -1}}
+  {% endif %}
+    properties:
+      name: {{role.name}}ContainersDeployment_Step{{step}}
+      servers: {get_param: [servers, {{role.name}}]}
+      config: {get_resource: {{role.name}}ContainersConfig_Step{{step}}}
+
+  {% endfor %}
+  # END CONTAINER CONFIG STEPS
+
+  {{role.name}}PostConfig:
+    type: OS::TripleO::Tasks::{{role.name}}PostConfig
+    depends_on:
+  {% for dep in roles %}
+      - {{dep.name}}Deployment_Step5
+      - {{primary_role_name}}DockerPuppetTasksDeployment5
+  {% endfor %}
+    properties:
+      servers:  {get_param: servers}
+      input_values:
+        update_identifier: {get_param: DeployIdentifier}
+
+  # Note, this should come last, so use depends_on to ensure
+  # this is created after any other resources.
+  {{role.name}}ExtraConfigPost:
+    depends_on:
+  {% for dep in roles %}
+      - {{dep.name}}PostConfig
+  {% endfor %}
+    type: OS::TripleO::NodeExtraConfigPost
+    properties:
+        servers: {get_param: [servers, {{role.name}}]}
+
+  {% if role.name == 'Controller' %}
+  ControllerPostPuppet:
+    depends_on:
+      - ControllerExtraConfigPost
+    type: OS::TripleO::Tasks::ControllerPostPuppet
+    properties:
+      servers: {get_param: [servers, Controller]}
+      input_values:
+        update_identifier: {get_param: DeployIdentifier}
+  {% endif %}
+
+{% endfor %}
diff --git a/docker/post-upgrade.j2.yaml b/docker/post-upgrade.j2.yaml
new file mode 100644
index 00000000..4477f868
--- /dev/null
+++ b/docker/post-upgrade.j2.yaml
@@ -0,0 +1,4 @@
+# Note the include here is the same as post.j2.yaml but the data used at
+# # the time of rendering is different if any roles disable upgrades
+{% set roles = roles|rejectattr('disable_upgrade_deployment')|list -%}
+{% include 'docker-steps.j2' %}
diff --git a/docker/post.j2.yaml b/docker/post.j2.yaml
index 76232d14..fd956215 100644
--- a/docker/post.j2.yaml
+++ b/docker/post.j2.yaml
@@ -1,325 +1 @@
-# certain initialization steps (run in a container) will occur
-# on the first role listed in the roles file
-{% set primary_role_name = roles[0].name -%}
-
-heat_template_version: ocata
-
-description: >
-  Post-deploy configuration steps via puppet for all roles,
-  as defined in ../roles_data.yaml
-
-parameters:
-  servers:
-    type: json
-    description: Mapping of Role name e.g Controller to a list of servers
-  role_data:
-    type: json
-    description: Mapping of Role name e.g Controller to the per-role data
-  DeployIdentifier:
-    default: ''
-    type: string
-    description: >
-      Setting this to a unique value will re-run any deployment tasks which
-      perform configuration on a Heat stack-update.
-  EndpointMap:
-    default: {}
-    description: Mapping of service endpoint -> protocol. Typically set
-                 via parameter_defaults in the resource registry.
-    type: json
-
-resources:
-
-  # These utility tasks use docker-puppet.py to execute tasks via puppet
-  # We only execute these on the first node in the primary role
-  {{primary_role_name}}DockerPuppetTasks:
-    type: OS::Heat::Value
-    properties:
-      type: json
-      value:
-        yaql:
-          expression:
-            dict($.data.docker_puppet_tasks.where($1 != null).selectMany($.items()).groupBy($[0], $[1]))
-          data:
-            docker_puppet_tasks: {get_param: [role_data, {{primary_role_name}}, docker_puppet_tasks]}
-
-# BEGIN primary_role_name docker-puppet-tasks (run only on a single node)
-{% for step in range(1, 6) %}
-
-  {{primary_role_name}}DockerPuppetJsonConfig{{step}}:
-      type: OS::Heat::StructuredConfig
-      properties:
-        group: json-file
-        config:
-          /var/lib/docker-puppet/docker-puppet-tasks{{step}}.json:
-            {get_attr: [{{primary_role_name}}DockerPuppetTasks, value, 'step_{{step}}']}
-
-  {{primary_role_name}}DockerPuppetJsonDeployment{{step}}:
-    type: OS::Heat::SoftwareDeployment
-    properties:
-      server: {get_param: [servers, {{primary_role_name}}, '0']}
-      config: {get_resource: {{primary_role_name}}DockerPuppetJsonConfig{{step}}}
-
-  {{primary_role_name}}DockerPuppetTasksConfig{{step}}:
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: script
-      config: {get_file: docker-puppet.py}
-      inputs:
-        - name: CONFIG
-        - name: NET_HOST
-        - name: NO_ARCHIVE
-        - name: STEP
-
-  {{primary_role_name}}DockerPuppetTasksDeployment{{step}}:
-    type: OS::Heat::SoftwareDeployment
-    depends_on:
-      {% for dep in roles %}
-      - {{dep.name}}Deployment_Step{{step}}
-      - {{dep.name}}ContainersDeployment_Step{{step}}
-      {% endfor %}
-      - {{primary_role_name}}DockerPuppetJsonDeployment{{step}}
-    properties:
-      name: {{primary_role_name}}DockerPuppetJsonDeployment{{step}}
-      server: {get_param: [servers, {{primary_role_name}}, '0']}
-      config: {get_resource: {{primary_role_name}}DockerPuppetTasksConfig{{step}}}
-      input_values:
-        CONFIG: /var/lib/docker-puppet/docker-puppet-tasks{{step}}.json
-        NET_HOST: 'true'
-        NO_ARCHIVE: 'true'
-        STEP: {{step}}
-
-{% endfor %}
-# END primary_role_name docker-puppet-tasks
-
-{% for role in roles %}
-  # Post deployment steps for all roles
-  # A single config is re-applied with an incrementing step number
-  # {{role.name}} Role steps
-  {{role.name}}ArtifactsConfig:
-    type: ../puppet/deploy-artifacts.yaml
-
-  {{role.name}}ArtifactsDeploy:
-    type: OS::Heat::StructuredDeploymentGroup
-    properties:
-      servers:  {get_param: [servers, {{role.name}}]}
-      config: {get_resource: {{role.name}}ArtifactsConfig}
-
-  {{role.name}}PreConfig:
-    type: OS::TripleO::Tasks::{{role.name}}PreConfig
-    properties:
-      servers: {get_param: [servers, {{role.name}}]}
-      input_values:
-        update_identifier: {get_param: DeployIdentifier}
-
-  {{role.name}}CreateConfigDir:
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: script
-      config: {get_file: create-config-dir.sh}
-
-  {{role.name}}CreateConfigDirDeployment:
-    type: OS::Heat::SoftwareDeploymentGroup
-    properties:
-      servers: {get_param: [servers, {{role.name}}]}
-      config: {get_resource: {{role.name}}CreateConfigDir}
-
-  # this creates a JSON config file for our docker-puppet.py script
-  {{role.name}}GenPuppetConfig:
-    type: OS::Heat::StructuredConfig
-    properties:
-      group: json-file
-      config:
-        /var/lib/docker-puppet/docker-puppet.json:
-          {get_param: [role_data, {{role.name}}, puppet_config]}
-
-  {{role.name}}GenPuppetDeployment:
-    type: OS::Heat::SoftwareDeploymentGroup
-    properties:
-      servers: {get_param: [servers, {{role.name}}]}
-      config: {get_resource: {{role.name}}GenPuppetConfig}
-
-  {{role.name}}GenerateConfig:
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: script
-      config: {get_file: docker-puppet.py}
-
-  {{role.name}}GenerateConfigDeployment:
-    type: OS::Heat::SoftwareDeploymentGroup
-    depends_on: [{{role.name}}GenPuppetDeployment, {{role.name}}ArtifactsDeploy, {{role.name}}CreateConfigDirDeployment]
-    properties:
-      name: {{role.name}}GenerateConfigDeployment
-      servers: {get_param: [servers, {{role.name}}]}
-      config: {get_resource: {{role.name}}GenerateConfig}
-
-  {{role.name}}PuppetStepConfig:
-    type: OS::Heat::Value
-    properties:
-      type: string
-      value:
-        yaql:
-          expression:
-            # select 'step_config' only from services that do not have a docker_image
-            $.data.service_names.zip($.data.step_config, $.data.docker_image).where($[2] = null).where($[1] != null).select($[1]).join("\n")
-          data:
-            service_names: {get_param: [role_data, {{role.name}}, service_names]}
-            step_config: {get_param: [role_data, {{role.name}}, step_config]}
-            docker_image: {get_param: [role_data, {{role.name}}, docker_image]}
-
-  {{role.name}}DockerConfig:
-    type: OS::Heat::Value
-    properties:
-      type: json
-      value:
-        yaql:
-          expression:
-            # select 'docker_config' only from services that have a docker_image
-            $.data.service_names.zip($.data.docker_config, $.data.docker_image).where($[2] != null).select($[1]).reduce($1.mergeWith($2), {})
-          data:
-            service_names: {get_param: [role_data, {{role.name}}, service_names]}
-            docker_config: {get_param: [role_data, {{role.name}}, docker_config]}
-            docker_image: {get_param: [role_data, {{role.name}}, docker_image]}
-
-  # Here we are dumping all the docker container startup configuration data
-  # so that we can have access to how they are started outside of heat
-  # and docker-cmd.  This lets us create command line tools to start and
-  # test these containers.
-  {{role.name}}DockerConfigJsonStartupData:
-    type: OS::Heat::StructuredConfig
-    properties:
-      group: json-file
-      config:
-        /var/lib/docker-container-startup-configs.json:
-          {get_attr: [{{role.name}}DockerConfig, value]}
-
-  {{role.name}}DockerConfigJsonStartupDataDeployment:
-    type: OS::Heat::SoftwareDeploymentGroup
-    properties:
-      config: {get_resource: {{role.name}}DockerConfigJsonStartupData}
-      servers: {get_param: [servers, {{role.name}}]}
-
-  {{role.name}}KollaJsonConfig:
-    type: OS::Heat::StructuredConfig
-    properties:
-      group: json-file
-      config:
-        {get_param: [role_data, {{role.name}}, kolla_config]}
-
-  {{role.name}}KollaJsonDeployment:
-    type: OS::Heat::SoftwareDeploymentGroup
-    properties:
-      name: {{role.name}}KollaJsonDeployment
-      config: {get_resource: {{role.name}}KollaJsonConfig}
-      servers: {get_param: [servers, {{role.name}}]}
-
-  # BEGIN BAREMETAL CONFIG STEPS
-
-  {% if role.name == 'Controller' %}
-  ControllerPrePuppet:
-    type: OS::TripleO::Tasks::ControllerPrePuppet
-    properties:
-      servers: {get_param: [servers, Controller]}
-      input_values:
-        update_identifier: {get_param: DeployIdentifier}
-  {% endif %}
-
-  {{role.name}}Config:
-    type: OS::TripleO::{{role.name}}Config
-    properties:
-      StepConfig: {get_attr: [{{role.name}}PuppetStepConfig, value]}
-
-  {% for step in range(1, 6) %}
-
-  {{role.name}}Deployment_Step{{step}}:
-    type: OS::Heat::StructuredDeploymentGroup
-  {% if step == 1 %}
-    depends_on: [{{role.name}}PreConfig, {{role.name}}ArtifactsDeploy]
-  {% else %}
-    depends_on:
-      {% for dep in roles %}
-      - {{dep.name}}Deployment_Step{{step -1}}
-      - {{dep.name}}ContainersDeployment_Step{{step -1}}
-      {% endfor %}
-      - {{primary_role_name}}DockerPuppetTasksDeployment{{step -1}}
-  {% endif %}
-    properties:
-      name: {{role.name}}Deployment_Step{{step}}
-      servers: {get_param: [servers, {{role.name}}]}
-      config: {get_resource: {{role.name}}Config}
-      input_values:
-        step: {{step}}
-        update_identifier: {get_param: DeployIdentifier}
-
-  {% endfor %}
-  # END BAREMETAL CONFIG STEPS
-
-  # BEGIN CONTAINER CONFIG STEPS
-  {% for step in range(1, 6) %}
-
-  {{role.name}}ContainersConfig_Step{{step}}:
-    type: OS::Heat::StructuredConfig
-    properties:
-      group: docker-cmd
-      config:
-        {get_attr: [{{role.name}}DockerConfig, value, step_{{step}}]}
-
-  {{role.name}}ContainersDeployment_Step{{step}}:
-    type: OS::Heat::StructuredDeploymentGroup
-  {% if step == 1 %}
-    depends_on:
-      - {{role.name}}PreConfig
-      - {{role.name}}KollaJsonDeployment
-      - {{role.name}}GenPuppetDeployment
-      - {{role.name}}GenerateConfigDeployment
-  {% else %}
-    depends_on:
-        {% for dep in roles %}
-        - {{dep.name}}ContainersDeployment_Step{{step -1}}
-        - {{dep.name}}Deployment_Step{{step}} # baremetal steps of the same level run first
-        - {{dep.name}}Deployment_Step{{step -1}}
-        {% endfor %}
-        - {{primary_role_name}}DockerPuppetTasksDeployment{{step -1}}
-  {% endif %}
-    properties:
-      name: {{role.name}}ContainersDeployment_Step{{step}}
-      servers: {get_param: [servers, {{role.name}}]}
-      config: {get_resource: {{role.name}}ContainersConfig_Step{{step}}}
-
-  {% endfor %}
-  # END CONTAINER CONFIG STEPS
-
-  {{role.name}}PostConfig:
-    type: OS::TripleO::Tasks::{{role.name}}PostConfig
-    depends_on:
-  {% for dep in roles %}
-      - {{dep.name}}Deployment_Step5
-      - {{primary_role_name}}DockerPuppetTasksDeployment5
-  {% endfor %}
-    properties:
-      servers:  {get_param: servers}
-      input_values:
-        update_identifier: {get_param: DeployIdentifier}
-
-  # Note, this should come last, so use depends_on to ensure
-  # this is created after any other resources.
-  {{role.name}}ExtraConfigPost:
-    depends_on:
-  {% for dep in roles %}
-      - {{dep.name}}PostConfig
-  {% endfor %}
-    type: OS::TripleO::NodeExtraConfigPost
-    properties:
-        servers: {get_param: [servers, {{role.name}}]}
-
-  {% if role.name == 'Controller' %}
-  ControllerPostPuppet:
-    depends_on:
-      - ControllerExtraConfigPost
-    type: OS::TripleO::Tasks::ControllerPostPuppet
-    properties:
-      servers: {get_param: [servers, Controller]}
-      input_values:
-        update_identifier: {get_param: DeployIdentifier}
-  {% endif %}
-
-{% endfor %}
+{% include 'docker-steps.j2' %}
diff --git a/docker/services/database/mongodb.yaml b/docker/services/database/mongodb.yaml
index 3c4146fd..e83f4f19 100644
--- a/docker/services/database/mongodb.yaml
+++ b/docker/services/database/mongodb.yaml
@@ -50,7 +50,6 @@ outputs:
           - "\n"
           - - "['Mongodb_database', 'Mongodb_user', 'Mongodb_replset'].each |String $val| { noop_resource($val) }"
             - {get_attr: [MongodbPuppetBase, role_data, step_config]}
-      upgrade_tasks: {get_attr: [MongodbPuppetBase, role_data, upgrade_tasks]}
       # BEGIN DOCKER SETTINGS #
       docker_image: &mongodb_image
         list_join:
@@ -100,3 +99,7 @@ outputs:
           volumes:
           - "mongodb:/var/lib/mongodb/"
           - "logs:/var/log/kolla:ro"
+      upgrade_tasks:
+        - name: Stop and disable mongodb service
+          tags: step2
+          service: name=mongod state=stopped enabled=no
diff --git a/docker/services/database/mysql.yaml b/docker/services/database/mysql.yaml
index fd92e99c..c34ebe93 100644
--- a/docker/services/database/mysql.yaml
+++ b/docker/services/database/mysql.yaml
@@ -59,7 +59,6 @@ outputs:
           - "\n"
           - - "['Mysql_datadir', 'Mysql_user', 'Mysql_database', 'Mysql_grant', 'Mysql_plugin'].each |String $val| { noop_resource($val) }"
             - {get_attr: [MysqlPuppetBase, role_data, step_config]}
-      upgrade_tasks: {get_attr: [MysqlPuppetBase, role_data, upgrade_tasks]}
       # BEGIN DOCKER SETTINGS #
       docker_image: &mysql_image
         list_join:
@@ -132,3 +131,7 @@ outputs:
           volumes:
             - "mariadb:/var/lib/mysql/:ro"
             - "/var/lib/config-data/mysql/root:/root:ro" #provides .my.cnf
+      upgrade_tasks:
+        - name: Stop and disable mysql service
+          tags: step2
+          service: name=mariadb state=stopped enabled=no
diff --git a/docker/services/glance-api.yaml b/docker/services/glance-api.yaml
index ab62f7e3..73d76ad5 100644
--- a/docker/services/glance-api.yaml
+++ b/docker/services/glance-api.yaml
@@ -97,3 +97,7 @@ outputs:
             volumes: *glance_volumes
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable glance_api service
+          tags: step2
+          service: name=openstack-glance-api state=stopped enabled=no
diff --git a/docker/services/heat-api-cfn.yaml b/docker/services/heat-api-cfn.yaml
index fbaacbe6..2f54c0f1 100644
--- a/docker/services/heat-api-cfn.yaml
+++ b/docker/services/heat-api-cfn.yaml
@@ -91,3 +91,7 @@ outputs:
               - /dev:/dev
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable heat_api_cfn service
+          tags: step2
+          service: name=openstack-heat-api-cfn state=stopped enabled=no
diff --git a/docker/services/heat-api.yaml b/docker/services/heat-api.yaml
index df9160f4..a212d254 100644
--- a/docker/services/heat-api.yaml
+++ b/docker/services/heat-api.yaml
@@ -91,3 +91,7 @@ outputs:
               - /dev:/dev
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable heat_api service
+          tags: step2
+          service: name=openstack-heat-api state=stopped enabled=no
diff --git a/docker/services/heat-engine.yaml b/docker/services/heat-engine.yaml
index c8259f93..c60a3840 100644
--- a/docker/services/heat-engine.yaml
+++ b/docker/services/heat-engine.yaml
@@ -93,3 +93,7 @@ outputs:
               - /etc/localtime:/etc/localtime:ro
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable heat_engine service
+          tags: step2
+          service: name=openstack-heat-engine state=stopped enabled=no
diff --git a/docker/services/ironic-api.yaml b/docker/services/ironic-api.yaml
index 97ba9706..ca42c9ec 100644
--- a/docker/services/ironic-api.yaml
+++ b/docker/services/ironic-api.yaml
@@ -100,3 +100,7 @@ outputs:
               - /etc/localtime:/etc/localtime:ro
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable ironic_api service
+          tags: step2
+          service: name=openstack-ironic-api state=stopped enabled=no
diff --git a/docker/services/ironic-conductor.yaml b/docker/services/ironic-conductor.yaml
index 2cc29054..ff470008 100644
--- a/docker/services/ironic-conductor.yaml
+++ b/docker/services/ironic-conductor.yaml
@@ -112,3 +112,7 @@ outputs:
               - ironic:/var/lib/ironic
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable ironic_conductor service
+          tags: step2
+          service: name=openstack-ironic-conductor state=stopped enabled=no
diff --git a/docker/services/keystone.yaml b/docker/services/keystone.yaml
index f9b94a74..358277a5 100644
--- a/docker/services/keystone.yaml
+++ b/docker/services/keystone.yaml
@@ -154,3 +154,7 @@ outputs:
             list_join:
             - '/'
             - [ {get_param: DockerNamespace}, {get_param: DockerKeystoneImage} ]
+      upgrade_tasks:
+        - name: Stop and disable keystone service (running under httpd)
+          tags: step2
+          service: name=httpd state=stopped enabled=no
diff --git a/docker/services/memcached.yaml b/docker/services/memcached.yaml
index 3e51f5b4..9467567f 100644
--- a/docker/services/memcached.yaml
+++ b/docker/services/memcached.yaml
@@ -70,3 +70,7 @@ outputs:
             command: ['/bin/bash', '-c', 'source /etc/sysconfig/memcached; /usr/bin/memcached -p ${PORT} -u ${USER} -m ${CACHESIZE} -c ${MAXCONN} $OPTIONS']
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable memcached service
+          tags: step2
+          service: name=memcached state=stopped enabled=no
diff --git a/docker/services/mistral-api.yaml b/docker/services/mistral-api.yaml
index 843d5b23..7680bc62 100644
--- a/docker/services/mistral-api.yaml
+++ b/docker/services/mistral-api.yaml
@@ -116,3 +116,7 @@ outputs:
               - /etc/localtime:/etc/localtime:ro
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable mistral_api service
+          tags: step2
+          service: name=openstack-mistral-api state=stopped enabled=no
diff --git a/docker/services/mistral-engine.yaml b/docker/services/mistral-engine.yaml
index e50fc0a6..d61ab1c2 100644
--- a/docker/services/mistral-engine.yaml
+++ b/docker/services/mistral-engine.yaml
@@ -88,3 +88,8 @@ outputs:
               - /etc/localtime:/etc/localtime:ro
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable mistral_engine service
+          tags: step2
+          service: name=openstack-mistral-engine state=stopped enabled=no
+
diff --git a/docker/services/mistral-executor.yaml b/docker/services/mistral-executor.yaml
index 8833154e..42286426 100644
--- a/docker/services/mistral-executor.yaml
+++ b/docker/services/mistral-executor.yaml
@@ -92,3 +92,7 @@ outputs:
               - /var/lib/config-data/nova/etc/nova:/etc/nova:ro
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable mistral_executor service
+          tags: step2
+          service: name=openstack-mistral-executor state=stopped enabled=no
diff --git a/docker/services/neutron-api.yaml b/docker/services/neutron-api.yaml
index bf19586c..71389046 100644
--- a/docker/services/neutron-api.yaml
+++ b/docker/services/neutron-api.yaml
@@ -106,3 +106,7 @@ outputs:
               - /etc/localtime:/etc/localtime:ro
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable neutron_api service
+          tags: step2
+          service: name=neutron-server state=stopped enabled=no
diff --git a/docker/services/neutron-dhcp.yaml b/docker/services/neutron-dhcp.yaml
index 15f3055e..ccde63f2 100644
--- a/docker/services/neutron-dhcp.yaml
+++ b/docker/services/neutron-dhcp.yaml
@@ -94,3 +94,7 @@ outputs:
               - /run/:/run
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable neutron_dhcp service
+          tags: step2
+          service: name=neutron-dhcp-agent state=stopped enabled=no
diff --git a/docker/services/neutron-ovs-agent.yaml b/docker/services/neutron-ovs-agent.yaml
index f6b29292..6dcf91d9 100644
--- a/docker/services/neutron-ovs-agent.yaml
+++ b/docker/services/neutron-ovs-agent.yaml
@@ -85,3 +85,7 @@ outputs:
               - /run:/run
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable neutron_ovs_agent service
+          tags: step2
+          service: name=neutron-openvswitch-agent state=stopped enabled=no
diff --git a/docker/services/nova-api.yaml b/docker/services/nova-api.yaml
index 28b3a9d0..8a892325 100644
--- a/docker/services/nova-api.yaml
+++ b/docker/services/nova-api.yaml
@@ -145,3 +145,7 @@ outputs:
               - '/usr/bin/nova-manage'
               - 'cell_v2'
               - 'discover_hosts'
+      upgrade_tasks:
+        - name: Stop and disable nova_api service
+          tags: step2
+          service: name=openstack-nova-api state=stopped enabled=no
diff --git a/docker/services/nova-conductor.yaml b/docker/services/nova-conductor.yaml
index 73acd0ac..8bc81e32 100644
--- a/docker/services/nova-conductor.yaml
+++ b/docker/services/nova-conductor.yaml
@@ -86,3 +86,7 @@ outputs:
               - /etc/localtime:/etc/localtime:ro
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable nova_conductor service
+          tags: step2
+          service: name=openstack-nova-conductor state=stopped enabled=no
diff --git a/docker/services/nova-placement.yaml b/docker/services/nova-placement.yaml
index 6629843f..8da48d37 100644
--- a/docker/services/nova-placement.yaml
+++ b/docker/services/nova-placement.yaml
@@ -108,3 +108,7 @@ outputs:
               - /etc/localtime:/etc/localtime:ro
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable nova_placement service (running under httpd)
+          tags: step2
+          service: name=httpd state=stopped enabled=no
diff --git a/docker/services/nova-scheduler.yaml b/docker/services/nova-scheduler.yaml
index e3955a2b..c24d5b26 100644
--- a/docker/services/nova-scheduler.yaml
+++ b/docker/services/nova-scheduler.yaml
@@ -85,3 +85,7 @@ outputs:
               - /etc/localtime:/etc/localtime:ro
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable nova_scheduler service
+          tags: step2
+          service: name=openstack-nova-scheduler state=stopped enabled=no
diff --git a/docker/services/rabbitmq.yaml b/docker/services/rabbitmq.yaml
index aa604439..ed440718 100644
--- a/docker/services/rabbitmq.yaml
+++ b/docker/services/rabbitmq.yaml
@@ -120,3 +120,7 @@ outputs:
               - rabbitmq:/var/lib/rabbitmq/
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable rabbitmq service
+          tags: step2
+          service: name=rabbitmq-server state=stopped enabled=no
diff --git a/docker/services/services.yaml b/docker/services/services.yaml
index 125c4468..3f094ff8 100644
--- a/docker/services/services.yaml
+++ b/docker/services/services.yaml
@@ -75,3 +75,13 @@ outputs:
         {get_attr: [ServiceChain, role_data, docker_config]}
       docker_puppet_tasks:
         {get_attr: [ServiceChain, role_data, docker_puppet_tasks]}
+      upgrade_tasks:
+        yaql:
+          # Note we use distinct() here to filter any identical tasks, e.g yum update for all services
+          expression: $.data.where($ != null).select($.get('upgrade_tasks')).where($ != null).flatten().distinct()
+          data: {get_attr: [ServiceChain, role_data]}
+      upgrade_batch_tasks:
+        yaql:
+          # Note we use distinct() here to filter any identical tasks, e.g yum update for all services
+          expression: $.data.where($ != null).select($.get('upgrade_batch_tasks')).where($ != null).flatten().distinct()
+          data: {get_attr: [ServiceChain, role_data]}
diff --git a/docker/services/swift-proxy.yaml b/docker/services/swift-proxy.yaml
index cdfefe2f..66118412 100644
--- a/docker/services/swift-proxy.yaml
+++ b/docker/services/swift-proxy.yaml
@@ -77,3 +77,7 @@ outputs:
               - /dev:/dev
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable swift_proxy service
+          tags: step2
+          service: name=openstack-swift-proxy state=stopped enabled=no
diff --git a/docker/services/swift-storage.yaml b/docker/services/swift-storage.yaml
index ab9946f3..2eb55632 100644
--- a/docker/services/swift-storage.yaml
+++ b/docker/services/swift-storage.yaml
@@ -344,3 +344,20 @@ outputs:
               - swift-srv:/srv
               - /dev:/dev
             environment: *kolla_env
+      upgrade_tasks:
+        - name: Stop and disable swift storage services
+          tags: step2
+          service: name={{ item }} state=stopped enabled=no
+          with_items:
+            - openstack-swift-account-auditor
+            - openstack-swift-account-reaper
+            - openstack-swift-account-replicator
+            - openstack-swift-account
+            - openstack-swift-container-auditor
+            - openstack-swift-container-replicator
+            - openstack-swift-container-updater
+            - openstack-swift-container
+            - openstack-swift-object-auditor
+            - openstack-swift-object-replicator
+            - openstack-swift-object-updater
+            - openstack-swift-object
diff --git a/docker/services/zaqar.yaml b/docker/services/zaqar.yaml
index 1704278e..30905ffe 100644
--- a/docker/services/zaqar.yaml
+++ b/docker/services/zaqar.yaml
@@ -100,3 +100,8 @@ outputs:
               - /etc/localtime:/etc/localtime:ro
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable zaqar service
+          tags: step2
+          service: name=openstack-zaqar.service state=stopped enabled=no
+