Add support for batched upgrades to composable upgrades

Some services (e.g ceph mon) require upgrading in batches (the old
upgrade architecture did the ceph mon upgrade one controller at a
time).  This interface enables doing the same, and over time we
can probably move more services into this interface (e.g when
services support rolling upgrades) to reduce downtime.

Change-Id: If581f301a5493ef33ac1386bdc22f9fca4f2544e
Partially-Implements: blueprint overcloud-upgrades-per-service

4 files changed, 86 insertions, 5 deletions

diff --git a/puppet/major_upgrade_steps.j2.yaml b/puppet/major_upgrade_steps.j2.yaml
index 433b03a0..44a391b2 100644
--- a/puppet/major_upgrade_steps.j2.yaml
+++ b/puppet/major_upgrade_steps.j2.yaml
@@ -1,3 +1,4 @@
+{% set upgrade_steps_max = 8 -%}
 heat_template_version: ocata
 description: 'Upgrade steps for all roles'
 
@@ -17,9 +18,53 @@ parameters:
 
 resources:
 
+# Upgrade Steps for all roles, batched updates
+# FIXME(shardy): would be nice to make the number of steps configurable
+{% for step in range(0, upgrade_steps_max) %}
+  {% for role in roles %}
+  # Step {{step}} resources
+  {{role.name}}UpgradeBatchConfig_Step{{step}}:
+    type: OS::TripleO::UpgradeConfig
+  # The UpgradeConfig resources could actually be created without
+  # serialization, but the event output is easier to follow if we
+  # do, and there should be minimal performance hit (creating the
+  # config is cheap compared to the time to apply the deployment).
+  {% if step > 0 %}
+    depends_on:
+      {% for dep in roles %}
+      - {{dep.name}}UpgradeBatch_Step{{step -1}}
+      {% endfor %}
+  {% endif %}
+    properties:
+      UpgradeStepConfig: {get_param: [role_data, {{role.name}}, upgrade_batch_tasks]}
+      step: {{step}}
+
+  {{role.name}}UpgradeBatch_Step{{step}}:
+    type: OS::Heat::StructuredDeploymentGroup
+  {% if step > 0 %}
+    depends_on:
+      {% for dep in roles %}
+      - {{dep.name}}UpgradeBatch_Step{{step -1}}
+      {% endfor %}
+  {% endif %}
+    update_policy:
+      batch_create:
+        max_batch_size: {{role.upgrade_batch_size|default(1)}}
+      rolling_update:
+        max_batch_size: {{role.upgrade_batch_size|default(1)}}
+    properties:
+      name: {{role.name}}UpgradeBatch_Step{{step}}
+      servers: {get_param: [servers, {{role.name}}]}
+      config: {get_resource: {{role.name}}UpgradeBatchConfig_Step{{step}}}
+      input_values:
+        role: {{role.name}}
+        update_identifier: {get_param: UpdateIdentifier}
+  {% endfor %}
+{% endfor %}
+
 # Upgrade Steps for all roles
 # FIXME(shardy): would be nice to make the number of steps configurable
-{% for step in range(0, 8) %}
+{% for step in range(0, upgrade_steps_max) %}
   {% for role in roles %}
   # Step {{step}} resources
   {{role.name}}UpgradeConfig_Step{{step}}:
@@ -28,13 +73,17 @@ resources:
   # serialization, but the event output is easier to follow if we
   # do, and there should be minimal performance hit (creating the
   # config is cheap compared to the time to apply the deployment).
-  {% if step > 0 %}
     depends_on:
+  {% if step > 0 %}
       {% for dep in roles %}
         {% if not dep.disable_upgrade_deployment|default(false) %}
       - {{dep.name}}Upgrade_Step{{step -1}}
         {% endif %}
       {% endfor %}
+  {% else %}
+      {% for dep in roles %}
+      - {{dep.name}}UpgradeBatch_Step{{upgrade_steps_max -1}}
+      {% endfor %}
   {% endif %}
     properties:
       UpgradeStepConfig: {get_param: [role_data, {{role.name}}, upgrade_tasks]}
@@ -42,13 +91,17 @@ resources:
   {% if not role.disable_upgrade_deployment|default(false) %}
   {{role.name}}Upgrade_Step{{step}}:
     type: OS::Heat::StructuredDeploymentGroup
-  {% if step > 0 %}
     depends_on:
+  {% if step > 0 %}
       {% for dep in roles %}
         {% if not dep.disable_upgrade_deployment|default(false) %}
       - {{dep.name}}Upgrade_Step{{step -1}}
         {% endif %}
       {% endfor %}
+  {% else %}
+      {% for dep in roles %}
+      - {{dep.name}}UpgradeBatch_Step{{upgrade_steps_max -1}}
+      {% endfor %}
   {% endif %}
     properties:
       name: {{role.name}}Upgrade_Step{{step}}
diff --git a/puppet/services/README.rst b/puppet/services/README.rst
index 6e4e9c1d..34cb350b 100644
--- a/puppet/services/README.rst
+++ b/puppet/services/README.rst
@@ -49,6 +49,28 @@ are re-asserted when applying latter ones.
 
    5) Service activation (Pacemaker)
 
+Batch Upgrade Steps
+-------------------
+
+Each service template may optionally define a `upgrade_batch_tasks` key, which
+is a list of ansible tasks to be performed during the upgrade process.
+
+Similar to the step_config, we allow a series of steps for the per-service
+upgrade sequence, defined as ansible tasks with a tag e.g "step1" for the first
+step, "step2" for the second, etc. Note that each step is performed in batches,
+then we move on to the next step which is also performed in batches (we don't
+perform all steps on one node, then move on to the next one which means you
+can sequence rolling upgrades of dependent services via the step value).
+
+The tasks performed at each step is service specific, but note that all batch
+upgrade steps are performed before the `upgrade_tasks` described below.  This
+means that all services that support rolling upgrades can be upgraded without
+downtime during `upgrade_batch_tasks`, then any remaining services are stopped
+and upgraded during `upgrade_tasks`
+
+The default batch size is 1, but this can be overridden for each role via the
+`upgrade_batch_size` option in roles_data.yaml
+
 Upgrade Steps
 -------------
 
diff --git a/puppet/services/services.yaml b/puppet/services/services.yaml
index 90268c78..80da5352 100644
--- a/puppet/services/services.yaml
+++ b/puppet/services/services.yaml
@@ -118,4 +118,9 @@ outputs:
           # Note we use distinct() here to filter any identical tasks, e.g yum update for all services
           expression: $.data.where($ != null).select($.get('upgrade_tasks')).where($ != null).flatten().distinct()
           data: {get_attr: [ServiceChain, role_data]}
+      upgrade_batch_tasks:
+        yaql:
+          # Note we use distinct() here to filter any identical tasks, e.g yum update for all services
+          expression: $.data.where($ != null).select($.get('upgrade_batch_tasks')).where($ != null).flatten().distinct()
+          data: {get_attr: [ServiceChain, role_data]}
       service_metadata_settings: {get_attr: [ServiceServerMetadataHook, metadata]}
diff --git a/roles_data.yaml b/roles_data.yaml
index 90250aa8..530e4376 100644
--- a/roles_data.yaml
+++ b/roles_data.yaml
@@ -17,8 +17,9 @@
 # disable_constraints: (boolean) optional, whether to disable Nova and Glance
 # constraints for each role specified in the templates.
 #
-# disable_upgrade_deployment: (boolean) optional, whether to run the composable upgrade
-# steps for all services that are deployed on the particular role.
+# upgrade_batch_size: (number): batch size for upgrades where tasks are
+# specified by services to run in batches vs all nodes at once.
+# This defaults to 1, but larger batches may be specified here.
 #
 # ServicesDefault: (list) optional default list of services to be deployed
 # on the role, defaults to an empty list. Sets the default for the