summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorSteven Hardy <shardy@redhat.com>2017-01-20 10:45:19 +0000
committerSteven Hardy <shardy@redhat.com>2017-01-25 21:03:01 +0000
commit1cdc514871f789eb75b13131670e2a753573449c (patch)
tree7e217761a9a126518fc98f3cc02ceea8686349b7
parent7dbd771a35e06bf1601e10c5d92e4b18955ce958 (diff)
Add support for batched upgrades to composable upgrades
Some services (e.g ceph mon) require upgrading in batches (the old upgrade architecture did the ceph mon upgrade one controller at a time). This interface enables doing the same, and over time we can probably move more services into this interface (e.g when services support rolling upgrades) to reduce downtime. Change-Id: If581f301a5493ef33ac1386bdc22f9fca4f2544e Partially-Implements: blueprint overcloud-upgrades-per-service
-rw-r--r--puppet/major_upgrade_steps.j2.yaml59
-rw-r--r--puppet/services/README.rst22
-rw-r--r--puppet/services/services.yaml5
-rw-r--r--roles_data.yaml5
4 files changed, 86 insertions, 5 deletions
diff --git a/puppet/major_upgrade_steps.j2.yaml b/puppet/major_upgrade_steps.j2.yaml
index 433b03a0..44a391b2 100644
--- a/puppet/major_upgrade_steps.j2.yaml
+++ b/puppet/major_upgrade_steps.j2.yaml
@@ -1,3 +1,4 @@
+{% set upgrade_steps_max = 8 -%}
heat_template_version: ocata
description: 'Upgrade steps for all roles'
@@ -17,9 +18,53 @@ parameters:
resources:
+# Upgrade Steps for all roles, batched updates
+# FIXME(shardy): would be nice to make the number of steps configurable
+{% for step in range(0, upgrade_steps_max) %}
+ {% for role in roles %}
+ # Step {{step}} resources
+ {{role.name}}UpgradeBatchConfig_Step{{step}}:
+ type: OS::TripleO::UpgradeConfig
+ # The UpgradeConfig resources could actually be created without
+ # serialization, but the event output is easier to follow if we
+ # do, and there should be minimal performance hit (creating the
+ # config is cheap compared to the time to apply the deployment).
+ {% if step > 0 %}
+ depends_on:
+ {% for dep in roles %}
+ - {{dep.name}}UpgradeBatch_Step{{step -1}}
+ {% endfor %}
+ {% endif %}
+ properties:
+ UpgradeStepConfig: {get_param: [role_data, {{role.name}}, upgrade_batch_tasks]}
+ step: {{step}}
+
+ {{role.name}}UpgradeBatch_Step{{step}}:
+ type: OS::Heat::StructuredDeploymentGroup
+ {% if step > 0 %}
+ depends_on:
+ {% for dep in roles %}
+ - {{dep.name}}UpgradeBatch_Step{{step -1}}
+ {% endfor %}
+ {% endif %}
+ update_policy:
+ batch_create:
+ max_batch_size: {{role.upgrade_batch_size|default(1)}}
+ rolling_update:
+ max_batch_size: {{role.upgrade_batch_size|default(1)}}
+ properties:
+ name: {{role.name}}UpgradeBatch_Step{{step}}
+ servers: {get_param: [servers, {{role.name}}]}
+ config: {get_resource: {{role.name}}UpgradeBatchConfig_Step{{step}}}
+ input_values:
+ role: {{role.name}}
+ update_identifier: {get_param: UpdateIdentifier}
+ {% endfor %}
+{% endfor %}
+
# Upgrade Steps for all roles
# FIXME(shardy): would be nice to make the number of steps configurable
-{% for step in range(0, 8) %}
+{% for step in range(0, upgrade_steps_max) %}
{% for role in roles %}
# Step {{step}} resources
{{role.name}}UpgradeConfig_Step{{step}}:
@@ -28,13 +73,17 @@ resources:
# serialization, but the event output is easier to follow if we
# do, and there should be minimal performance hit (creating the
# config is cheap compared to the time to apply the deployment).
- {% if step > 0 %}
depends_on:
+ {% if step > 0 %}
{% for dep in roles %}
{% if not dep.disable_upgrade_deployment|default(false) %}
- {{dep.name}}Upgrade_Step{{step -1}}
{% endif %}
{% endfor %}
+ {% else %}
+ {% for dep in roles %}
+ - {{dep.name}}UpgradeBatch_Step{{upgrade_steps_max -1}}
+ {% endfor %}
{% endif %}
properties:
UpgradeStepConfig: {get_param: [role_data, {{role.name}}, upgrade_tasks]}
@@ -42,13 +91,17 @@ resources:
{% if not role.disable_upgrade_deployment|default(false) %}
{{role.name}}Upgrade_Step{{step}}:
type: OS::Heat::StructuredDeploymentGroup
- {% if step > 0 %}
depends_on:
+ {% if step > 0 %}
{% for dep in roles %}
{% if not dep.disable_upgrade_deployment|default(false) %}
- {{dep.name}}Upgrade_Step{{step -1}}
{% endif %}
{% endfor %}
+ {% else %}
+ {% for dep in roles %}
+ - {{dep.name}}UpgradeBatch_Step{{upgrade_steps_max -1}}
+ {% endfor %}
{% endif %}
properties:
name: {{role.name}}Upgrade_Step{{step}}
diff --git a/puppet/services/README.rst b/puppet/services/README.rst
index 6e4e9c1d..34cb350b 100644
--- a/puppet/services/README.rst
+++ b/puppet/services/README.rst
@@ -49,6 +49,28 @@ are re-asserted when applying latter ones.
5) Service activation (Pacemaker)
+Batch Upgrade Steps
+-------------------
+
+Each service template may optionally define a `upgrade_batch_tasks` key, which
+is a list of ansible tasks to be performed during the upgrade process.
+
+Similar to the step_config, we allow a series of steps for the per-service
+upgrade sequence, defined as ansible tasks with a tag e.g "step1" for the first
+step, "step2" for the second, etc. Note that each step is performed in batches,
+then we move on to the next step which is also performed in batches (we don't
+perform all steps on one node, then move on to the next one which means you
+can sequence rolling upgrades of dependent services via the step value).
+
+The tasks performed at each step is service specific, but note that all batch
+upgrade steps are performed before the `upgrade_tasks` described below. This
+means that all services that support rolling upgrades can be upgraded without
+downtime during `upgrade_batch_tasks`, then any remaining services are stopped
+and upgraded during `upgrade_tasks`
+
+The default batch size is 1, but this can be overridden for each role via the
+`upgrade_batch_size` option in roles_data.yaml
+
Upgrade Steps
-------------
diff --git a/puppet/services/services.yaml b/puppet/services/services.yaml
index 90268c78..80da5352 100644
--- a/puppet/services/services.yaml
+++ b/puppet/services/services.yaml
@@ -118,4 +118,9 @@ outputs:
# Note we use distinct() here to filter any identical tasks, e.g yum update for all services
expression: $.data.where($ != null).select($.get('upgrade_tasks')).where($ != null).flatten().distinct()
data: {get_attr: [ServiceChain, role_data]}
+ upgrade_batch_tasks:
+ yaql:
+ # Note we use distinct() here to filter any identical tasks, e.g yum update for all services
+ expression: $.data.where($ != null).select($.get('upgrade_batch_tasks')).where($ != null).flatten().distinct()
+ data: {get_attr: [ServiceChain, role_data]}
service_metadata_settings: {get_attr: [ServiceServerMetadataHook, metadata]}
diff --git a/roles_data.yaml b/roles_data.yaml
index 90250aa8..530e4376 100644
--- a/roles_data.yaml
+++ b/roles_data.yaml
@@ -17,8 +17,9 @@
# disable_constraints: (boolean) optional, whether to disable Nova and Glance
# constraints for each role specified in the templates.
#
-# disable_upgrade_deployment: (boolean) optional, whether to run the composable upgrade
-# steps for all services that are deployed on the particular role.
+# upgrade_batch_size: (number): batch size for upgrades where tasks are
+# specified by services to run in batches vs all nodes at once.
+# This defaults to 1, but larger batches may be specified here.
#
# ServicesDefault: (list) optional default list of services to be deployed
# on the role, defaults to an empty list. Sets the default for the