From dbece39f549395c85701566edbff9fc19977665d Mon Sep 17 00:00:00 2001 From: Steven Hardy Date: Thu, 1 Dec 2016 10:00:57 +0000 Subject: Initial support for composable upgrades with Heat+Ansible This shows how we could wire in the upgrade steps using Ansible as was previously proposed e.g in https://review.openstack.org/#/c/321416/ but it's more closely integrated with the new composable services architecture. It's also very similar to the approach taken by SpinalStack where ansible snippets per-service were combined then run in a series of steps using Ansible tags. This patch just enables upgrade of keystone - we'll add support for other patches in subsequent patches. Partially-Implements: blueprint overcloud-upgrades-per-service Change-Id: I39f5426cb9da0b40bec4a7a3a4a353f69319bdf9 --- environments/major-upgrade-composable-steps.yaml | 3 + overcloud-resource-registry-puppet.j2.yaml | 4 + overcloud.j2.yaml | 21 +++++ puppet/major_upgrade_steps.j2.yaml | 98 ++++++++++++++++++++++++ puppet/services/README.rst | 30 +++++++- puppet/services/database/mysql.yaml | 8 ++ puppet/services/haproxy.yaml | 7 ++ puppet/services/keepalived.yaml | 8 ++ puppet/services/keystone.yaml | 11 +++ puppet/services/rabbitmq.yaml | 9 ++- puppet/services/services.yaml | 5 ++ puppet/services/tripleo-packages.yaml | 4 + puppet/upgrade_config.yaml | 48 ++++++++++++ 13 files changed, 253 insertions(+), 3 deletions(-) create mode 100644 environments/major-upgrade-composable-steps.yaml create mode 100644 puppet/major_upgrade_steps.j2.yaml create mode 100644 puppet/upgrade_config.yaml diff --git a/environments/major-upgrade-composable-steps.yaml b/environments/major-upgrade-composable-steps.yaml new file mode 100644 index 00000000..7e10014b --- /dev/null +++ b/environments/major-upgrade-composable-steps.yaml @@ -0,0 +1,3 @@ +resource_registry: + OS::TripleO::UpgradeSteps: ../puppet/major_upgrade_steps.yaml + OS::TripleO::PostDeploySteps: OS::Heat::None diff --git a/overcloud-resource-registry-puppet.j2.yaml b/overcloud-resource-registry-puppet.j2.yaml index 30b9f2b9..21013bdc 100644 --- a/overcloud-resource-registry-puppet.j2.yaml +++ b/overcloud-resource-registry-puppet.j2.yaml @@ -100,6 +100,10 @@ resource_registry: # validation resources OS::TripleO::AllNodes::Validation: all-nodes-validation.yaml + # Upgrade resources + OS::TripleO::UpgradeConfig: puppet/upgrade_config.yaml + OS::TripleO::UpgradeSteps: OS::Heat::None + # services OS::TripleO::Services: puppet/services/services.yaml OS::TripleO::Services::Apache: puppet/services/apache.yaml diff --git a/overcloud.j2.yaml b/overcloud.j2.yaml index ba1c6b36..e2c05eef 100644 --- a/overcloud.j2.yaml +++ b/overcloud.j2.yaml @@ -604,6 +604,21 @@ resources: {{role.name}}: {get_attr: [{{role.name}}ServiceChain, role_data]} {% endfor %} + # Upgrade steps for all roles + AllNodesUpgradeSteps: + type: OS::TripleO::UpgradeSteps + depends_on: AllNodesDeploySteps + properties: + servers: +{% for role in roles %} + {{role.name}}: {get_attr: [{{role.name}}, attributes, nova_server_resource]} +{% endfor %} + role_data: +{% for role in roles %} + {{role.name}}: {get_attr: [{{role.name}}ServiceChain, role_data]} +{% endfor %} + + outputs: ManagedEndpoints: description: Asserts that the keystone endpoints have been provisioned. @@ -635,4 +650,10 @@ outputs: value: {% for role in roles %} {{role.name}}: {get_attr: [{{role.name}}ServiceChain, role_data, service_names]} +{% endfor %} + RoleData: + description: The configuration data associated with each role + value: +{% for role in roles %} + {{role.name}}: {get_attr: [{{role.name}}ServiceChain, role_data]} {% endfor %} diff --git a/puppet/major_upgrade_steps.j2.yaml b/puppet/major_upgrade_steps.j2.yaml new file mode 100644 index 00000000..f8dad433 --- /dev/null +++ b/puppet/major_upgrade_steps.j2.yaml @@ -0,0 +1,98 @@ +heat_template_version: 2016-10-14 +description: 'Upgrade steps for all roles' + +parameters: + servers: + type: json + + role_data: + type: json + description: Mapping of Role name e.g Controller to the per-role data + + UpdateIdentifier: + type: string + description: > + Setting to a previously unused value during stack-update will trigger + the Upgrade resources to re-run on all roles. + + UpgradeInitCommand: + type: string + description: | + Command or script snippet to run on all overcloud nodes to + initialize the upgrade process. E.g. a repository switch. + default: '' + +resources: + + # For the UpgradeInit also rename /etc/resolv.conf.save for +bug/1567004 + UpgradeInitConfig: + type: OS::Heat::SoftwareConfig + properties: + group: script + config: + list_join: + - '' + - - "#!/bin/bash\n\n" + - "if [[ -f /etc/resolv.conf.save ]] ; then rm /etc/resolv.conf.save; fi\n\n" + - get_param: UpgradeInitCommand + +{% for role in roles %} + {{role.name}}Upgrade_Init: + type: OS::Heat::StructuredDeploymentGroup + properties: + name: {{role.name}}Upgrade_Init + servers: {get_param: [servers, {{role.name}}]} + config: {get_resource: UpgradeInitConfig} +{% endfor %} + +# Upgrade Steps for all roles +# FIXME(shardy): would be nice to make the number of steps configurable +{% for step in range(1, 8) %} + {% for role in roles %} + # Step {{step}} resources + {{role.name}}UpgradeConfig_Step{{step}}: + type: OS::TripleO::UpgradeConfig + # The UpgradeConfig resources could actually be created without + # serialization, but the event output is easier to follow if we + # do, and there should be minimal performance hit (creating the + # config is cheap compared to the time to apply the deployment). + depends_on: + {% if step == 1 %} + - {{role.name}}Upgrade_Init + {% else %} + {% for dep in roles %} + - {{dep.name}}Upgrade_Step{{step -1}} + {% endfor %} + {% endif %} + properties: + UpgradeStepConfig: {get_param: [role_data, {{role.name}}, upgrade_tasks]} + step: {{step}} + + {{role.name}}Upgrade_Step{{step}}: + type: OS::Heat::StructuredDeploymentGroup + {% if step > 1 %} + depends_on: + {% for dep in roles %} + - {{dep.name}}Upgrade_Step{{step -1}} + {% endfor %} + {% endif %} + properties: + name: {{role.name}}Upgrade_Step{{step}} + servers: {get_param: [servers, {{role.name}}]} + config: {get_resource: {{role.name}}UpgradeConfig_Step{{step}}} + input_values: + role: {{role.name}} + update_identifier: {get_param: UpdateIdentifier} + {% endfor %} +{% endfor %} + +outputs: + # Output the config for each role, just use Step1 as the config should be + # the same for all steps (only the tag provided differs) + upgrade_configs: + description: The per-role upgrade configuration used + value: +{% for role in roles %} + {{role.name.lower()}}: {get_attr: [{{role.name}}UpgradeConfig_Step1, upgrade_config]} +{% endfor %} + diff --git a/puppet/services/README.rst b/puppet/services/README.rst index 3accff36..856b306e 100644 --- a/puppet/services/README.rst +++ b/puppet/services/README.rst @@ -22,8 +22,8 @@ Config Settings Each service may define a config_settings output variable which returns Hiera settings to be configured. -Steps ------ +Deployment Steps +---------------- Each service may define an output variable which returns a puppet manifest snippet that will run at each of the following steps. Earlier manifests @@ -48,3 +48,29 @@ are re-asserted when applying latter ones. 4) General OpenStack Services 5) Service activation (Pacemaker) + +Upgrade Steps +------------- + +Each service template may optionally define a `upgrade_tasks` key, which is a +list of ansible tasks to be performed during the upgrade process. + +Similar to the step_config, we allow a series of steps for the per-service +upgrade sequence, defined as ansible tasks with a tag e.g "step1" for the first +step, "step2" for the second, etc. + + Steps/tages correlate to the following: + + 1) Quiesce the control-plane, e.g disable LoadBalancer, stop pacemaker cluster + + 2) Stop all control-plane services, ready for upgrade + + 3) Perform a package update, (either specific packages or the whole system) + + 4) Start services needed for migration tasks (e.g DB) + + 5) Perform any migration tasks, e.g DB sync commands + + 6) Start control-plane services + + 7) Any additional online migration tasks (e.g data migrations) diff --git a/puppet/services/database/mysql.yaml b/puppet/services/database/mysql.yaml index bed8f7d3..cacf6db0 100644 --- a/puppet/services/database/mysql.yaml +++ b/puppet/services/database/mysql.yaml @@ -92,3 +92,11 @@ outputs: $NETWORK: {get_param: [ServiceNetMap, MysqlNetwork]} step_config: | include ::tripleo::profile::base::database::mysql + upgrade_tasks: + - name: Stop service + tags: step2 + service: name=mariadb state=stopped + - name: Start service + tags: step4 + service: name=mariadb state=started + diff --git a/puppet/services/haproxy.yaml b/puppet/services/haproxy.yaml index c8edade5..675a79ec 100644 --- a/puppet/services/haproxy.yaml +++ b/puppet/services/haproxy.yaml @@ -77,3 +77,10 @@ outputs: - get_attr: [HAProxyInternalTLS, role_data, certificates_specs] step_config: | include ::tripleo::profile::base::haproxy + upgrade_tasks: + - name: Stop haproxy service + tags: step1 + service: name=haproxy state=stopped + - name: Start haproxy service + tags: step4 # Needed at step 4 for mysql + service: name=haproxy state=started diff --git a/puppet/services/keepalived.yaml b/puppet/services/keepalived.yaml index 6f2c44ec..b4f1a100 100644 --- a/puppet/services/keepalived.yaml +++ b/puppet/services/keepalived.yaml @@ -64,3 +64,11 @@ outputs: - tripleo::keepalived::public_virtual_interface: {get_param: PublicVirtualInterface} step_config: | include ::tripleo::profile::base::keepalived + upgrade_tasks: + - name: Stop keepalived service + tags: step1 + service: name=keepalived state=stopped + - name: Start keepalived service + tags: step4 # Needed at step 4 for mysql + service: name=keepalived state=started + diff --git a/puppet/services/keystone.yaml b/puppet/services/keystone.yaml index c2a282d4..f021e18b 100644 --- a/puppet/services/keystone.yaml +++ b/puppet/services/keystone.yaml @@ -248,3 +248,14 @@ outputs: keystone::db::mysql::allowed_hosts: - '%' - "%{hiera('mysql_bind_host')}" + # Ansible tasks to handle upgrade + upgrade_tasks: + - name: Stop keystone service (running under httpd) + tags: step2 + service: name=httpd state=stopped + - name: Sync keystone DB + tags: step5 + command: keystone-manage db_sync + - name: Start keystone service (running under httpd) + tags: step6 + service: name=httpd state=started diff --git a/puppet/services/rabbitmq.yaml b/puppet/services/rabbitmq.yaml index 44a09a42..b77e0a91 100644 --- a/puppet/services/rabbitmq.yaml +++ b/puppet/services/rabbitmq.yaml @@ -97,6 +97,13 @@ outputs: # internal_api_subnet - > IP/CIDR rabbitmq::node_ip_address: {get_param: [ServiceNetMap, RabbitmqNetwork]} rabbitmq::nr_ha_queues: {get_param: RabbitHAQueues} - step_config: | include ::tripleo::profile::base::rabbitmq + upgrade_tasks: + - name: Stop rabbitmq service + tags: step2 + service: name=rabbitmq-server state=stopped + - name: Start rabbitmq service + tags: step6 + service: name=rabbitmq-server state=started + diff --git a/puppet/services/services.yaml b/puppet/services/services.yaml index ffe2d2d4..13df5bbe 100644 --- a/puppet/services/services.yaml +++ b/puppet/services/services.yaml @@ -108,3 +108,8 @@ outputs: expression: $.data.role_data.where($ != null).select($.get('service_config_settings')).where($ != null).reduce($1.mergeWith($2), {}) data: {role_data: {get_attr: [ServiceChain, role_data]}} step_config: {list_join: ["\n", {get_attr: [ServiceChain, role_data, step_config]}]} + upgrade_tasks: + yaql: + # Note we use distinct() here to filter any identical tasks, e.g yum update for all services + expression: $.data.where($ != null).select($.get('upgrade_tasks')).where($ != null).flatten().distinct() + data: {get_attr: [ServiceChain, role_data]} diff --git a/puppet/services/tripleo-packages.yaml b/puppet/services/tripleo-packages.yaml index 124f5fe8..69912fa5 100644 --- a/puppet/services/tripleo-packages.yaml +++ b/puppet/services/tripleo-packages.yaml @@ -32,3 +32,7 @@ outputs: tripleo::packages::enable_install: {get_param: EnablePackageInstall} step_config: | include ::tripleo::packages + upgrade_tasks: + - name: Update all packages + tags: step3 + yum: name=* state=latest diff --git a/puppet/upgrade_config.yaml b/puppet/upgrade_config.yaml new file mode 100644 index 00000000..c67e10b3 --- /dev/null +++ b/puppet/upgrade_config.yaml @@ -0,0 +1,48 @@ +heat_template_version: 2016-10-14 +description: 'Upgrade for via ansible by applying a step related tag' + +parameters: + UpgradeStepConfig: + type: json + description: Config (ansible yaml) that will be used to step through the deployment. + default: '' + + step: + type: string + description: Step number of the upgrade + +resources: + + AnsibleConfig: + type: OS::Heat::Value + properties: + value: + str_replace: + template: CONFIG + params: + CONFIG: + - hosts: localhost + connection: local + tasks: {get_param: UpgradeStepConfig} + + AnsibleUpgradeConfigImpl: + type: OS::Heat::SoftwareConfig + properties: + group: ansible + options: + tags: + str_replace: + template: "stepSTEP" + params: + STEP: {get_param: step} + inputs: + - name: role + config: {get_attr: [AnsibleConfig, value]} + +outputs: + OS::stack_id: + description: The software config which runs ansible with tags + value: {get_resource: AnsibleUpgradeConfigImpl} + upgrade_config: + description: The configuration file used for upgrade + value: {get_attr: [AnsibleConfig, value]} -- cgit 1.2.3-korg