Automatically backup and restore Swift rings from the undercloud

Swift rings created or updated on the overcloud nodes will now be
stored on the undercloud at the end of the deployment.  An
additional consistency check is executed before storing them,
ensuring all rings within the cluster are identical.

These rings will be retrieved (before Puppet runs) by every node
when an UPDATE is executed, and by doing this will be in a
consistent state across the cluster.

This makes it possible to add, remove or replace nodes in an
existing cluster without manual operator interaction.

Closes-Bug: 1609421
Depends-On: Ic3da38cffdd993c768bdb137c17d625dff1aa372
Change-Id: I758179182265da5160c06bb95f4c6258dc0edcd6

4 files changed, 94 insertions, 0 deletions

diff --git a/extraconfig/tasks/swift-ring-deploy.yaml b/extraconfig/tasks/swift-ring-deploy.yaml
new file mode 100644
index 00000000..d17f78ae
--- /dev/null
+++ b/extraconfig/tasks/swift-ring-deploy.yaml
@@ -0,0 +1,31 @@
+heat_template_version: ocata
+
+parameters:
+  servers:
+    type: json
+  SwiftRingGetTempurl:
+    default: ''
+    description: A temporary Swift URL to download rings from.
+    type: string
+
+resources:
+  SwiftRingDeployConfig:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      inputs:
+        - name: swift_ring_get_tempurl
+      config: |
+        #!/bin/sh
+        pushd /
+        curl --insecure --silent "${swift_ring_get_tempurl}" | tar xz || true
+        popd
+
+  SwiftRingDeploy:
+    type: OS::Heat::SoftwareDeployments
+    properties:
+      name: SwiftRingDeploy
+      config: {get_resource: SwiftRingDeployConfig}
+      servers:  {get_param: servers}
+      input_values:
+        swift_ring_get_tempurl: {get_param: SwiftRingGetTempurl}
diff --git a/extraconfig/tasks/swift-ring-update.yaml b/extraconfig/tasks/swift-ring-update.yaml
new file mode 100644
index 00000000..440c6883
--- /dev/null
+++ b/extraconfig/tasks/swift-ring-update.yaml
@@ -0,0 +1,42 @@
+heat_template_version: ocata
+
+parameters:
+  servers:
+    type: json
+  SwiftRingPutTempurl:
+    default: ''
+    description: A temporary Swift URL to upload rings to.
+    type: string
+
+resources:
+  SwiftRingUpdateConfig:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      inputs:
+        - name: swift_ring_put_tempurl
+      config: |
+        #!/bin/sh
+        TMP_DATA=$(mktemp -d)
+        function cleanup {
+          rm -Rf "$TMP_DATA"
+        }
+        trap cleanup EXIT
+        # sanity check in case rings are not consistent within cluster
+        swift-recon --md5 | grep -q "doesn't match" && exit 1
+        pushd ${TMP_DATA}
+        tar -cvzf swift-rings.tar.gz /etc/swift/*.builder /etc/swift/*.ring.gz /etc/swift/backups/*
+        resp=`curl --insecure --silent -X PUT "${swift_ring_put_tempurl}" --write-out "%{http_code}" --data-binary @swift-rings.tar.gz`
+        popd
+        if [ "$resp" != "201" ]; then
+            exit 1
+        fi
+
+  SwiftRingUpdate:
+    type: OS::Heat::SoftwareDeployments
+    properties:
+      name: SwiftRingUpdate
+      config: {get_resource: SwiftRingUpdateConfig}
+      servers: {get_param: servers}
+      input_values:
+        swift_ring_put_tempurl: {get_param: SwiftRingPutTempurl}
diff --git a/overcloud-resource-registry-puppet.j2.yaml b/overcloud-resource-registry-puppet.j2.yaml
index 471a0ff1..3dd16b05 100644
--- a/overcloud-resource-registry-puppet.j2.yaml
+++ b/overcloud-resource-registry-puppet.j2.yaml
@@ -11,6 +11,9 @@ resource_registry:
   OS::TripleO::Tasks::UpdateWorkflow: OS::Heat::None
   OS::TripleO::Tasks::PackageUpdate: extraconfig/tasks/yum_update.yaml
 
+  OS::TripleO::Tasks::SwiftRingDeploy: extraconfig/tasks/swift-ring-deploy.yaml
+  OS::TripleO::Tasks::SwiftRingUpdate: extraconfig/tasks/swift-ring-update.yaml
+
 {% for role in roles %}
   OS::TripleO::{{role.name}}::PreNetworkConfig: OS::Heat::None
   OS::TripleO::{{role.name}}PostDeploySteps: puppet/post.yaml
diff --git a/puppet/puppet-steps.j2 b/puppet/puppet-steps.j2
index c3b54ccd..4eca2333 100644
--- a/puppet/puppet-steps.j2
+++ b/puppet/puppet-steps.j2
@@ -32,6 +32,13 @@
         update_identifier: {get_param: DeployIdentifier}
   {% endif %}
 
+  {% if role.name in ['Controller', 'ObjectStorage'] %}
+  {{role.name}}SwiftRingDeploy:
+    type: OS::TripleO::Tasks::SwiftRingDeploy
+    properties:
+      servers: {get_param: [servers, {{role.name}}]}
+  {% endif %}
+
   # Step through a series of configuration steps
 {% for step in range(1, 6) %}
   {{role.name}}Deployment_Step{{step}}:
@@ -85,4 +92,15 @@
       input_values:
         update_identifier: {get_param: DeployIdentifier}
   {% endif %}
+
+  {% if role.name in ['Controller', 'ObjectStorage'] %}
+  {{role.name}}SwiftRingUpdate:
+    type: OS::TripleO::Tasks::SwiftRingUpdate
+    depends_on:
+  {% for dep in roles %}
+      - {{dep.name}}Deployment_Step5
+  {% endfor %}
+    properties:
+      servers: {get_param: [servers, {{role.name}}]}
+  {% endif %}
 {% endfor %}