From 2ada42c10949dfb668581d9690011a16383f23d5 Mon Sep 17 00:00:00 2001 From: Mathieu Bultel Date: Mon, 28 Aug 2017 17:24:47 +0200 Subject: Retry if the pacemaker_resource commands failed Add a retry when the pacemaker_resource command wasn't apply correctly, more info here: https://bugzilla.redhat.com/show_bug.cgi?id=1482116 This is the same approach puppet-pacemaker uses and provides eventual consistency when multiple nodes change the cluster CIB concurrently. This change depends-on : https://review.gerrithub.io/375982 The return code is not available in the current ansible-pacemaker package. Change-Id: I8da03f5c4a6d442617b81be5793a9724cc8842bf (cherry picked from commit e92430d8d03fc2ce2d0ce192b96209f2c5c04169) --- docker/services/pacemaker/cinder-backup.yaml | 6 ++++++ docker/services/pacemaker/cinder-volume.yaml | 6 ++++++ docker/services/pacemaker/database/mysql.yaml | 6 ++++++ docker/services/pacemaker/database/redis.yaml | 6 ++++++ docker/services/pacemaker/haproxy.yaml | 6 ++++++ docker/services/pacemaker/rabbitmq.yaml | 6 ++++++ 6 files changed, 36 insertions(+) diff --git a/docker/services/pacemaker/cinder-backup.yaml b/docker/services/pacemaker/cinder-backup.yaml index c2117c04..cdb8c1bc 100644 --- a/docker/services/pacemaker/cinder-backup.yaml +++ b/docker/services/pacemaker/cinder-backup.yaml @@ -188,6 +188,9 @@ outputs: resource: openstack-cinder-backup state: disable wait_for_resource: true + register: output + retries: 5 + until: output.rc == 0 when: is_bootstrap_node - name: Delete the stopped openstack-cinder-backup cluster resource. tags: step2 @@ -195,6 +198,9 @@ outputs: resource: openstack-cinder-backup state: delete wait_for_resource: true + register: output + retries: 5 + until: output.rc == 0 when: is_bootstrap_node - name: Disable cinder_backup service tags: step2 diff --git a/docker/services/pacemaker/cinder-volume.yaml b/docker/services/pacemaker/cinder-volume.yaml index a4f69517..15c5e099 100644 --- a/docker/services/pacemaker/cinder-volume.yaml +++ b/docker/services/pacemaker/cinder-volume.yaml @@ -206,6 +206,9 @@ outputs: resource: openstack-cinder-volume state: disable wait_for_resource: true + register: output + retries: 5 + until: output.rc == 0 when: is_bootstrap_node - name: Delete the stopped openstack-cinder-volume cluster resource. tags: step2 @@ -213,6 +216,9 @@ outputs: resource: openstack-cinder-volume state: delete wait_for_resource: true + register: output + retries: 5 + until: output.rc == 0 when: is_bootstrap_node - name: Disable cinder_volume service from boot tags: step2 diff --git a/docker/services/pacemaker/database/mysql.yaml b/docker/services/pacemaker/database/mysql.yaml index f57f779e..1e1b2baf 100644 --- a/docker/services/pacemaker/database/mysql.yaml +++ b/docker/services/pacemaker/database/mysql.yaml @@ -272,6 +272,9 @@ outputs: resource: galera state: disable wait_for_resource: true + register: output + retries: 5 + until: output.rc == 0 when: is_bootstrap_node - name: Delete the stopped galera cluster resource. tags: step2 @@ -279,6 +282,9 @@ outputs: resource: galera state: delete wait_for_resource: true + register: output + retries: 5 + until: output.rc == 0 when: is_bootstrap_node - name: Disable mysql service tags: step2 diff --git a/docker/services/pacemaker/database/redis.yaml b/docker/services/pacemaker/database/redis.yaml index ac641201..4ebeae85 100644 --- a/docker/services/pacemaker/database/redis.yaml +++ b/docker/services/pacemaker/database/redis.yaml @@ -228,6 +228,9 @@ outputs: resource: {get_attr: [RedisBase, role_data, service_name]} state: disable wait_for_resource: true + register: output + retries: 5 + until: output.rc == 0 when: is_bootstrap_node - name: Delete the stopped redis cluster resource. tags: step2 @@ -235,6 +238,9 @@ outputs: resource: {get_attr: [RedisBase, role_data, service_name]} state: delete wait_for_resource: true + register: output + retries: 5 + until: output.rc == 0 when: is_bootstrap_node - name: Disable redis service tags: step2 diff --git a/docker/services/pacemaker/haproxy.yaml b/docker/services/pacemaker/haproxy.yaml index 3cdc5255..29bd22af 100644 --- a/docker/services/pacemaker/haproxy.yaml +++ b/docker/services/pacemaker/haproxy.yaml @@ -230,6 +230,9 @@ outputs: resource: {get_attr: [HAProxyBase, role_data, service_name]} state: disable wait_for_resource: true + register: output + retries: 5 + until: output.rc == 0 when: is_bootstrap_node - name: Delete the stopped haproxy cluster resource. tags: step2 @@ -237,4 +240,7 @@ outputs: resource: {get_attr: [HAProxyBase, role_data, service_name]} state: delete wait_for_resource: true + register: output + retries: 5 + until: output.rc == 0 when: is_bootstrap_node diff --git a/docker/services/pacemaker/rabbitmq.yaml b/docker/services/pacemaker/rabbitmq.yaml index ba1abaf9..b0993743 100644 --- a/docker/services/pacemaker/rabbitmq.yaml +++ b/docker/services/pacemaker/rabbitmq.yaml @@ -221,6 +221,9 @@ outputs: resource: {get_attr: [RabbitmqBase, role_data, service_name]} state: disable wait_for_resource: true + register: output + retries: 5 + until: output.rc == 0 when: is_bootstrap_node - name: Delete the stopped rabbitmq cluster resource. tags: step2 @@ -228,6 +231,9 @@ outputs: resource: {get_attr: [RabbitmqBase, role_data, service_name]} state: delete wait_for_resource: true + register: output + retries: 5 + until: output.rc == 0 when: is_bootstrap_node - name: Disable rabbitmq service tags: step2 -- cgit 1.2.3-korg