Retry if the pacemaker_resource commands failed

Add a retry when the pacemaker_resource command
wasn't apply correctly, more info here:
https://bugzilla.redhat.com/show_bug.cgi?id=1482116

This is the same approach puppet-pacemaker uses
and provides eventual consistency when multiple
nodes change the cluster CIB concurrently.

This change depends-on :
https://review.gerrithub.io/375982

The return code is not available in the current
ansible-pacemaker package.

Change-Id: I8da03f5c4a6d442617b81be5793a9724cc8842bf
(cherry picked from commit e92430d8d03fc2ce2d0ce192b96209f2c5c04169)

6 files changed, 36 insertions, 0 deletions

diff --git a/docker/services/pacemaker/cinder-backup.yaml b/docker/services/pacemaker/cinder-backup.yaml
index c2117c04..cdb8c1bc 100644
--- a/docker/services/pacemaker/cinder-backup.yaml
+++ b/docker/services/pacemaker/cinder-backup.yaml
@@ -188,6 +188,9 @@ outputs:
             resource: openstack-cinder-backup
             state: disable
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
         - name: Delete the stopped openstack-cinder-backup cluster resource.
           tags: step2
@@ -195,6 +198,9 @@ outputs:
             resource: openstack-cinder-backup
             state: delete
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
         - name: Disable cinder_backup service
           tags: step2
diff --git a/docker/services/pacemaker/cinder-volume.yaml b/docker/services/pacemaker/cinder-volume.yaml
index a4f69517..15c5e099 100644
--- a/docker/services/pacemaker/cinder-volume.yaml
+++ b/docker/services/pacemaker/cinder-volume.yaml
@@ -206,6 +206,9 @@ outputs:
             resource: openstack-cinder-volume
             state: disable
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
         - name: Delete the stopped openstack-cinder-volume cluster resource.
           tags: step2
@@ -213,6 +216,9 @@ outputs:
             resource: openstack-cinder-volume
             state: delete
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
         - name: Disable cinder_volume service from boot
           tags: step2
diff --git a/docker/services/pacemaker/database/mysql.yaml b/docker/services/pacemaker/database/mysql.yaml
index f57f779e..1e1b2baf 100644
--- a/docker/services/pacemaker/database/mysql.yaml
+++ b/docker/services/pacemaker/database/mysql.yaml
@@ -272,6 +272,9 @@ outputs:
             resource: galera
             state: disable
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
         - name: Delete the stopped galera cluster resource.
           tags: step2
@@ -279,6 +282,9 @@ outputs:
             resource: galera
             state: delete
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
         - name: Disable mysql service
           tags: step2
diff --git a/docker/services/pacemaker/database/redis.yaml b/docker/services/pacemaker/database/redis.yaml
index ac641201..4ebeae85 100644
--- a/docker/services/pacemaker/database/redis.yaml
+++ b/docker/services/pacemaker/database/redis.yaml
@@ -228,6 +228,9 @@ outputs:
             resource: {get_attr: [RedisBase, role_data, service_name]}
             state: disable
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
         - name: Delete the stopped redis cluster resource.
           tags: step2
@@ -235,6 +238,9 @@ outputs:
             resource: {get_attr: [RedisBase, role_data, service_name]}
             state: delete
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
         - name: Disable redis service
           tags: step2
diff --git a/docker/services/pacemaker/haproxy.yaml b/docker/services/pacemaker/haproxy.yaml
index 3cdc5255..29bd22af 100644
--- a/docker/services/pacemaker/haproxy.yaml
+++ b/docker/services/pacemaker/haproxy.yaml
@@ -230,6 +230,9 @@ outputs:
             resource: {get_attr: [HAProxyBase, role_data, service_name]}
             state: disable
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
         - name: Delete the stopped haproxy cluster resource.
           tags: step2
@@ -237,4 +240,7 @@ outputs:
             resource: {get_attr: [HAProxyBase, role_data, service_name]}
             state: delete
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
diff --git a/docker/services/pacemaker/rabbitmq.yaml b/docker/services/pacemaker/rabbitmq.yaml
index ba1abaf9..b0993743 100644
--- a/docker/services/pacemaker/rabbitmq.yaml
+++ b/docker/services/pacemaker/rabbitmq.yaml
@@ -221,6 +221,9 @@ outputs:
             resource: {get_attr: [RabbitmqBase, role_data, service_name]}
             state: disable
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
         - name: Delete the stopped rabbitmq cluster resource.
           tags: step2
@@ -228,6 +231,9 @@ outputs:
             resource: {get_attr: [RabbitmqBase, role_data, service_name]}
             state: delete
             wait_for_resource: true
+          register: output
+          retries: 5
+          until: output.rc == 0
           when: is_bootstrap_node
         - name: Disable rabbitmq service
           tags: step2