One time delete pacemaker resources during upgrade to containers

This change allows running the major upgrade composable docker
steps multiple times by not trying to delete the pacemaker resources
if they're not reported as started or in master state.

Closes-bug: 1716031
Depends-On: I8da03f5c4a6d442617b81be5793a9724cc8842bf
Change-Id: Ifcf9de8c82550a90a9fb118052d43fdbcdc6ca7e
(cherry picked from commit 64d7be1e3d4552e06cbc53f788572e530cc5c3bb)

4 files changed, 40 insertions, 8 deletions

diff --git a/docker/services/pacemaker/database/mysql.yaml b/docker/services/pacemaker/database/mysql.yaml
index 1e1b2baf..4d5a0a15 100644
--- a/docker/services/pacemaker/database/mysql.yaml
+++ b/docker/services/pacemaker/database/mysql.yaml
@@ -266,6 +266,14 @@ outputs:
         - name: set is_bootstrap_node fact
           tags: common
           set_fact: is_bootstrap_node={{bootstrap_node.stdout|lower == ansible_hostname|lower}}
+        - name: Check cluster resource status
+          tags: step2
+          pacemaker_resource:
+            resource: galera
+            state: master
+            check_mode: true
+          ignore_errors: true
+          register: galera_res
         - name: Disable the galera cluster resource
           tags: step2
           pacemaker_resource:
@@ -275,7 +283,7 @@ outputs:
           register: output
           retries: 5
           until: output.rc == 0
-          when: is_bootstrap_node
+          when: is_bootstrap_node and galera_res|succeeded
         - name: Delete the stopped galera cluster resource.
           tags: step2
           pacemaker_resource:
@@ -285,7 +293,7 @@ outputs:
           register: output
           retries: 5
           until: output.rc == 0
-          when: is_bootstrap_node
+          when: is_bootstrap_node and galera_res|succeeded
         - name: Disable mysql service
           tags: step2
           service: name=mariadb enabled=no
diff --git a/docker/services/pacemaker/database/redis.yaml b/docker/services/pacemaker/database/redis.yaml
index 4ebeae85..4d26a084 100644
--- a/docker/services/pacemaker/database/redis.yaml
+++ b/docker/services/pacemaker/database/redis.yaml
@@ -222,6 +222,14 @@ outputs:
         - name: set is_bootstrap_node fact
           tags: common
           set_fact: is_bootstrap_node={{bootstrap_node.stdout|lower == ansible_hostname|lower}}
+        - name: Check cluster resource status
+          tags: step2
+          pacemaker_resource:
+            resource: {get_attr: [RedisBase, role_data, service_name]}
+            state: master
+            check_mode: true
+          ignore_errors: true
+          register: redis_res
         - name: Disable the redis cluster resource
           tags: step2
           pacemaker_resource:
@@ -231,7 +239,7 @@ outputs:
           register: output
           retries: 5
           until: output.rc == 0
-          when: is_bootstrap_node
+          when: is_bootstrap_node and redis_res|succeeded
         - name: Delete the stopped redis cluster resource.
           tags: step2
           pacemaker_resource:
@@ -241,7 +249,7 @@ outputs:
           register: output
           retries: 5
           until: output.rc == 0
-          when: is_bootstrap_node
+          when: is_bootstrap_node and redis_res|succeeded
         - name: Disable redis service
           tags: step2
           service: name=redis enabled=no
diff --git a/docker/services/pacemaker/haproxy.yaml b/docker/services/pacemaker/haproxy.yaml
index 29bd22af..a325f286 100644
--- a/docker/services/pacemaker/haproxy.yaml
+++ b/docker/services/pacemaker/haproxy.yaml
@@ -224,6 +224,14 @@ outputs:
         - name: set is_bootstrap_node fact
           tags: common
           set_fact: is_bootstrap_node={{bootstrap_node.stdout|lower == ansible_hostname|lower}}
+        - name: Check cluster resource status
+          tags: step2
+          pacemaker_resource:
+            resource: {get_attr: [HAProxyBase, role_data, service_name]}
+            state: started
+            check_mode: true
+          ignore_errors: true
+          register: haproxy_res
         - name: Disable the haproxy cluster resource.
           tags: step2
           pacemaker_resource:
@@ -233,7 +241,7 @@ outputs:
           register: output
           retries: 5
           until: output.rc == 0
-          when: is_bootstrap_node
+          when: is_bootstrap_node and haproxy_res|succeeded
         - name: Delete the stopped haproxy cluster resource.
           tags: step2
           pacemaker_resource:
@@ -243,4 +251,4 @@ outputs:
           register: output
           retries: 5
           until: output.rc == 0
-          when: is_bootstrap_node
+          when: is_bootstrap_node and haproxy_res|succeeded
diff --git a/docker/services/pacemaker/rabbitmq.yaml b/docker/services/pacemaker/rabbitmq.yaml
index b0993743..7333689c 100644
--- a/docker/services/pacemaker/rabbitmq.yaml
+++ b/docker/services/pacemaker/rabbitmq.yaml
@@ -215,6 +215,14 @@ outputs:
         - name: set is_bootstrap_node fact
           tags: common
           set_fact: is_bootstrap_node={{bootstrap_node.stdout|lower == ansible_hostname|lower}}
+        - name: Check cluster resource status
+          tags: step2
+          pacemaker_resource:
+            resource: {get_attr: [RabbitmqBase, role_data, service_name]}
+            state: started
+            check_mode: true
+          ignore_errors: true
+          register: rabbitmq_res
         - name: Disable the rabbitmq cluster resource.
           tags: step2
           pacemaker_resource:
@@ -224,7 +232,7 @@ outputs:
           register: output
           retries: 5
           until: output.rc == 0
-          when: is_bootstrap_node
+          when: is_bootstrap_node and rabbitmq_res|succeeded
         - name: Delete the stopped rabbitmq cluster resource.
           tags: step2
           pacemaker_resource:
@@ -234,7 +242,7 @@ outputs:
           register: output
           retries: 5
           until: output.rc == 0
-          when: is_bootstrap_node
+          when: is_bootstrap_node and rabbitmq_res|succeeded
         - name: Disable rabbitmq service
           tags: step2
           service: name=rabbitmq-server enabled=no