Remove non-containerized pacemaker resources on upgrade

Adds upgrade_tasks to remove the pacemaker resources using the
ansible-pacemaker module.

Resources are disabled and removed in step2 (called only on
bootstrap node) and then the cluster stop is moved to step3

The existing systemd/service call is kept but only to disable
services after they are disabled/deleted from the cluster.

Related-Bug: 1701485
Co-Authored-By: Damien Ciabrini <dciabrin@redhat.com>
Change-Id: Ia597d240ea5834c50a8f6c4fac0b6ed417b8535c

6 files changed, 140 insertions, 10 deletions

diff --git a/docker/services/pacemaker/cinder-backup.yaml b/docker/services/pacemaker/cinder-backup.yaml
index ee8ee124..26ae9bca 100644
--- a/docker/services/pacemaker/cinder-backup.yaml
+++ b/docker/services/pacemaker/cinder-backup.yaml
@@ -139,6 +139,27 @@ outputs:
             - /var/lib/cinder
             - /var/log/containers/cinder
       upgrade_tasks:
-        - name: Stop and disable cinder_backup service
+        - name: get bootstrap nodeid
+          tags: common
+          command: hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid
+          register: bootstrap_node
+        - name: set is_bootstrap_node fact
+          tags: common
+          set_fact: is_bootstrap_node={{bootstrap_node.stdout|lower == ansible_hostname|lower}}
+        - name: Disable the openstack-cinder-backup cluster resource
           tags: step2
-          service: name=openstack-cinder-backup state=stopped enabled=no
+          pacemaker_resource:
+            resource: openstack-cinder-backup
+            state: disable
+            wait_for_resource: true
+          when: is_bootstrap_node
+        - name: Delete the stopped openstack-cinder-backup cluster resource.
+          tags: step2
+          pacemaker_resource:
+            resource: openstack-cinder-backup
+            state: delete
+            wait_for_resource: true
+          when: is_bootstrap_node
+        - name: Disable cinder_backup service
+          tags: step2
+          service: name=openstack-cinder-backup enabled=no
diff --git a/docker/services/pacemaker/cinder-volume.yaml b/docker/services/pacemaker/cinder-volume.yaml
index d016cf83..262e999d 100644
--- a/docker/services/pacemaker/cinder-volume.yaml
+++ b/docker/services/pacemaker/cinder-volume.yaml
@@ -157,6 +157,30 @@ outputs:
             executable: /bin/bash
             creates: /dev/loop2
       upgrade_tasks:
-        - name: Stop and disable cinder_volume service
+        - name: get bootstrap nodeid
+          tags: common
+          command: hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid
+          register: bootstrap_node
+        - name: set is_bootstrap_node fact
+          tags: common
+          set_fact: is_bootstrap_node={{bootstrap_node.stdout|lower == ansible_hostname|lower}}
+        - name: Disable the openstack-cinder-volume cluster resource
           tags: step2
-          service: name=openstack-cinder-volume state=stopped enabled=no
+          pacemaker_resource:
+            resource: openstack-cinder-volume
+            state: disable
+            wait_for_resource: true
+          when: is_bootstrap_node
+        - name: Delete the stopped openstack-cinder-volume cluster resource.
+          tags: step2
+          pacemaker_resource:
+            resource: openstack-cinder-volume
+            state: delete
+            wait_for_resource: true
+          when: is_bootstrap_node
+        - name: Disable cinder_volume service from boot
+          tags: step2
+          service: name=openstack-cinder-volume enabled=no
+
+
+
diff --git a/docker/services/pacemaker/database/mysql.yaml b/docker/services/pacemaker/database/mysql.yaml
index f38cccfc..22c29b29 100644
--- a/docker/services/pacemaker/database/mysql.yaml
+++ b/docker/services/pacemaker/database/mysql.yaml
@@ -164,6 +164,27 @@ outputs:
             path: /var/lib/mysql
             state: directory
       upgrade_tasks:
-        - name: Stop and disable mysql service
+        - name: get bootstrap nodeid
+          tags: common
+          command: hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid
+          register: bootstrap_node
+        - name: set is_bootstrap_node fact
+          tags: common
+          set_fact: is_bootstrap_node={{bootstrap_node.stdout|lower == ansible_hostname|lower}}
+        - name: Disable the galera cluster resource
           tags: step2
-          service: name=mariadb state=stopped enabled=no
+          pacemaker_resource:
+            resource: galera
+            state: disable
+            wait_for_resource: true
+          when: is_bootstrap_node
+        - name: Delete the stopped galera cluster resource.
+          tags: step2
+          pacemaker_resource:
+            resource: galera
+            state: delete
+            wait_for_resource: true
+          when: is_bootstrap_node
+        - name: Disable mysql service
+          tags: step2
+          service: name=mariadb enabled=no
diff --git a/docker/services/pacemaker/database/redis.yaml b/docker/services/pacemaker/database/redis.yaml
index e124b045..df7ae7f4 100644
--- a/docker/services/pacemaker/database/redis.yaml
+++ b/docker/services/pacemaker/database/redis.yaml
@@ -139,6 +139,27 @@ outputs:
             path: /var/lib/redis
             state: directory
       upgrade_tasks:
-        - name: Stop and disable redis service
+        - name: get bootstrap nodeid
+          tags: common
+          command: hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid
+          register: bootstrap_node
+        - name: set is_bootstrap_node fact
+          tags: common
+          set_fact: is_bootstrap_node={{bootstrap_node.stdout|lower == ansible_hostname|lower}}
+        - name: Disable the redis cluster resource
           tags: step2
-          service: name=redis state=stopped enabled=no
+          pacemaker_resource:
+            resource: {get_attr: [RedisBase, role_data, service_name]}
+            state: disable
+            wait_for_resource: true
+          when: is_bootstrap_node
+        - name: Delete the stopped redis cluster resource.
+          tags: step2
+          pacemaker_resource:
+            resource: {get_attr: [RedisBase, role_data, service_name]}
+            state: delete
+            wait_for_resource: true
+          when: is_bootstrap_node
+        - name: Disable redis service
+          tags: step2
+          service: name=redis enabled=no
diff --git a/docker/services/pacemaker/haproxy.yaml b/docker/services/pacemaker/haproxy.yaml
index 86c460fa..24155912 100644
--- a/docker/services/pacemaker/haproxy.yaml
+++ b/docker/services/pacemaker/haproxy.yaml
@@ -137,3 +137,25 @@ outputs:
                   - /dev/shm:/dev/shm:rw
       metadata_settings:
         get_attr: [HAProxyBase, role_data, metadata_settings]
+      upgrade_tasks:
+        - name: get bootstrap nodeid
+          tags: common
+          command: hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid
+          register: bootstrap_node
+        - name: set is_bootstrap_node fact
+          tags: common
+          set_fact: is_bootstrap_node={{bootstrap_node.stdout|lower == ansible_hostname|lower}}
+        - name: Disable the haproxy cluster resource.
+          tags: step2
+          pacemaker_resource:
+            resource: {get_attr: [HAProxyBase, role_data, service_name]}
+            state: disable
+            wait_for_resource: true
+          when: is_bootstrap_node
+        - name: Delete the stopped haproxy cluster resource.
+          tags: step2
+          pacemaker_resource:
+            resource: {get_attr: [HAProxyBase, role_data, service_name]}
+            state: delete
+            wait_for_resource: true
+          when: is_bootstrap_node
diff --git a/docker/services/pacemaker/rabbitmq.yaml b/docker/services/pacemaker/rabbitmq.yaml
index 19af94b2..dc56bcce 100644
--- a/docker/services/pacemaker/rabbitmq.yaml
+++ b/docker/services/pacemaker/rabbitmq.yaml
@@ -157,6 +157,27 @@ outputs:
             echo 'export ERL_EPMD_PORT=4370' >> /etc/rabbitmq/rabbitmq-env.conf
             for pid in $(pgrep epmd); do if [ "$(lsns -o NS -p $pid)" == "$(lsns -o NS -p 1)" ]; then kill $pid; break; fi; done
       upgrade_tasks:
-        - name: Stop and disable rabbitmq service
+        - name: get bootstrap nodeid
+          tags: common
+          command: hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid
+          register: bootstrap_node
+        - name: set is_bootstrap_node fact
+          tags: common
+          set_fact: is_bootstrap_node={{bootstrap_node.stdout|lower == ansible_hostname|lower}}
+        - name: Disable the rabbitmq cluster resource.
           tags: step2
-          service: name=rabbitmq-server state=stopped enabled=no
+          pacemaker_resource:
+            resource: {get_attr: [RabbitmqBase, role_data, service_name]}
+            state: disable
+            wait_for_resource: true
+          when: is_bootstrap_node
+        - name: Delete the stopped rabbitmq cluster resource.
+          tags: step2
+          pacemaker_resource:
+            resource: {get_attr: [RabbitmqBase, role_data, service_name]}
+            state: delete
+            wait_for_resource: true
+          when: is_bootstrap_node
+        - name: Disable rabbitmq service
+          tags: step2
+          service: name=rabbitmq-server enabled=no