From cdc3477b7780c751807fe18c3767762189cf03b4 Mon Sep 17 00:00:00 2001 From: marios Date: Tue, 4 Jul 2017 16:52:26 +0300 Subject: Remove non-containerized pacemaker resources on upgrade Adds upgrade_tasks to remove the pacemaker resources using the ansible-pacemaker module. Resources are disabled and removed in step2 (called only on bootstrap node) and then the cluster stop is moved to step3 The existing systemd/service call is kept but only to disable services after they are disabled/deleted from the cluster. Related-Bug: 1701485 Co-Authored-By: Damien Ciabrini Change-Id: Ia597d240ea5834c50a8f6c4fac0b6ed417b8535c --- docker/services/pacemaker/cinder-backup.yaml | 25 ++++++++++++++++++++++-- docker/services/pacemaker/cinder-volume.yaml | 28 +++++++++++++++++++++++++-- docker/services/pacemaker/database/mysql.yaml | 25 ++++++++++++++++++++++-- docker/services/pacemaker/database/redis.yaml | 25 ++++++++++++++++++++++-- docker/services/pacemaker/haproxy.yaml | 22 +++++++++++++++++++++ docker/services/pacemaker/rabbitmq.yaml | 25 ++++++++++++++++++++++-- environments/docker-ha.yaml | 3 --- puppet/services/pacemaker.yaml | 19 +----------------- 8 files changed, 141 insertions(+), 31 deletions(-) diff --git a/docker/services/pacemaker/cinder-backup.yaml b/docker/services/pacemaker/cinder-backup.yaml index ee8ee124..26ae9bca 100644 --- a/docker/services/pacemaker/cinder-backup.yaml +++ b/docker/services/pacemaker/cinder-backup.yaml @@ -139,6 +139,27 @@ outputs: - /var/lib/cinder - /var/log/containers/cinder upgrade_tasks: - - name: Stop and disable cinder_backup service + - name: get bootstrap nodeid + tags: common + command: hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid + register: bootstrap_node + - name: set is_bootstrap_node fact + tags: common + set_fact: is_bootstrap_node={{bootstrap_node.stdout|lower == ansible_hostname|lower}} + - name: Disable the openstack-cinder-backup cluster resource tags: step2 - service: name=openstack-cinder-backup state=stopped enabled=no + pacemaker_resource: + resource: openstack-cinder-backup + state: disable + wait_for_resource: true + when: is_bootstrap_node + - name: Delete the stopped openstack-cinder-backup cluster resource. + tags: step2 + pacemaker_resource: + resource: openstack-cinder-backup + state: delete + wait_for_resource: true + when: is_bootstrap_node + - name: Disable cinder_backup service + tags: step2 + service: name=openstack-cinder-backup enabled=no diff --git a/docker/services/pacemaker/cinder-volume.yaml b/docker/services/pacemaker/cinder-volume.yaml index d016cf83..262e999d 100644 --- a/docker/services/pacemaker/cinder-volume.yaml +++ b/docker/services/pacemaker/cinder-volume.yaml @@ -157,6 +157,30 @@ outputs: executable: /bin/bash creates: /dev/loop2 upgrade_tasks: - - name: Stop and disable cinder_volume service + - name: get bootstrap nodeid + tags: common + command: hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid + register: bootstrap_node + - name: set is_bootstrap_node fact + tags: common + set_fact: is_bootstrap_node={{bootstrap_node.stdout|lower == ansible_hostname|lower}} + - name: Disable the openstack-cinder-volume cluster resource tags: step2 - service: name=openstack-cinder-volume state=stopped enabled=no + pacemaker_resource: + resource: openstack-cinder-volume + state: disable + wait_for_resource: true + when: is_bootstrap_node + - name: Delete the stopped openstack-cinder-volume cluster resource. + tags: step2 + pacemaker_resource: + resource: openstack-cinder-volume + state: delete + wait_for_resource: true + when: is_bootstrap_node + - name: Disable cinder_volume service from boot + tags: step2 + service: name=openstack-cinder-volume enabled=no + + + diff --git a/docker/services/pacemaker/database/mysql.yaml b/docker/services/pacemaker/database/mysql.yaml index f38cccfc..22c29b29 100644 --- a/docker/services/pacemaker/database/mysql.yaml +++ b/docker/services/pacemaker/database/mysql.yaml @@ -164,6 +164,27 @@ outputs: path: /var/lib/mysql state: directory upgrade_tasks: - - name: Stop and disable mysql service + - name: get bootstrap nodeid + tags: common + command: hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid + register: bootstrap_node + - name: set is_bootstrap_node fact + tags: common + set_fact: is_bootstrap_node={{bootstrap_node.stdout|lower == ansible_hostname|lower}} + - name: Disable the galera cluster resource tags: step2 - service: name=mariadb state=stopped enabled=no + pacemaker_resource: + resource: galera + state: disable + wait_for_resource: true + when: is_bootstrap_node + - name: Delete the stopped galera cluster resource. + tags: step2 + pacemaker_resource: + resource: galera + state: delete + wait_for_resource: true + when: is_bootstrap_node + - name: Disable mysql service + tags: step2 + service: name=mariadb enabled=no diff --git a/docker/services/pacemaker/database/redis.yaml b/docker/services/pacemaker/database/redis.yaml index e124b045..df7ae7f4 100644 --- a/docker/services/pacemaker/database/redis.yaml +++ b/docker/services/pacemaker/database/redis.yaml @@ -139,6 +139,27 @@ outputs: path: /var/lib/redis state: directory upgrade_tasks: - - name: Stop and disable redis service + - name: get bootstrap nodeid + tags: common + command: hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid + register: bootstrap_node + - name: set is_bootstrap_node fact + tags: common + set_fact: is_bootstrap_node={{bootstrap_node.stdout|lower == ansible_hostname|lower}} + - name: Disable the redis cluster resource tags: step2 - service: name=redis state=stopped enabled=no + pacemaker_resource: + resource: {get_attr: [RedisBase, role_data, service_name]} + state: disable + wait_for_resource: true + when: is_bootstrap_node + - name: Delete the stopped redis cluster resource. + tags: step2 + pacemaker_resource: + resource: {get_attr: [RedisBase, role_data, service_name]} + state: delete + wait_for_resource: true + when: is_bootstrap_node + - name: Disable redis service + tags: step2 + service: name=redis enabled=no diff --git a/docker/services/pacemaker/haproxy.yaml b/docker/services/pacemaker/haproxy.yaml index 86c460fa..24155912 100644 --- a/docker/services/pacemaker/haproxy.yaml +++ b/docker/services/pacemaker/haproxy.yaml @@ -137,3 +137,25 @@ outputs: - /dev/shm:/dev/shm:rw metadata_settings: get_attr: [HAProxyBase, role_data, metadata_settings] + upgrade_tasks: + - name: get bootstrap nodeid + tags: common + command: hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid + register: bootstrap_node + - name: set is_bootstrap_node fact + tags: common + set_fact: is_bootstrap_node={{bootstrap_node.stdout|lower == ansible_hostname|lower}} + - name: Disable the haproxy cluster resource. + tags: step2 + pacemaker_resource: + resource: {get_attr: [HAProxyBase, role_data, service_name]} + state: disable + wait_for_resource: true + when: is_bootstrap_node + - name: Delete the stopped haproxy cluster resource. + tags: step2 + pacemaker_resource: + resource: {get_attr: [HAProxyBase, role_data, service_name]} + state: delete + wait_for_resource: true + when: is_bootstrap_node diff --git a/docker/services/pacemaker/rabbitmq.yaml b/docker/services/pacemaker/rabbitmq.yaml index 19af94b2..dc56bcce 100644 --- a/docker/services/pacemaker/rabbitmq.yaml +++ b/docker/services/pacemaker/rabbitmq.yaml @@ -157,6 +157,27 @@ outputs: echo 'export ERL_EPMD_PORT=4370' >> /etc/rabbitmq/rabbitmq-env.conf for pid in $(pgrep epmd); do if [ "$(lsns -o NS -p $pid)" == "$(lsns -o NS -p 1)" ]; then kill $pid; break; fi; done upgrade_tasks: - - name: Stop and disable rabbitmq service + - name: get bootstrap nodeid + tags: common + command: hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid + register: bootstrap_node + - name: set is_bootstrap_node fact + tags: common + set_fact: is_bootstrap_node={{bootstrap_node.stdout|lower == ansible_hostname|lower}} + - name: Disable the rabbitmq cluster resource. tags: step2 - service: name=rabbitmq-server state=stopped enabled=no + pacemaker_resource: + resource: {get_attr: [RabbitmqBase, role_data, service_name]} + state: disable + wait_for_resource: true + when: is_bootstrap_node + - name: Delete the stopped rabbitmq cluster resource. + tags: step2 + pacemaker_resource: + resource: {get_attr: [RabbitmqBase, role_data, service_name]} + state: delete + wait_for_resource: true + when: is_bootstrap_node + - name: Disable rabbitmq service + tags: step2 + service: name=rabbitmq-server enabled=no diff --git a/environments/docker-ha.yaml b/environments/docker-ha.yaml index 442262b3..784a2a3b 100644 --- a/environments/docker-ha.yaml +++ b/environments/docker-ha.yaml @@ -3,9 +3,6 @@ # ...deploy..-e docker.yaml -e docker-ha.yaml resource_registry: # Pacemaker runs on the host - OS::TripleO::Tasks::ControllerPreConfig: ../extraconfig/tasks/pre_puppet_pacemaker.yaml - OS::TripleO::Tasks::ControllerPostConfig: ../extraconfig/tasks/post_puppet_pacemaker.yaml - OS::TripleO::Tasks::ControllerPostPuppetRestart: ../extraconfig/tasks/post_puppet_pacemaker_restart.yaml OS::TripleO::Services::Pacemaker: ../puppet/services/pacemaker.yaml OS::TripleO::Services::PacemakerRemote: ../puppet/services/pacemaker_remote.yaml diff --git a/puppet/services/pacemaker.yaml b/puppet/services/pacemaker.yaml index 893e8418..158d04bd 100644 --- a/puppet/services/pacemaker.yaml +++ b/puppet/services/pacemaker.yaml @@ -105,11 +105,6 @@ parameters: description: Whether to deploy a LoadBalancer on the Controller type: boolean - PacemakerResources: - type: comma_delimited_list - description: List of resources managed by pacemaker - default: ['rabbitmq', 'galera'] - outputs: role_data: description: Role data for the Pacemaker role. @@ -156,20 +151,8 @@ outputs: async: 30 poll: 4 - name: Stop pacemaker cluster - tags: step2 + tags: step3 pacemaker_cluster: state=offline - name: Start pacemaker cluster tags: step4 pacemaker_cluster: state=online - - name: Check pacemaker resource - tags: step4 - pacemaker_is_active: - resource: "{{ item }}" - max_wait: 500 - with_items: {get_param: PacemakerResources} - - name: Check pacemaker haproxy resource - tags: step4 - pacemaker_is_active: - resource: haproxy - max_wait: 500 - when: {get_param: EnableLoadBalancer} -- cgit 1.2.3-korg