From faed52cdedf1b067c3f39d49827cb843d55edd45 Mon Sep 17 00:00:00 2001 From: QiLiang Date: Fri, 14 Oct 2016 06:11:07 +0800 Subject: Add recovery scripts - add recovery shell scripts - add boot-recovery role to stop services during system boot which could cause boot pending - configure nfs mount in /etc/fstab cause system boot pending, so mount nfs during ansible-playbook run. - kill mysqld before mysql recovery, running mysqld may cause mysql recover failure JIRA: COMPASS-474 Change-Id: I0f6f0ee935fbe3fbbe28a451a02decfb01a6165b Signed-off-by: QiLiang --- .../ansible/openstack/HA-ansible-multinodes.yml | 7 ++++++ .../openstack_mitaka/HA-ansible-multinodes.yml | 7 ++++++ .../openstack_mitaka/roles/glance/tasks/nfs.yml | 8 ++++++- .../HA-ansible-multinodes.yml | 7 ++++++ .../roles/glance/tasks/nfs.yml | 8 ++++++- .../HA-ansible-multinodes.yml | 7 ++++++ .../ansible/roles/boot-recovery/tasks/main.yml | 26 ++++++++++++++++++++++ .../ansible/roles/boot-recovery/vars/Debian.yml | 14 ++++++++++++ .../ansible/roles/boot-recovery/vars/RedHat.yml | 15 +++++++++++++ .../ansible/roles/boot-recovery/vars/main.yml | 11 +++++++++ .../roles/controller-recovery/vars/Debian.yml | 1 + .../roles/controller-recovery/vars/RedHat.yml | 1 + .../database/tasks/mariadb_cluster_debian.yml | 8 +++++++ .../database/tasks/mariadb_cluster_redhat.yml | 8 +++++++ .../adapters/ansible/roles/glance/tasks/main.yml | 2 ++ deploy/adapters/ansible/roles/glance/tasks/nfs.yml | 8 ++++++- 16 files changed, 135 insertions(+), 3 deletions(-) create mode 100755 deploy/adapters/ansible/roles/boot-recovery/tasks/main.yml create mode 100755 deploy/adapters/ansible/roles/boot-recovery/vars/Debian.yml create mode 100755 deploy/adapters/ansible/roles/boot-recovery/vars/RedHat.yml create mode 100755 deploy/adapters/ansible/roles/boot-recovery/vars/main.yml (limited to 'deploy/adapters') diff --git a/deploy/adapters/ansible/openstack/HA-ansible-multinodes.yml b/deploy/adapters/ansible/openstack/HA-ansible-multinodes.yml index 7f61a1cf..95102d2b 100644 --- a/deploy/adapters/ansible/openstack/HA-ansible-multinodes.yml +++ b/deploy/adapters/ansible/openstack/HA-ansible-multinodes.yml @@ -233,6 +233,13 @@ roles: - ext-network +- hosts: controller + remote_user: root + accelerate: true + max_fail_percentage: 0 + roles: + - boot-recovery + - hosts: controller remote_user: root accelerate: true diff --git a/deploy/adapters/ansible/openstack_mitaka/HA-ansible-multinodes.yml b/deploy/adapters/ansible/openstack_mitaka/HA-ansible-multinodes.yml index 7ef467ee..c04445d8 100644 --- a/deploy/adapters/ansible/openstack_mitaka/HA-ansible-multinodes.yml +++ b/deploy/adapters/ansible/openstack_mitaka/HA-ansible-multinodes.yml @@ -242,6 +242,13 @@ roles: - tacker +- hosts: controller + remote_user: root + accelerate: true + max_fail_percentage: 0 + roles: + - boot-recovery + - hosts: controller remote_user: root accelerate: true diff --git a/deploy/adapters/ansible/openstack_mitaka/roles/glance/tasks/nfs.yml b/deploy/adapters/ansible/openstack_mitaka/roles/glance/tasks/nfs.yml index 07dfacdd..deec81f8 100644 --- a/deploy/adapters/ansible/openstack_mitaka/roles/glance/tasks/nfs.yml +++ b/deploy/adapters/ansible/openstack_mitaka/roles/glance/tasks/nfs.yml @@ -42,10 +42,14 @@ - name: get mount info command: mount register: mount_info + tags: + - recovery - name: get nfs server shell: awk -F'=' '/compass_server/ {print $2}' /etc/compass.conf register: ip_info + tags: + - recovery - name: restart host nfs service service: name={{ item }} state=restarted enabled=yes @@ -55,7 +59,9 @@ shell: | mount -t nfs -onfsvers=3 {{ ip_info.stdout_lines[0] }}:/opt/images /var/lib/glance/images sed -i '/\/var\/lib\/glance\/images/d' /etc/fstab - echo {{ ip_info.stdout_lines[0] }}:/opt/images /var/lib/glance/images/ nfs nfsvers=3 >> /etc/fstab + #echo {{ ip_info.stdout_lines[0] }}:/opt/images /var/lib/glance/images/ nfs nfsvers=3 >> /etc/fstab when: mount_info.stdout.find('images') == -1 retries: 5 delay: 3 + tags: + - recovery diff --git a/deploy/adapters/ansible/openstack_mitaka_xenial/HA-ansible-multinodes.yml b/deploy/adapters/ansible/openstack_mitaka_xenial/HA-ansible-multinodes.yml index ec4c53f4..ac31b682 100644 --- a/deploy/adapters/ansible/openstack_mitaka_xenial/HA-ansible-multinodes.yml +++ b/deploy/adapters/ansible/openstack_mitaka_xenial/HA-ansible-multinodes.yml @@ -242,6 +242,13 @@ roles: - ext-network +- hosts: controller + remote_user: root + accelerate: true + max_fail_percentage: 0 + roles: + - boot-recovery + - hosts: controller remote_user: root accelerate: true diff --git a/deploy/adapters/ansible/openstack_mitaka_xenial/roles/glance/tasks/nfs.yml b/deploy/adapters/ansible/openstack_mitaka_xenial/roles/glance/tasks/nfs.yml index 07dfacdd..deec81f8 100644 --- a/deploy/adapters/ansible/openstack_mitaka_xenial/roles/glance/tasks/nfs.yml +++ b/deploy/adapters/ansible/openstack_mitaka_xenial/roles/glance/tasks/nfs.yml @@ -42,10 +42,14 @@ - name: get mount info command: mount register: mount_info + tags: + - recovery - name: get nfs server shell: awk -F'=' '/compass_server/ {print $2}' /etc/compass.conf register: ip_info + tags: + - recovery - name: restart host nfs service service: name={{ item }} state=restarted enabled=yes @@ -55,7 +59,9 @@ shell: | mount -t nfs -onfsvers=3 {{ ip_info.stdout_lines[0] }}:/opt/images /var/lib/glance/images sed -i '/\/var\/lib\/glance\/images/d' /etc/fstab - echo {{ ip_info.stdout_lines[0] }}:/opt/images /var/lib/glance/images/ nfs nfsvers=3 >> /etc/fstab + #echo {{ ip_info.stdout_lines[0] }}:/opt/images /var/lib/glance/images/ nfs nfsvers=3 >> /etc/fstab when: mount_info.stdout.find('images') == -1 retries: 5 delay: 3 + tags: + - recovery diff --git a/deploy/adapters/ansible/openstack_newton_xenial/HA-ansible-multinodes.yml b/deploy/adapters/ansible/openstack_newton_xenial/HA-ansible-multinodes.yml index 3d5b0a1c..9e8ec15b 100644 --- a/deploy/adapters/ansible/openstack_newton_xenial/HA-ansible-multinodes.yml +++ b/deploy/adapters/ansible/openstack_newton_xenial/HA-ansible-multinodes.yml @@ -242,6 +242,13 @@ roles: - ext-network +- hosts: controller + remote_user: root + accelerate: true + max_fail_percentage: 0 + roles: + - boot-recovery + - hosts: controller remote_user: root accelerate: true diff --git a/deploy/adapters/ansible/roles/boot-recovery/tasks/main.yml b/deploy/adapters/ansible/roles/boot-recovery/tasks/main.yml new file mode 100755 index 00000000..67206bf6 --- /dev/null +++ b/deploy/adapters/ansible/roles/boot-recovery/tasks/main.yml @@ -0,0 +1,26 @@ +############################################################################## +# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- +- name: Register RECOVERY + set_fact: RECOVERY_ENV={{RECOVERY_ENV | default('False')}} + tags: + - recovery-stop-service + +- include_vars: "{{ ansible_os_family }}.yml" + when: RECOVERY_ENV + tags: + - recovery-stop-service + +- name: stop controller services + service: name={{ item }} state=stopped enabled=yes + with_items: controller_services | union(controller_services_noarch) + when: RECOVERY_ENV + tags: + - recovery-stop-service + diff --git a/deploy/adapters/ansible/roles/boot-recovery/vars/Debian.yml b/deploy/adapters/ansible/roles/boot-recovery/vars/Debian.yml new file mode 100755 index 00000000..084deebc --- /dev/null +++ b/deploy/adapters/ansible/roles/boot-recovery/vars/Debian.yml @@ -0,0 +1,14 @@ +############################################################################## +# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- +controller_services: + - cron + - aodh-expirer + - neutron-openvswitch-agent + - mysql diff --git a/deploy/adapters/ansible/roles/boot-recovery/vars/RedHat.yml b/deploy/adapters/ansible/roles/boot-recovery/vars/RedHat.yml new file mode 100755 index 00000000..c46f79c8 --- /dev/null +++ b/deploy/adapters/ansible/roles/boot-recovery/vars/RedHat.yml @@ -0,0 +1,15 @@ +############################################################################## +# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- +controller_services: + - cron + - neutron-openvswitch-agent + - openstack-aodh-expirer + - mysql + diff --git a/deploy/adapters/ansible/roles/boot-recovery/vars/main.yml b/deploy/adapters/ansible/roles/boot-recovery/vars/main.yml new file mode 100755 index 00000000..22af29f4 --- /dev/null +++ b/deploy/adapters/ansible/roles/boot-recovery/vars/main.yml @@ -0,0 +1,11 @@ +############################################################################## +# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- +controller_services_noarch: [] + diff --git a/deploy/adapters/ansible/roles/controller-recovery/vars/Debian.yml b/deploy/adapters/ansible/roles/controller-recovery/vars/Debian.yml index 34675f6b..62753413 100644 --- a/deploy/adapters/ansible/roles/controller-recovery/vars/Debian.yml +++ b/deploy/adapters/ansible/roles/controller-recovery/vars/Debian.yml @@ -37,4 +37,5 @@ controller_services: - aodh-notifier - aodh-evaluator - aodh-listener + - cron diff --git a/deploy/adapters/ansible/roles/controller-recovery/vars/RedHat.yml b/deploy/adapters/ansible/roles/controller-recovery/vars/RedHat.yml index 35c0a955..145acecd 100644 --- a/deploy/adapters/ansible/roles/controller-recovery/vars/RedHat.yml +++ b/deploy/adapters/ansible/roles/controller-recovery/vars/RedHat.yml @@ -36,4 +36,5 @@ controller_services: - openstack-aodh-notifier - openstack-aodh-evaluator - openstack-aodh-listener + - cron diff --git a/deploy/adapters/ansible/roles/database/tasks/mariadb_cluster_debian.yml b/deploy/adapters/ansible/roles/database/tasks/mariadb_cluster_debian.yml index 6b670312..f083a40f 100644 --- a/deploy/adapters/ansible/roles/database/tasks/mariadb_cluster_debian.yml +++ b/deploy/adapters/ansible/roles/database/tasks/mariadb_cluster_debian.yml @@ -7,6 +7,14 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## --- +- name: Register RECOVERY + set_fact: RECOVERY_ENV={{RECOVERY_ENV | default('False')}} + +- name: killall mysqld processes + shell: sudo killall -9 mysqld + when: RECOVERY_ENV + ignore_errors: True + - name: get cluster status shell: mysql --silent --skip-column-names -e 'SHOW STATUS LIKE "wsrep_evs_state"'|awk '{print $2}' register: cluster_status diff --git a/deploy/adapters/ansible/roles/database/tasks/mariadb_cluster_redhat.yml b/deploy/adapters/ansible/roles/database/tasks/mariadb_cluster_redhat.yml index da1b863c..cfd778f1 100644 --- a/deploy/adapters/ansible/roles/database/tasks/mariadb_cluster_redhat.yml +++ b/deploy/adapters/ansible/roles/database/tasks/mariadb_cluster_redhat.yml @@ -7,6 +7,14 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## --- +- name: Register RECOVERY + set_fact: RECOVERY_ENV={{RECOVERY_ENV | default('False')}} + +- name: killall mysqld processes + shell: sudo killall -9 mysqld + when: RECOVERY_ENV + ignore_errors: True + - name: get cluster status shell: mysql --silent --skip-column-names -e 'SHOW STATUS LIKE "wsrep_evs_state"'|awk '{print $2}' register: cluster_status diff --git a/deploy/adapters/ansible/roles/glance/tasks/main.yml b/deploy/adapters/ansible/roles/glance/tasks/main.yml index a78ba771..caece26c 100644 --- a/deploy/adapters/ansible/roles/glance/tasks/main.yml +++ b/deploy/adapters/ansible/roles/glance/tasks/main.yml @@ -8,6 +8,8 @@ ############################################################################## --- - include_vars: "{{ ansible_os_family }}.yml" + tags: + - recovery - include: glance_install.yml tags: diff --git a/deploy/adapters/ansible/roles/glance/tasks/nfs.yml b/deploy/adapters/ansible/roles/glance/tasks/nfs.yml index 7895c386..179229de 100644 --- a/deploy/adapters/ansible/roles/glance/tasks/nfs.yml +++ b/deploy/adapters/ansible/roles/glance/tasks/nfs.yml @@ -38,10 +38,14 @@ - name: get mount info command: mount register: mount_info + tags: + - recovery - name: get nfs server shell: awk -F'=' '/compass_server/ {print $2}' /etc/compass.conf register: ip_info + tags: + - recovery - name: restart host nfs service service: name={{ item }} state=restarted enabled=yes @@ -51,7 +55,9 @@ shell: | mount -t nfs -onfsvers=3 {{ ip_info.stdout_lines[0] }}:/opt/images /var/lib/glance/images sed -i '/\/var\/lib\/glance\/images/d' /etc/fstab - echo {{ ip_info.stdout_lines[0] }}:/opt/images /var/lib/glance/images/ nfs nfsvers=3 >> /etc/fstab + #echo {{ ip_info.stdout_lines[0] }}:/opt/images /var/lib/glance/images/ nfs nfsvers=3 >> /etc/fstab when: mount_info.stdout.find('images') == -1 retries: 5 delay: 3 + tags: + - recovery -- cgit 1.2.3-korg