summaryrefslogtreecommitdiffstats
path: root/deploy/adapters
diff options
context:
space:
mode:
authorQiLiang <liangqi1@huawei.com>2016-10-14 06:11:07 +0800
committerQiLiang <liangqi1@huawei.com>2016-10-15 05:28:32 +0800
commitfaed52cdedf1b067c3f39d49827cb843d55edd45 (patch)
tree8d34ec4e9e44167b067b30713bf231eff4fdfc58 /deploy/adapters
parentc85cae06ce80e7090b707974a50ab32de4aec28b (diff)
Add recovery scripts
- add recovery shell scripts - add boot-recovery role to stop services during system boot which could cause boot pending - configure nfs mount in /etc/fstab cause system boot pending, so mount nfs during ansible-playbook run. - kill mysqld before mysql recovery, running mysqld may cause mysql recover failure JIRA: COMPASS-474 Change-Id: I0f6f0ee935fbe3fbbe28a451a02decfb01a6165b Signed-off-by: QiLiang <liangqi1@huawei.com>
Diffstat (limited to 'deploy/adapters')
-rw-r--r--deploy/adapters/ansible/openstack/HA-ansible-multinodes.yml7
-rw-r--r--deploy/adapters/ansible/openstack_mitaka/HA-ansible-multinodes.yml7
-rw-r--r--deploy/adapters/ansible/openstack_mitaka/roles/glance/tasks/nfs.yml8
-rw-r--r--deploy/adapters/ansible/openstack_mitaka_xenial/HA-ansible-multinodes.yml7
-rw-r--r--deploy/adapters/ansible/openstack_mitaka_xenial/roles/glance/tasks/nfs.yml8
-rw-r--r--deploy/adapters/ansible/openstack_newton_xenial/HA-ansible-multinodes.yml7
-rwxr-xr-xdeploy/adapters/ansible/roles/boot-recovery/tasks/main.yml26
-rwxr-xr-xdeploy/adapters/ansible/roles/boot-recovery/vars/Debian.yml14
-rwxr-xr-xdeploy/adapters/ansible/roles/boot-recovery/vars/RedHat.yml15
-rwxr-xr-xdeploy/adapters/ansible/roles/boot-recovery/vars/main.yml11
-rw-r--r--deploy/adapters/ansible/roles/controller-recovery/vars/Debian.yml1
-rw-r--r--deploy/adapters/ansible/roles/controller-recovery/vars/RedHat.yml1
-rw-r--r--deploy/adapters/ansible/roles/database/tasks/mariadb_cluster_debian.yml8
-rw-r--r--deploy/adapters/ansible/roles/database/tasks/mariadb_cluster_redhat.yml8
-rw-r--r--deploy/adapters/ansible/roles/glance/tasks/main.yml2
-rw-r--r--deploy/adapters/ansible/roles/glance/tasks/nfs.yml8
16 files changed, 135 insertions, 3 deletions
diff --git a/deploy/adapters/ansible/openstack/HA-ansible-multinodes.yml b/deploy/adapters/ansible/openstack/HA-ansible-multinodes.yml
index 7f61a1cf..95102d2b 100644
--- a/deploy/adapters/ansible/openstack/HA-ansible-multinodes.yml
+++ b/deploy/adapters/ansible/openstack/HA-ansible-multinodes.yml
@@ -238,6 +238,13 @@
accelerate: true
max_fail_percentage: 0
roles:
+ - boot-recovery
+
+- hosts: controller
+ remote_user: root
+ accelerate: true
+ max_fail_percentage: 0
+ roles:
- controller-recovery
- hosts: compute
diff --git a/deploy/adapters/ansible/openstack_mitaka/HA-ansible-multinodes.yml b/deploy/adapters/ansible/openstack_mitaka/HA-ansible-multinodes.yml
index 7ef467ee..c04445d8 100644
--- a/deploy/adapters/ansible/openstack_mitaka/HA-ansible-multinodes.yml
+++ b/deploy/adapters/ansible/openstack_mitaka/HA-ansible-multinodes.yml
@@ -247,6 +247,13 @@
accelerate: true
max_fail_percentage: 0
roles:
+ - boot-recovery
+
+- hosts: controller
+ remote_user: root
+ accelerate: true
+ max_fail_percentage: 0
+ roles:
- controller-recovery
- hosts: compute
diff --git a/deploy/adapters/ansible/openstack_mitaka/roles/glance/tasks/nfs.yml b/deploy/adapters/ansible/openstack_mitaka/roles/glance/tasks/nfs.yml
index 07dfacdd..deec81f8 100644
--- a/deploy/adapters/ansible/openstack_mitaka/roles/glance/tasks/nfs.yml
+++ b/deploy/adapters/ansible/openstack_mitaka/roles/glance/tasks/nfs.yml
@@ -42,10 +42,14 @@
- name: get mount info
command: mount
register: mount_info
+ tags:
+ - recovery
- name: get nfs server
shell: awk -F'=' '/compass_server/ {print $2}' /etc/compass.conf
register: ip_info
+ tags:
+ - recovery
- name: restart host nfs service
service: name={{ item }} state=restarted enabled=yes
@@ -55,7 +59,9 @@
shell: |
mount -t nfs -onfsvers=3 {{ ip_info.stdout_lines[0] }}:/opt/images /var/lib/glance/images
sed -i '/\/var\/lib\/glance\/images/d' /etc/fstab
- echo {{ ip_info.stdout_lines[0] }}:/opt/images /var/lib/glance/images/ nfs nfsvers=3 >> /etc/fstab
+ #echo {{ ip_info.stdout_lines[0] }}:/opt/images /var/lib/glance/images/ nfs nfsvers=3 >> /etc/fstab
when: mount_info.stdout.find('images') == -1
retries: 5
delay: 3
+ tags:
+ - recovery
diff --git a/deploy/adapters/ansible/openstack_mitaka_xenial/HA-ansible-multinodes.yml b/deploy/adapters/ansible/openstack_mitaka_xenial/HA-ansible-multinodes.yml
index ec4c53f4..ac31b682 100644
--- a/deploy/adapters/ansible/openstack_mitaka_xenial/HA-ansible-multinodes.yml
+++ b/deploy/adapters/ansible/openstack_mitaka_xenial/HA-ansible-multinodes.yml
@@ -247,6 +247,13 @@
accelerate: true
max_fail_percentage: 0
roles:
+ - boot-recovery
+
+- hosts: controller
+ remote_user: root
+ accelerate: true
+ max_fail_percentage: 0
+ roles:
- controller-recovery
- hosts: compute
diff --git a/deploy/adapters/ansible/openstack_mitaka_xenial/roles/glance/tasks/nfs.yml b/deploy/adapters/ansible/openstack_mitaka_xenial/roles/glance/tasks/nfs.yml
index 07dfacdd..deec81f8 100644
--- a/deploy/adapters/ansible/openstack_mitaka_xenial/roles/glance/tasks/nfs.yml
+++ b/deploy/adapters/ansible/openstack_mitaka_xenial/roles/glance/tasks/nfs.yml
@@ -42,10 +42,14 @@
- name: get mount info
command: mount
register: mount_info
+ tags:
+ - recovery
- name: get nfs server
shell: awk -F'=' '/compass_server/ {print $2}' /etc/compass.conf
register: ip_info
+ tags:
+ - recovery
- name: restart host nfs service
service: name={{ item }} state=restarted enabled=yes
@@ -55,7 +59,9 @@
shell: |
mount -t nfs -onfsvers=3 {{ ip_info.stdout_lines[0] }}:/opt/images /var/lib/glance/images
sed -i '/\/var\/lib\/glance\/images/d' /etc/fstab
- echo {{ ip_info.stdout_lines[0] }}:/opt/images /var/lib/glance/images/ nfs nfsvers=3 >> /etc/fstab
+ #echo {{ ip_info.stdout_lines[0] }}:/opt/images /var/lib/glance/images/ nfs nfsvers=3 >> /etc/fstab
when: mount_info.stdout.find('images') == -1
retries: 5
delay: 3
+ tags:
+ - recovery
diff --git a/deploy/adapters/ansible/openstack_newton_xenial/HA-ansible-multinodes.yml b/deploy/adapters/ansible/openstack_newton_xenial/HA-ansible-multinodes.yml
index 3d5b0a1c..9e8ec15b 100644
--- a/deploy/adapters/ansible/openstack_newton_xenial/HA-ansible-multinodes.yml
+++ b/deploy/adapters/ansible/openstack_newton_xenial/HA-ansible-multinodes.yml
@@ -247,6 +247,13 @@
accelerate: true
max_fail_percentage: 0
roles:
+ - boot-recovery
+
+- hosts: controller
+ remote_user: root
+ accelerate: true
+ max_fail_percentage: 0
+ roles:
- controller-recovery
- hosts: compute
diff --git a/deploy/adapters/ansible/roles/boot-recovery/tasks/main.yml b/deploy/adapters/ansible/roles/boot-recovery/tasks/main.yml
new file mode 100755
index 00000000..67206bf6
--- /dev/null
+++ b/deploy/adapters/ansible/roles/boot-recovery/tasks/main.yml
@@ -0,0 +1,26 @@
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+- name: Register RECOVERY
+ set_fact: RECOVERY_ENV={{RECOVERY_ENV | default('False')}}
+ tags:
+ - recovery-stop-service
+
+- include_vars: "{{ ansible_os_family }}.yml"
+ when: RECOVERY_ENV
+ tags:
+ - recovery-stop-service
+
+- name: stop controller services
+ service: name={{ item }} state=stopped enabled=yes
+ with_items: controller_services | union(controller_services_noarch)
+ when: RECOVERY_ENV
+ tags:
+ - recovery-stop-service
+
diff --git a/deploy/adapters/ansible/roles/boot-recovery/vars/Debian.yml b/deploy/adapters/ansible/roles/boot-recovery/vars/Debian.yml
new file mode 100755
index 00000000..084deebc
--- /dev/null
+++ b/deploy/adapters/ansible/roles/boot-recovery/vars/Debian.yml
@@ -0,0 +1,14 @@
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+controller_services:
+ - cron
+ - aodh-expirer
+ - neutron-openvswitch-agent
+ - mysql
diff --git a/deploy/adapters/ansible/roles/boot-recovery/vars/RedHat.yml b/deploy/adapters/ansible/roles/boot-recovery/vars/RedHat.yml
new file mode 100755
index 00000000..c46f79c8
--- /dev/null
+++ b/deploy/adapters/ansible/roles/boot-recovery/vars/RedHat.yml
@@ -0,0 +1,15 @@
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+controller_services:
+ - cron
+ - neutron-openvswitch-agent
+ - openstack-aodh-expirer
+ - mysql
+
diff --git a/deploy/adapters/ansible/roles/boot-recovery/vars/main.yml b/deploy/adapters/ansible/roles/boot-recovery/vars/main.yml
new file mode 100755
index 00000000..22af29f4
--- /dev/null
+++ b/deploy/adapters/ansible/roles/boot-recovery/vars/main.yml
@@ -0,0 +1,11 @@
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+controller_services_noarch: []
+
diff --git a/deploy/adapters/ansible/roles/controller-recovery/vars/Debian.yml b/deploy/adapters/ansible/roles/controller-recovery/vars/Debian.yml
index 34675f6b..62753413 100644
--- a/deploy/adapters/ansible/roles/controller-recovery/vars/Debian.yml
+++ b/deploy/adapters/ansible/roles/controller-recovery/vars/Debian.yml
@@ -37,4 +37,5 @@ controller_services:
- aodh-notifier
- aodh-evaluator
- aodh-listener
+ - cron
diff --git a/deploy/adapters/ansible/roles/controller-recovery/vars/RedHat.yml b/deploy/adapters/ansible/roles/controller-recovery/vars/RedHat.yml
index 35c0a955..145acecd 100644
--- a/deploy/adapters/ansible/roles/controller-recovery/vars/RedHat.yml
+++ b/deploy/adapters/ansible/roles/controller-recovery/vars/RedHat.yml
@@ -36,4 +36,5 @@ controller_services:
- openstack-aodh-notifier
- openstack-aodh-evaluator
- openstack-aodh-listener
+ - cron
diff --git a/deploy/adapters/ansible/roles/database/tasks/mariadb_cluster_debian.yml b/deploy/adapters/ansible/roles/database/tasks/mariadb_cluster_debian.yml
index 6b670312..f083a40f 100644
--- a/deploy/adapters/ansible/roles/database/tasks/mariadb_cluster_debian.yml
+++ b/deploy/adapters/ansible/roles/database/tasks/mariadb_cluster_debian.yml
@@ -7,6 +7,14 @@
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
---
+- name: Register RECOVERY
+ set_fact: RECOVERY_ENV={{RECOVERY_ENV | default('False')}}
+
+- name: killall mysqld processes
+ shell: sudo killall -9 mysqld
+ when: RECOVERY_ENV
+ ignore_errors: True
+
- name: get cluster status
shell: mysql --silent --skip-column-names -e 'SHOW STATUS LIKE "wsrep_evs_state"'|awk '{print $2}'
register: cluster_status
diff --git a/deploy/adapters/ansible/roles/database/tasks/mariadb_cluster_redhat.yml b/deploy/adapters/ansible/roles/database/tasks/mariadb_cluster_redhat.yml
index da1b863c..cfd778f1 100644
--- a/deploy/adapters/ansible/roles/database/tasks/mariadb_cluster_redhat.yml
+++ b/deploy/adapters/ansible/roles/database/tasks/mariadb_cluster_redhat.yml
@@ -7,6 +7,14 @@
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
---
+- name: Register RECOVERY
+ set_fact: RECOVERY_ENV={{RECOVERY_ENV | default('False')}}
+
+- name: killall mysqld processes
+ shell: sudo killall -9 mysqld
+ when: RECOVERY_ENV
+ ignore_errors: True
+
- name: get cluster status
shell: mysql --silent --skip-column-names -e 'SHOW STATUS LIKE "wsrep_evs_state"'|awk '{print $2}'
register: cluster_status
diff --git a/deploy/adapters/ansible/roles/glance/tasks/main.yml b/deploy/adapters/ansible/roles/glance/tasks/main.yml
index a78ba771..caece26c 100644
--- a/deploy/adapters/ansible/roles/glance/tasks/main.yml
+++ b/deploy/adapters/ansible/roles/glance/tasks/main.yml
@@ -8,6 +8,8 @@
##############################################################################
---
- include_vars: "{{ ansible_os_family }}.yml"
+ tags:
+ - recovery
- include: glance_install.yml
tags:
diff --git a/deploy/adapters/ansible/roles/glance/tasks/nfs.yml b/deploy/adapters/ansible/roles/glance/tasks/nfs.yml
index 7895c386..179229de 100644
--- a/deploy/adapters/ansible/roles/glance/tasks/nfs.yml
+++ b/deploy/adapters/ansible/roles/glance/tasks/nfs.yml
@@ -38,10 +38,14 @@
- name: get mount info
command: mount
register: mount_info
+ tags:
+ - recovery
- name: get nfs server
shell: awk -F'=' '/compass_server/ {print $2}' /etc/compass.conf
register: ip_info
+ tags:
+ - recovery
- name: restart host nfs service
service: name={{ item }} state=restarted enabled=yes
@@ -51,7 +55,9 @@
shell: |
mount -t nfs -onfsvers=3 {{ ip_info.stdout_lines[0] }}:/opt/images /var/lib/glance/images
sed -i '/\/var\/lib\/glance\/images/d' /etc/fstab
- echo {{ ip_info.stdout_lines[0] }}:/opt/images /var/lib/glance/images/ nfs nfsvers=3 >> /etc/fstab
+ #echo {{ ip_info.stdout_lines[0] }}:/opt/images /var/lib/glance/images/ nfs nfsvers=3 >> /etc/fstab
when: mount_info.stdout.find('images') == -1
retries: 5
delay: 3
+ tags:
+ - recovery