Add recovery scripts

- add recovery shell scripts
- add boot-recovery role to stop services during system boot which
  could cause boot pending
- configure nfs mount in /etc/fstab cause system boot pending,
  so mount nfs during ansible-playbook run.
- kill mysqld before mysql recovery, running mysqld may cause mysql
  recover failure

JIRA: COMPASS-474

Change-Id: I0f6f0ee935fbe3fbbe28a451a02decfb01a6165b
Signed-off-by: QiLiang <liangqi1@huawei.com>

16 files changed, 135 insertions, 3 deletions

diff --git a/deploy/adapters/ansible/openstack/HA-ansible-multinodes.yml b/deploy/adapters/ansible/openstack/HA-ansible-multinodes.yml
index 7f61a1cf..95102d2b 100644
--- a/deploy/adapters/ansible/openstack/HA-ansible-multinodes.yml
+++ b/deploy/adapters/ansible/openstack/HA-ansible-multinodes.yml
@@ -238,6 +238,13 @@
   accelerate: true
   max_fail_percentage: 0
   roles:
+    - boot-recovery
+
+- hosts: controller
+  remote_user: root
+  accelerate: true
+  max_fail_percentage: 0
+  roles:
     - controller-recovery
 
 - hosts: compute
diff --git a/deploy/adapters/ansible/openstack_mitaka/HA-ansible-multinodes.yml b/deploy/adapters/ansible/openstack_mitaka/HA-ansible-multinodes.yml
index 7ef467ee..c04445d8 100644
--- a/deploy/adapters/ansible/openstack_mitaka/HA-ansible-multinodes.yml
+++ b/deploy/adapters/ansible/openstack_mitaka/HA-ansible-multinodes.yml
@@ -247,6 +247,13 @@
   accelerate: true
   max_fail_percentage: 0
   roles:
+    - boot-recovery
+
+- hosts: controller
+  remote_user: root
+  accelerate: true
+  max_fail_percentage: 0
+  roles:
     - controller-recovery
 
 - hosts: compute
diff --git a/deploy/adapters/ansible/openstack_mitaka/roles/glance/tasks/nfs.yml b/deploy/adapters/ansible/openstack_mitaka/roles/glance/tasks/nfs.yml
index 07dfacdd..deec81f8 100644
--- a/deploy/adapters/ansible/openstack_mitaka/roles/glance/tasks/nfs.yml
+++ b/deploy/adapters/ansible/openstack_mitaka/roles/glance/tasks/nfs.yml
@@ -42,10 +42,14 @@
 - name: get mount info
   command: mount
   register: mount_info
+  tags:
+    - recovery
 
 - name: get nfs server
   shell: awk -F'=' '/compass_server/ {print $2}' /etc/compass.conf
   register: ip_info
+  tags:
+    - recovery
 
 - name: restart host nfs service
   service: name={{ item }} state=restarted enabled=yes
@@ -55,7 +59,9 @@
   shell: |
     mount -t nfs  -onfsvers=3 {{ ip_info.stdout_lines[0] }}:/opt/images /var/lib/glance/images
     sed -i '/\/var\/lib\/glance\/images/d' /etc/fstab
-    echo {{ ip_info.stdout_lines[0] }}:/opt/images /var/lib/glance/images/ nfs nfsvers=3 >> /etc/fstab
+    #echo {{ ip_info.stdout_lines[0] }}:/opt/images /var/lib/glance/images/ nfs nfsvers=3 >> /etc/fstab
   when: mount_info.stdout.find('images') == -1
   retries: 5
   delay: 3
+  tags:
+    - recovery
diff --git a/deploy/adapters/ansible/openstack_mitaka_xenial/HA-ansible-multinodes.yml b/deploy/adapters/ansible/openstack_mitaka_xenial/HA-ansible-multinodes.yml
index ec4c53f4..ac31b682 100644
--- a/deploy/adapters/ansible/openstack_mitaka_xenial/HA-ansible-multinodes.yml
+++ b/deploy/adapters/ansible/openstack_mitaka_xenial/HA-ansible-multinodes.yml
@@ -247,6 +247,13 @@
   accelerate: true
   max_fail_percentage: 0
   roles:
+    - boot-recovery
+
+- hosts: controller
+  remote_user: root
+  accelerate: true
+  max_fail_percentage: 0
+  roles:
     - controller-recovery
 
 - hosts: compute
diff --git a/deploy/adapters/ansible/openstack_mitaka_xenial/roles/glance/tasks/nfs.yml b/deploy/adapters/ansible/openstack_mitaka_xenial/roles/glance/tasks/nfs.yml
index 07dfacdd..deec81f8 100644
--- a/deploy/adapters/ansible/openstack_mitaka_xenial/roles/glance/tasks/nfs.yml
+++ b/deploy/adapters/ansible/openstack_mitaka_xenial/roles/glance/tasks/nfs.yml
@@ -42,10 +42,14 @@
 - name: get mount info
   command: mount
   register: mount_info
+  tags:
+    - recovery
 
 - name: get nfs server
   shell: awk -F'=' '/compass_server/ {print $2}' /etc/compass.conf
   register: ip_info
+  tags:
+    - recovery
 
 - name: restart host nfs service
   service: name={{ item }} state=restarted enabled=yes
@@ -55,7 +59,9 @@
   shell: |
     mount -t nfs  -onfsvers=3 {{ ip_info.stdout_lines[0] }}:/opt/images /var/lib/glance/images
     sed -i '/\/var\/lib\/glance\/images/d' /etc/fstab
-    echo {{ ip_info.stdout_lines[0] }}:/opt/images /var/lib/glance/images/ nfs nfsvers=3 >> /etc/fstab
+    #echo {{ ip_info.stdout_lines[0] }}:/opt/images /var/lib/glance/images/ nfs nfsvers=3 >> /etc/fstab
   when: mount_info.stdout.find('images') == -1
   retries: 5
   delay: 3
+  tags:
+    - recovery
diff --git a/deploy/adapters/ansible/openstack_newton_xenial/HA-ansible-multinodes.yml b/deploy/adapters/ansible/openstack_newton_xenial/HA-ansible-multinodes.yml
index 3d5b0a1c..9e8ec15b 100644
--- a/deploy/adapters/ansible/openstack_newton_xenial/HA-ansible-multinodes.yml
+++ b/deploy/adapters/ansible/openstack_newton_xenial/HA-ansible-multinodes.yml
@@ -247,6 +247,13 @@
   accelerate: true
   max_fail_percentage: 0
   roles:
+    - boot-recovery
+
+- hosts: controller
+  remote_user: root
+  accelerate: true
+  max_fail_percentage: 0
+  roles:
     - controller-recovery
 
 - hosts: compute
diff --git a/deploy/adapters/ansible/roles/boot-recovery/tasks/main.yml b/deploy/adapters/ansible/roles/boot-recovery/tasks/main.yml
new file mode 100755
index 00000000..67206bf6
--- /dev/null
+++ b/deploy/adapters/ansible/roles/boot-recovery/tasks/main.yml
@@ -0,0 +1,26 @@
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+- name: Register RECOVERY
+  set_fact: RECOVERY_ENV={{RECOVERY_ENV | default('False')}}
+  tags:
+    - recovery-stop-service
+
+- include_vars: "{{ ansible_os_family }}.yml"
+  when: RECOVERY_ENV
+  tags:
+    - recovery-stop-service
+
+- name: stop controller services
+  service: name={{ item }} state=stopped enabled=yes
+  with_items: controller_services | union(controller_services_noarch)
+  when: RECOVERY_ENV
+  tags:
+    - recovery-stop-service
+
diff --git a/deploy/adapters/ansible/roles/boot-recovery/vars/Debian.yml b/deploy/adapters/ansible/roles/boot-recovery/vars/Debian.yml
new file mode 100755
index 00000000..084deebc
--- /dev/null
+++ b/deploy/adapters/ansible/roles/boot-recovery/vars/Debian.yml
@@ -0,0 +1,14 @@
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+controller_services:
+    - cron
+    - aodh-expirer
+    - neutron-openvswitch-agent
+    - mysql
diff --git a/deploy/adapters/ansible/roles/boot-recovery/vars/RedHat.yml b/deploy/adapters/ansible/roles/boot-recovery/vars/RedHat.yml
new file mode 100755
index 00000000..c46f79c8
--- /dev/null
+++ b/deploy/adapters/ansible/roles/boot-recovery/vars/RedHat.yml
@@ -0,0 +1,15 @@
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+controller_services:
+    - cron
+    - neutron-openvswitch-agent
+    - openstack-aodh-expirer
+    - mysql
+
diff --git a/deploy/adapters/ansible/roles/boot-recovery/vars/main.yml b/deploy/adapters/ansible/roles/boot-recovery/vars/main.yml
new file mode 100755
index 00000000..22af29f4
--- /dev/null
+++ b/deploy/adapters/ansible/roles/boot-recovery/vars/main.yml
@@ -0,0 +1,11 @@
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+controller_services_noarch: []
+
diff --git a/deploy/adapters/ansible/roles/controller-recovery/vars/Debian.yml b/deploy/adapters/ansible/roles/controller-recovery/vars/Debian.yml
index 34675f6b..62753413 100644
--- a/deploy/adapters/ansible/roles/controller-recovery/vars/Debian.yml
+++ b/deploy/adapters/ansible/roles/controller-recovery/vars/Debian.yml
@@ -37,4 +37,5 @@ controller_services:
     - aodh-notifier
     - aodh-evaluator
     - aodh-listener
+    - cron
 
diff --git a/deploy/adapters/ansible/roles/controller-recovery/vars/RedHat.yml b/deploy/adapters/ansible/roles/controller-recovery/vars/RedHat.yml
index 35c0a955..145acecd 100644
--- a/deploy/adapters/ansible/roles/controller-recovery/vars/RedHat.yml
+++ b/deploy/adapters/ansible/roles/controller-recovery/vars/RedHat.yml
@@ -36,4 +36,5 @@ controller_services:
     - openstack-aodh-notifier
     - openstack-aodh-evaluator
     - openstack-aodh-listener
+    - cron
 
diff --git a/deploy/adapters/ansible/roles/database/tasks/mariadb_cluster_debian.yml b/deploy/adapters/ansible/roles/database/tasks/mariadb_cluster_debian.yml
index 6b670312..f083a40f 100644
--- a/deploy/adapters/ansible/roles/database/tasks/mariadb_cluster_debian.yml
+++ b/deploy/adapters/ansible/roles/database/tasks/mariadb_cluster_debian.yml
@@ -7,6 +7,14 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 ---
+- name: Register RECOVERY
+  set_fact: RECOVERY_ENV={{RECOVERY_ENV | default('False')}}
+
+- name: killall mysqld processes
+  shell: sudo killall -9  mysqld
+  when: RECOVERY_ENV
+  ignore_errors: True
+
 - name: get cluster status
   shell: mysql --silent --skip-column-names -e 'SHOW STATUS LIKE "wsrep_evs_state"'|awk '{print $2}'
   register: cluster_status
diff --git a/deploy/adapters/ansible/roles/database/tasks/mariadb_cluster_redhat.yml b/deploy/adapters/ansible/roles/database/tasks/mariadb_cluster_redhat.yml
index da1b863c..cfd778f1 100644
--- a/deploy/adapters/ansible/roles/database/tasks/mariadb_cluster_redhat.yml
+++ b/deploy/adapters/ansible/roles/database/tasks/mariadb_cluster_redhat.yml
@@ -7,6 +7,14 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 ---
+- name: Register RECOVERY
+  set_fact: RECOVERY_ENV={{RECOVERY_ENV | default('False')}}
+
+- name: killall mysqld processes
+  shell: sudo killall -9 mysqld
+  when: RECOVERY_ENV
+  ignore_errors: True
+
 - name: get cluster status
   shell: mysql --silent --skip-column-names -e 'SHOW STATUS LIKE "wsrep_evs_state"'|awk '{print $2}'
   register: cluster_status
diff --git a/deploy/adapters/ansible/roles/glance/tasks/main.yml b/deploy/adapters/ansible/roles/glance/tasks/main.yml
index a78ba771..caece26c 100644
--- a/deploy/adapters/ansible/roles/glance/tasks/main.yml
+++ b/deploy/adapters/ansible/roles/glance/tasks/main.yml
@@ -8,6 +8,8 @@
 ##############################################################################
 ---
 - include_vars: "{{ ansible_os_family }}.yml"
+  tags:
+    - recovery
 
 - include: glance_install.yml
   tags:
diff --git a/deploy/adapters/ansible/roles/glance/tasks/nfs.yml b/deploy/adapters/ansible/roles/glance/tasks/nfs.yml
index 7895c386..179229de 100644
--- a/deploy/adapters/ansible/roles/glance/tasks/nfs.yml
+++ b/deploy/adapters/ansible/roles/glance/tasks/nfs.yml
@@ -38,10 +38,14 @@
 - name: get mount info
   command: mount
   register: mount_info
+  tags:
+    - recovery
 
 - name: get nfs server
   shell: awk -F'=' '/compass_server/ {print $2}' /etc/compass.conf
   register: ip_info
+  tags:
+    - recovery
 
 - name: restart host nfs service
   service: name={{ item }} state=restarted enabled=yes
@@ -51,7 +55,9 @@
   shell: |
     mount -t nfs  -onfsvers=3 {{ ip_info.stdout_lines[0] }}:/opt/images /var/lib/glance/images
     sed -i '/\/var\/lib\/glance\/images/d' /etc/fstab
-    echo {{ ip_info.stdout_lines[0] }}:/opt/images /var/lib/glance/images/ nfs nfsvers=3 >> /etc/fstab
+    #echo {{ ip_info.stdout_lines[0] }}:/opt/images /var/lib/glance/images/ nfs nfsvers=3 >> /etc/fstab
   when: mount_info.stdout.find('images') == -1
   retries: 5
   delay: 3
+  tags:
+    - recovery