Add controller and compute recovery roles

After all hosts system restart, login to the compass-core,
change directory /var/ansible/run/<openstack-cluster> and execute
below command:

    ansible-playbook \
        -i inventories/inventory.yml HA-ansible-multinodes.yml \
        -t recovery \
        -e "RECOVERY_ENV=True"

JIRA: COMPASS-474

Change-Id: I2ed80550c043b14643382609c48d724019cc1cfb
Signed-off-by: QiLiang <liangqi1@huawei.com>
(cherry picked from commit a894d5924182e0466f5466e66b6111076d992500)

13 files changed, 232 insertions, 0 deletions

diff --git a/deploy/adapters/ansible/openstack/HA-ansible-multinodes.yml b/deploy/adapters/ansible/openstack/HA-ansible-multinodes.yml
index 27aef19c..7f61a1cf 100644
--- a/deploy/adapters/ansible/openstack/HA-ansible-multinodes.yml
+++ b/deploy/adapters/ansible/openstack/HA-ansible-multinodes.yml
@@ -232,3 +232,18 @@
   max_fail_percentage: 0
   roles:
     - ext-network
+
+- hosts: controller
+  remote_user: root
+  accelerate: true
+  max_fail_percentage: 0
+  roles:
+    - controller-recovery
+
+- hosts: compute
+  remote_user: root
+  accelerate: true
+  max_fail_percentage: 0
+  roles:
+    - compute-recovery
+
diff --git a/deploy/adapters/ansible/openstack_mitaka/HA-ansible-multinodes.yml b/deploy/adapters/ansible/openstack_mitaka/HA-ansible-multinodes.yml
index 25b469e0..7ef467ee 100644
--- a/deploy/adapters/ansible/openstack_mitaka/HA-ansible-multinodes.yml
+++ b/deploy/adapters/ansible/openstack_mitaka/HA-ansible-multinodes.yml
@@ -241,3 +241,18 @@
   max_fail_percentage: 0
   roles:
     - tacker
+
+- hosts: controller
+  remote_user: root
+  accelerate: true
+  max_fail_percentage: 0
+  roles:
+    - controller-recovery
+
+- hosts: compute
+  remote_user: root
+  accelerate: true
+  max_fail_percentage: 0
+  roles:
+    - compute-recovery
+
diff --git a/deploy/adapters/ansible/openstack_mitaka_xenial/HA-ansible-multinodes.yml b/deploy/adapters/ansible/openstack_mitaka_xenial/HA-ansible-multinodes.yml
index 1300ab64..c6b1c9ab 100644
--- a/deploy/adapters/ansible/openstack_mitaka_xenial/HA-ansible-multinodes.yml
+++ b/deploy/adapters/ansible/openstack_mitaka_xenial/HA-ansible-multinodes.yml
@@ -243,3 +243,17 @@
   roles:
     - ext-network
 
+- hosts: controller
+  remote_user: root
+  accelerate: true
+  max_fail_percentage: 0
+  roles:
+    - controller-recovery
+
+- hosts: compute
+  remote_user: root
+  accelerate: true
+  max_fail_percentage: 0
+  roles:
+    - compute-recovery
+
diff --git a/deploy/adapters/ansible/roles/compute-recovery/tasks/main.yml b/deploy/adapters/ansible/roles/compute-recovery/tasks/main.yml
new file mode 100644
index 00000000..b333b9e0
--- /dev/null
+++ b/deploy/adapters/ansible/roles/compute-recovery/tasks/main.yml
@@ -0,0 +1,26 @@
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+- name: Register RECOVERY
+  set_fact: RECOVERY_ENV={{RECOVERY_ENV | default('False')}}
+  tags:
+    - recovery
+
+- include_vars: "{{ ansible_os_family }}.yml"
+  when: RECOVERY_ENV
+  tags:
+    - recovery
+
+- name: restart compute services
+  service: name={{ item }} state=restarted enabled=yes
+  with_items: compute_services | union(compute_services_noarch)
+  when: RECOVERY_ENV
+  tags:
+    - recovery
+
diff --git a/deploy/adapters/ansible/roles/compute-recovery/vars/Debian.yml b/deploy/adapters/ansible/roles/compute-recovery/vars/Debian.yml
new file mode 100644
index 00000000..23200a50
--- /dev/null
+++ b/deploy/adapters/ansible/roles/compute-recovery/vars/Debian.yml
@@ -0,0 +1,15 @@
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+compute_services:
+    - nova-compute
+    - neutron-openvswitch-agent
+    - cinder-volume
+    - ceilometer-agent-compute
+
diff --git a/deploy/adapters/ansible/roles/compute-recovery/vars/RedHat.yml b/deploy/adapters/ansible/roles/compute-recovery/vars/RedHat.yml
new file mode 100644
index 00000000..c96b062a
--- /dev/null
+++ b/deploy/adapters/ansible/roles/compute-recovery/vars/RedHat.yml
@@ -0,0 +1,15 @@
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+compute_services:
+    - openstack-nova-compute
+    - neutron-openvswitch-agent
+    - openstack-cinder-volume
+    - openstack-ceilometer-compute
+
diff --git a/deploy/adapters/ansible/roles/compute-recovery/vars/main.yml b/deploy/adapters/ansible/roles/compute-recovery/vars/main.yml
new file mode 100644
index 00000000..c2c0d8ee
--- /dev/null
+++ b/deploy/adapters/ansible/roles/compute-recovery/vars/main.yml
@@ -0,0 +1,10 @@
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+compute_services_noarch: []
diff --git a/deploy/adapters/ansible/roles/controller-recovery/tasks/main.yml b/deploy/adapters/ansible/roles/controller-recovery/tasks/main.yml
new file mode 100644
index 00000000..ccf3e46b
--- /dev/null
+++ b/deploy/adapters/ansible/roles/controller-recovery/tasks/main.yml
@@ -0,0 +1,26 @@
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+- name: Register RECOVERY
+  set_fact: RECOVERY_ENV={{RECOVERY_ENV | default('False')}}
+  tags:
+    - recovery
+
+- include_vars: "{{ ansible_os_family }}.yml"
+  when: RECOVERY_ENV
+  tags:
+    - recovery
+
+- name: restart controller services
+  service: name={{ item }} state=restarted enabled=yes
+  with_items: controller_services | union(controller_services_noarch)
+  when: RECOVERY_ENV
+  tags:
+    - recovery
+
diff --git a/deploy/adapters/ansible/roles/controller-recovery/vars/Debian.yml b/deploy/adapters/ansible/roles/controller-recovery/vars/Debian.yml
new file mode 100644
index 00000000..34675f6b
--- /dev/null
+++ b/deploy/adapters/ansible/roles/controller-recovery/vars/Debian.yml
@@ -0,0 +1,40 @@
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+controller_services:
+    - keepalived
+    - apache2
+    - nova-api
+    - nova-cert
+    - nova-conductor
+    - nova-consoleauth
+    - nova-novncproxy
+    - nova-scheduler
+    - neutron-server
+    - cinder-api
+    - cinder-scheduler
+    - glance-registry
+    - glance-api
+    - openvswitch-switch
+    - neutron-openvswitch-agent
+    - neutron-l3-agent
+    - neutron-dhcp-agent
+    - neutron-metadata-agent
+    - ceilometer-agent-central
+    - ceilometer-agent-notification
+    - ceilometer-api
+    - ceilometer-collector
+    - heat-api
+    - heat-api-cfn
+    - heat-engine
+    - aodh-api
+    - aodh-notifier
+    - aodh-evaluator
+    - aodh-listener
+
diff --git a/deploy/adapters/ansible/roles/controller-recovery/vars/RedHat.yml b/deploy/adapters/ansible/roles/controller-recovery/vars/RedHat.yml
new file mode 100644
index 00000000..35c0a955
--- /dev/null
+++ b/deploy/adapters/ansible/roles/controller-recovery/vars/RedHat.yml
@@ -0,0 +1,39 @@
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+controller_services:
+    - keepalived
+    - httpd
+    - openstack-nova-api
+    - openstack-nova-cert
+    - openstack-nova-conductor
+    - openstack-nova-consoleauth
+    - openstack-nova-novncproxy
+    - openstack-nova-scheduler
+    - neutron-server
+    - openstack-cinder-api
+    - openstack-cinder-scheduler
+    - openstack-glance-api
+    - openstack-glance-registry
+    - neutron-openvswitch-agent
+    - neutron-l3-agent
+    - neutron-dhcp-agent
+    - neutron-metadata-agent
+    - openstack-ceilometer-central
+    - openstack-ceilometer-notification
+    - openstack-ceilometer-api
+    - openstack-ceilometer-collector
+    - openstack-heat-api
+    - openstack-heat-api-cfn
+    - openstack-heat-engine
+    - openstack-aodh-api
+    - openstack-aodh-notifier
+    - openstack-aodh-evaluator
+    - openstack-aodh-listener
+
diff --git a/deploy/adapters/ansible/roles/controller-recovery/vars/main.yml b/deploy/adapters/ansible/roles/controller-recovery/vars/main.yml
new file mode 100644
index 00000000..22af29f4
--- /dev/null
+++ b/deploy/adapters/ansible/roles/controller-recovery/vars/main.yml
@@ -0,0 +1,11 @@
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+controller_services_noarch: []
+
diff --git a/deploy/adapters/ansible/roles/database/tasks/main.yml b/deploy/adapters/ansible/roles/database/tasks/main.yml
index 124b2639..f28da23b 100644
--- a/deploy/adapters/ansible/roles/database/tasks/main.yml
+++ b/deploy/adapters/ansible/roles/database/tasks/main.yml
@@ -10,9 +10,13 @@
 - include_vars: "{{ ansible_os_family }}.yml"
   tags:
     - test_mongo
+    - recovery
 
 - include: mariadb_install.yml
+
 - include: mariadb_cluster.yml
+  tags:
+    - recovery
 
 - include: mariadb_config.yml
   when:
diff --git a/deploy/adapters/ansible/roles/setup-network/tasks/main.yml b/deploy/adapters/ansible/roles/setup-network/tasks/main.yml
index 727b24ea..7873c073 100644
--- a/deploy/adapters/ansible/roles/setup-network/tasks/main.yml
+++ b/deploy/adapters/ansible/roles/setup-network/tasks/main.yml
@@ -55,6 +55,8 @@
 
 - name: run scripts
   shell: python /opt/setup_networks/setup_networks.py
+  tags:
+    - recovery
 
 - name: add to boot scripts
   service: name=net_init enabled=yes