Add recovery scripts

- add recovery shell scripts
- add boot-recovery role to stop services during system boot which
  could cause boot pending
- configure nfs mount in /etc/fstab cause system boot pending,
  so mount nfs during ansible-playbook run.
- kill mysqld before mysql recovery, running mysqld may cause mysql
  recover failure

JIRA: COMPASS-474

Change-Id: I0f6f0ee935fbe3fbbe28a451a02decfb01a6165b
Signed-off-by: QiLiang <liangqi1@huawei.com>

1 files changed, 40 insertions, 0 deletions

diff --git a/deploy/recovery.sh b/deploy/recovery.sh
new file mode 100644
index 00000000..db85848f
--- /dev/null
+++ b/deploy/recovery.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+function recover_cluster() {
+    recover_nets
+    recover_compass
+
+    i=0
+    MAX_RETRY_TIMES=2
+    while [ $i -lt $MAX_RETRY_TIMES ]; do
+        let i+=1
+
+        if [[ ! -z $VIRT_NUMBER ]];then
+            recover_host_vms
+        else
+            reboot_hosts
+        fi
+
+        ret=$(check_hosts_reachable 500)
+        if [[ "$ret" == "ok" ]];then
+            break
+        fi
+    done
+
+    if [[ $i -ge $MAX_RETRY_TIMES ]]; then
+        echo "Recovery Failure !!!"
+        exit 1
+    fi
+
+    wait_controller_nodes_ok
+    recover_hosts
+}
+