diff options
author | QiLiang <liangqi1@huawei.com> | 2016-10-14 06:11:07 +0800 |
---|---|---|
committer | QiLiang <liangqi1@huawei.com> | 2016-10-15 05:28:32 +0800 |
commit | faed52cdedf1b067c3f39d49827cb843d55edd45 (patch) | |
tree | 8d34ec4e9e44167b067b30713bf231eff4fdfc58 /deploy/compass_vm.sh | |
parent | c85cae06ce80e7090b707974a50ab32de4aec28b (diff) |
Add recovery scripts
- add recovery shell scripts
- add boot-recovery role to stop services during system boot which
could cause boot pending
- configure nfs mount in /etc/fstab cause system boot pending,
so mount nfs during ansible-playbook run.
- kill mysqld before mysql recovery, running mysqld may cause mysql
recover failure
JIRA: COMPASS-474
Change-Id: I0f6f0ee935fbe3fbbe28a451a02decfb01a6165b
Signed-off-by: QiLiang <liangqi1@huawei.com>
Diffstat (limited to 'deploy/compass_vm.sh')
-rwxr-xr-x | deploy/compass_vm.sh | 72 |
1 files changed, 72 insertions, 0 deletions
diff --git a/deploy/compass_vm.sh b/deploy/compass_vm.sh index dc391acc..7e2ce40b 100755 --- a/deploy/compass_vm.sh +++ b/deploy/compass_vm.sh @@ -151,3 +151,75 @@ function launch_compass() { set +e log_info "launch_compass exit" } + +function recover_compass() { + log_info "recover_compass enter" + + sudo virsh start compass + + if ! wait_ok 500;then + log_error "install os timeout" + exit 1 + fi + + log_info "launch_compass exit" +} + +function _check_hosts_reachable() { + retry=0 + + while true; do + sleep 1 + let retry+=1 + if [[ $retry -ge $1 ]]; then + log_error "hosts boot time out" + echo "fail" + return + fi + + ssh $ssh_args root@$MGMT_IP " + cd /var/ansible/run/$ADAPTER_NAME'-'$CLUSTER_NAME; + ansible -i inventories/inventory.yml $2 -m ping + " > /dev/null + if [ $? == 0 ]; then + break + fi + done + echo "ok" +} + +function check_hosts_reachable() { + ret=$(_check_hosts_reachable $1 compute) + if [[ "$ret" == "fail" ]]; then + echo $ret + return + fi + + ret=$(_check_hosts_reachable 100 controller) + echo $ret +} + +function recover_hosts() { + ssh $ssh_args root@$MGMT_IP " + cd /var/ansible/run/$ADAPTER_NAME'-'$CLUSTER_NAME; + ansible-playbook \ + -i inventories/inventory.yml HA-ansible-multinodes.yml \ + -t recovery \ + -e 'RECOVERY_ENV=True' + " + if [ $? == 0 ]; then + echo "Recovery Complete!" + fi +} + +function wait_controller_nodes_ok() { + sleep 100 + ssh $ssh_args root@$MGMT_IP " + cd /var/ansible/run/$ADAPTER_NAME'-'$CLUSTER_NAME; + ansible-playbook \ + -i inventories/inventory.yml HA-ansible-multinodes.yml \ + -t recovery-stop-service \ + -e 'RECOVERY_ENV=True' + " + sleep 30 +} |