#!/bin/bash
##############################################################################
# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################

function recover_cluster() {
    recover_nets
    recover_compass

    i=0
    MAX_RETRY_TIMES=2
    while [ $i -lt $MAX_RETRY_TIMES ]; do
        let i+=1

        if [[ ! -z $VIRT_NUMBER ]];then
            recover_host_vms
        else
            reboot_hosts
        fi

        ret=$(check_hosts_reachable 500)
        if [[ "$ret" == "ok" ]];then
            break
        fi
    done

    if [[ $i -ge $MAX_RETRY_TIMES ]]; then
        echo "Recovery Failure !!!"
        exit 1
    fi

    wait_controller_nodes_ok
    recover_hosts
}