improve cleanup() in run.sh

- make sure all hosts are unset force_down flag

- load all functions for inspector and installer first in order
  to avoid 'command not found' error

- also, print VM if it has error state

Change-Id: If56cba201e1a5f3669a97ffb3d28b3c55c7cdb58
Signed-off-by: Ryota MIBU <r-mibu@cq.jp.nec.com>

4 files changed, 48 insertions, 15 deletions

diff --git a/tests/functions-common b/tests/functions-common
index db2565a3..08d489e5 100644
--- a/tests/functions-common
+++ b/tests/functions-common
@@ -70,3 +70,20 @@ function die_if_not_set {
     $xtrace
 }
 
+# Wait until the condition is met.
+# wait_until condition timeout interval
+function wait_until {
+    local condition="$1"
+    local timeout=$2
+    local interval=$3
+
+    while eval ${condition}
+    do
+        sleep ${interval}
+        timeout=$((${timeout} - ${interval}))
+        if [[ ${timeout} < 0 ]]; then
+            err $LINENO "timed out ($condition)..."
+            return 1
+        fi
+    done
+}
diff --git a/tests/lib/inspector b/tests/lib/inspector
index 2fb7c409..963e9519 100644
--- a/tests/lib/inspector
+++ b/tests/lib/inspector
@@ -13,11 +13,6 @@ function is_inspector {
 }
 
 function start_inspector {
-    if ! is_inspector_supported $INSPECTOR_TYPE; then
-        die $LINENO"INSPECTOR_TYPE=$INSPECTOR_TYPE is not supported."
-    fi
-
-    source $TOP_DIR/lib/inspectors/$INSPECTOR_TYPE
     start_inspector_$INSPECTOR_TYPE
 }
 
@@ -28,3 +23,10 @@ function stop_inspector {
 function cleanup_inspector {
     cleanup_inspector_$INSPECTOR_TYPE
 }
+
+
+if ! is_inspector_supported $INSPECTOR_TYPE; then
+    die $LINENO"INSPECTOR_TYPE=$INSPECTOR_TYPE is not supported."
+fi
+
+source $TOP_DIR/lib/inspectors/$INSPECTOR_TYPE
diff --git a/tests/lib/installer b/tests/lib/installer
index cdde6eff..807023f3 100644
--- a/tests/lib/installer
+++ b/tests/lib/installer
@@ -15,12 +15,6 @@ function is_installer {
 }
 
 function setup_installer {
-    if ! is_installer_supported $INSTALLER_TYPE; then
-        die $LINENO"INSTALLER_TYPE=$INSTALLER_TYPE is not supported."
-    fi
-
-    source $TOP_DIR/lib/installers/$INSTALLER_TYPE
-
     if ! is_set INSTALLER_IP; then
         get_installer_ip
     fi
@@ -32,3 +26,10 @@ function setup_installer {
 function cleanup_installer {
     cleanup_installer_$INSTALLER_TYPE
 }
+
+
+if ! is_installer_supported $INSTALLER_TYPE; then
+    die $LINENO"INSTALLER_TYPE=$INSTALLER_TYPE is not supported."
+fi
+
+source $TOP_DIR/lib/installers/$INSTALLER_TYPE
diff --git a/tests/run.sh b/tests/run.sh
index df6db979..e7afb518 100755
--- a/tests/run.sh
+++ b/tests/run.sh
@@ -222,6 +222,7 @@ wait_for_vm_launch() {
             return 0
         fi
         if [[ "$state" == "ERROR" ]]; then
+            openstack $as_doctor_user server show $VM_NAME
             die $LINENO "vm state is ERROR"
         fi
         count=$(($count+1))
@@ -304,6 +305,18 @@ check_host_status() {
     fi
 }
 
+unset_forced_down_hosts() {
+    for host in $(openstack compute service list --service nova-compute \
+                  -f value -c Host -c State | sed -n -e '/down$/s/ *down$//p')
+    do
+        python ./nova_force_down.py $host --unset
+    done
+
+    echo "waiting disabled compute host back to be enabled..."
+    wait_until 'openstack compute service list --service nova-compute
+                -f value -c State | grep -q down' 240 5
+}
+
 cleanup() {
     set +e
     echo "cleanup..."
@@ -311,10 +324,10 @@ cleanup() {
     stop_inspector
     stop_consumer
 
-    echo "waiting disabled compute host back to be enabled..."
-    python ./nova_force_down.py "$COMPUTE_HOST" --unset
-    sleep 240
-    check_host_status "UP"
+    unset_forced_down_hosts
+    # TODO: We need to make sure the target compute host is back to IP
+    #       reachable. wait_ping() will be added by tojuvone .
+    sleep 110
     scp $ssh_opts_cpu "$COMPUTE_USER@$COMPUTE_IP:disable_network.log" .
 
     openstack $as_doctor_user server list | grep -q " $VM_NAME " && openstack $as_doctor_user server delete "$VM_NAME"