Update stack creation status check

- add more debug info
- authorized_keys modify for root access
- extend timeout time for ssh login

JIRA: BOTTLENECK-33

Change-Id: Ib2679cad7c7a9a73442d05795ac89f9a47d56176
Signed-off-by: QiLiang <liangqi1@huawei.com>

4 files changed, 108 insertions, 16 deletions

diff --git a/utils/infra_setup/heat_template/HOT_create_instance.sh b/utils/infra_setup/heat_template/HOT_create_instance.sh
index bcd9f048..753dd4d7 100755
--- a/utils/infra_setup/heat_template/HOT_create_instance.sh
+++ b/utils/infra_setup/heat_template/HOT_create_instance.sh
@@ -14,7 +14,9 @@ bottlenecks_env_prepare()
     git config --global http.sslVerify false
     git clone ${BOTTLENECKS_REPO} ${BOTTLENECKS_REPO_DIR}
     if [ x"$GERRIT_REFSPEC_DEBUG" != x ]; then
+        cd ${BOTTLENECKS_REPO_DIR}
         git fetch $BOTTLENECKS_REPO $GERRIT_REFSPEC_DEBUG && git checkout FETCH_HEAD
+        cd -
     fi
 
     source $BOTTLENECKS_REPO_DIR/rubbos/rubbos_scripts/1-1-1/scripts/env_preparation.sh
@@ -29,13 +31,26 @@ wait_heat_stack_complete() {
         if [ x$status = x"CREATE_COMPLETE" ]; then
             echo "bottlenecks stacke create complete"
             heat stack-show bottlenecks
+            nova list | grep rubbos_
             break;
+        elif [ x$status = x"CREATE_FAILED" ]; then
+            echo "bottlenecks stacke create failed !!!"
+            heat stack-show bottlenecks
+            exit 1
+        fi
+
+        if [ $BOTTLENECKS_DEBUG = True ]; then
+            heat stack-show bottlenecks
+            nova list | grep rubbos_
+            for i in $(nova list | grep rubbos_ | grep ERROR | awk '{print $2}')
+            do
+                 nova show $i
+            done
         fi
-        echo "bottlenecks stack status $status"
         sleep 1
         let retry+=1
         if [[ $retry -ge $1 ]];then
-            echo "Heat stack create timeout!!!"
+            echo "Heat stack create timeout, status $status !!!"
             exit 1
         fi
     done
@@ -45,7 +60,7 @@ wait_rubbos_control_ok() {
     control_ip=$(nova list | grep rubbos_control | awk '{print $13}')
 
     retry=0
-    until timeout 1s ssh $ssh_args ec2-user@$control_ip "exit" >/dev/null 2>&1
+    until timeout 3s ssh $ssh_args ec2-user@$control_ip "exit" >/dev/null 2>&1
     do
         echo "retry connect rubbos control $retry"
         sleep 1
@@ -65,6 +80,27 @@ bottlenecks_check_instance_ok()
     wait_heat_stack_complete 120
     wait_rubbos_control_ok 300
     nova list | grep rubbos_
+    if [ $BOTTLENECKS_DEBUG = True ]; then
+        date
+        while true
+        do
+            for i in rubbos_benchmark rubbos_client1 rubbos_client2 rubbos_client3 \
+                     rubbos_client4 rubbos_control rubbos_httpd rubbos_mysql1 rubbos_tomcat1
+            do
+               echo "logging $i"
+               nova console-log $i | tail -n 2 | grep Cloud-init | grep finished
+               if [ $? != 0 ]; then
+                   break
+               fi
+               if [ $i = rubbos_tomcat1 ]; then
+                   echo "all vm Cloud-init finished!"
+                   date
+                   return
+               fi
+            done
+            sleep 10
+        done
+    fi
 }
 
 bottlenecks_create_instance()
@@ -75,7 +111,7 @@ bottlenecks_create_instance()
     nova keypair-add --pub_key $KEY_PATH/bottlenecks_key.pub $KEY_NAME
 
     echo "create flavor"
-    nova flavor-create $FLAVOR_NAME 200 2048 10 1
+    nova flavor-create $FLAVOR_NAME 200 4096 20 4
 
     echo "use heat template to create stack"
     cd $HOT_PATH
@@ -172,10 +208,11 @@ main()
 {
     echo "create instances with heat template"
 
+    BOTTLENECKS_DEBUG=True
     BOTTLENECKS_REPO=https://gerrit.opnfv.org/gerrit/bottlenecks
     BOTTLENECKS_REPO_DIR=/tmp/opnfvrepo/bottlenecks
-    #IMAGE_URL=http://artifacts.opnfv.org/bottlenecks/rubbos/bottlenecks-trusty-server.img
-    IMAGE_URL=https://cloud-images.ubuntu.com/trusty/current/trusty-server-cloudimg-amd64-disk1.img
+    IMAGE_URL=http://artifacts.opnfv.org/bottlenecks/rubbos/bottlenecks-trusty-server.img
+    #IMAGE_URL=https://cloud-images.ubuntu.com/trusty/current/trusty-server-cloudimg-amd64-disk1.img
     IMAGE_NAME=bottlenecks-trusty-server
     KEY_PATH=$BOTTLENECKS_REPO_DIR/utils/infra_setup/bottlenecks_key
     HOT_PATH=$BOTTLENECKS_REPO_DIR/utils/infra_setup/heat_template
diff --git a/utils/infra_setup/vm_dev_setup/common.sh b/utils/infra_setup/vm_dev_setup/common.sh
index 625a447e..027081f4 100644
--- a/utils/infra_setup/vm_dev_setup/common.sh
+++ b/utils/infra_setup/vm_dev_setup/common.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -ex
+
 generate_ssh_key() {
     if [ ! -d ~/.ssh ]; then
         mkdir ~/.ssh
@@ -39,15 +41,60 @@ EOF
 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCwmx8/86Vc1yqqplMPkm/w6wB2r5Yl6CjON85Ko5SzPEpnysJwO5ygbMClQTbzCUztIpFeO8D6xdWlhPmhMEJIWtRh+Dx0swB+ByMraGTP92xxKo2ukWzYI6eu25LmJX+a1lmVPmewQq+6dPzIwgCZvwD8UVe5L+d0EP2rb3iniQVoBzksUmhL3N6jp2Lgpscfi1yXFgjFdLpDsCcFcM+Q6wQUC2+qGBYa5RxqercWzISSbvAsYwC7UepmjFpcj3W/WX2QcOCDIlAzeURvC8PbOdzCtgeMhuhH2499BcIUSzPMpjzvpRiHr5SeF/zrPeW+2Ei1MfHi1qrBCxS8JBqt bottlenecks@bottlenecks.opnfv.org
 EOF
     chmod 600 ~/.ssh/id_rsa
+
+    if [ ! -d /root/.ssh ]; then
+        mkdir /root/.ssh
+    fi
+
+    sudo cat << EOF > /root/.ssh/id_rsa
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAsJsfP/OlXNcqqqZTD5Jv8OsAdq+WJegozjfOSqOUszxKZ8rC
+cDucoGzApUE28wlM7SKRXjvA+sXVpYT5oTBCSFrUYfg8dLMAfgcjK2hkz/dscSqN
+rpFs2COnrtuS5iV/mtZZlT5nsEKvunT8yMIAmb8A/FFXuS/ndBD9q294p4kFaAc5
+LFJoS9zeo6di4KbHH4tclxYIxXS6Q7AnBXDPkOsEFAtvqhgWGuUcanq3FsyEkm7w
+LGMAu1HqZoxaXI91v1l9kHDggyJQM3lEbwvD2zncwrYHjIboR9uPfQXCFEszzKY8
+76UYh6+Unhf86z3lvthItTHx4taqwQsUvCQarQIDAQABAoIBAHF4ea+hYFIyfVt/
+2kZbFEl69KVNU++vJui4uncCe3xd9ICTqjJqWsxIk86aBDBLfX2vhC9DvU5i5k6f
+DbUfSLcshOEdmdpxWJOhClvNi1iD5M0hRj8vePu2jPqwYcY1ubNGnfBiVETP1BC9
+Md3bqH8gVgXITdDSkqfkctg/mwoo0gCvW7J+PWs8Nw5yYZlc9hfMpyNBqAexMX3H
+znrCbqj7S/7IKxlQ1AuNRJnAMrz4pZoFPwcjNv5dHWGBuY3/d3V6oHtdphAyT5Vt
+t5Hxrl/27wQcqrYm6G1/wp7q+6UfvQ7vsuBwaE62wjzodr1KP3KtzrjORz+OElrP
+RHdhW4ECgYEA4PmP6kHLKYW5K6z9TTmQaocTIk7pN71RoxkIZ6eTP7raXhbUTb6o
+fq3bEVjd6K9ZMiSjE/5z0ZHUzlKtjVkZFxQdCQhWzUuxj66zhQV657SUr5s/t+cR
+H04BARya5q5tA799PsDeKz8TzkfkWXbXDL3tOxDgS3KZx3eY3Xu5ucUCgYEAyPX1
+wwBB10pGmzyk4VItGkyH5SsDMA9aKh3tyHI25iuOg7x0VJ1TWXwPXjgtOvqOa9/C
+aww9WuCCK2Hda560eF4iU0a4kua/6OnCSdW6ATn+pmld84LSIXfe4lT0JKst3ast
+NpnCFT5/aeeJVrgr0od9GegjDkRVxz4OJmIDM8kCgYBUp/QsrUdav5QXSEgkyBV8
+0Ik1rsX5kIbovaG5B4jUQWasYyDOhID48kjt9SfDyD/jK4AmJOjGIn8WiGwANVfI
+pGvsmzO2mtYdABtTYeWGzR/qGJxYgl2iDwM3vODZDye8clEZzQ+M7HZyeAEIXTy8
+8bcUM9yC54PQWEOwjD8uFQKBgCzIAnftur5K4uj83pEHSt2taCr9Jryy7xiriq08
+JlesJRneCN5CIKy0JIiOZCXcZ5hKSHyqQZHaracVK84rk3OvJl8AR2kU2ncAgDcL
+1WMolUJoAHOfF0w19gjhfXYeXx4iYvTj1of9YU8sNUKJz9oeLxZy0D0BpUu83FJZ
+TPVhAoGBAMhFV9+VrZJSXaAw0NlZx2k2/Hq2lmUc30UZYXDW8k77qgF3EJHi3ZSl
+PBjN5vmoGcf7d7xRjAIMjsNINPdtzHJCca16lsVRc20qG1RqMiCKcqveExqpMqn1
+oV2vefV/XJtj43/jrlrGfCoI+RNVImsvp4dwKzr7YOI8Fcqf1e3G
+-----END RSA PRIVATE KEY-----
+EOF
+
+    sudo cat << EOF > /root/.ssh/id_rsa.pub
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCwmx8/86Vc1yqqplMPkm/w6wB2r5Yl6CjON85Ko5SzPEpnysJwO5ygbMClQTbzCUztIpFeO8D6xdWlhPmhMEJIWtRh+Dx0swB+ByMraGTP92xxKo2ukWzYI6eu25LmJX+a1lmVPmewQq+6dPzIwgCZvwD8UVe5L+d0EP2rb3iniQVoBzksUmhL3N6jp2Lgpscfi1yXFgjFdLpDsCcFcM+Q6wQUC2+qGBYa5RxqercWzISSbvAsYwC7UepmjFpcj3W/WX2QcOCDIlAzeURvC8PbOdzCtgeMhuhH2499BcIUSzPMpjzvpRiHr5SeF/zrPeW+2Ei1MfHi1qrBCxS8JBqt bottlenecks@bottlenecks.opnfv.org
+EOF
+    sudo chmod 600 /root/.ssh/id_rsa
+
+    sudo sed -ie 's/ssh-rsa/\n&/g' /root/.ssh/authorized_keys
+    sudo sed -ie '/echo/d' /root/.ssh/authorized_keys
 }
 
 configue_nameserver()
 {
     echo "Bottlenecks: configue nameserver"
     sudo rm /etc/resolv.conf
-    sudo echo "nameserver $1" > /etc/resolv.conf
     sudo echo "nameserver 8.8.8.8" >> /etc/resolv.conf
     sudo echo "nameserver 8.8.4.4" >> /etc/resolv.conf
+
+    sudo ifconfig
+    sudo cat /etc/resolv.conf
+    ping -c 5 www.google.com
 }
 
 install_packages()
@@ -63,3 +110,5 @@ install_packages()
     done
 }
 
+set +ex
+
diff --git a/utils/infra_setup/vm_dev_setup/setup_env.sh b/utils/infra_setup/vm_dev_setup/setup_env.sh
index 8ba76600..d09bc774 100755
--- a/utils/infra_setup/vm_dev_setup/setup_env.sh
+++ b/utils/infra_setup/vm_dev_setup/setup_env.sh
@@ -6,14 +6,14 @@ wait_vm_ok() {
     ip=$1
 
     retry=0
-    until timeout 1s ssh $ssh_args ec2-user@$ip "exit" >/dev/null 2>&1
+    until timeout 10s ssh $ssh_args ec2-user@$ip "exit" >/dev/null 2>&1
     do
         echo "retry connect rubbos vm ip $ip $retry"
         sleep 1
         let retry+=1
         if [[ $retry -ge $2 ]];then
             echo "rubbos control start timeout !!!"
-            exit 1
+            #exit 1
         fi
     done
 }
@@ -29,19 +29,26 @@ bottlenecks_prepare_env()
     for i in $rubbos_benchmark $rubbos_client1 $rubbos_client2 \
              $rubbos_client3 $rubbos_client4 $rubbos_httpd $rubbos_mysql1 $rubbos_tomcat1
     do
-        wait_vm_ok $i 120
+        wait_vm_ok $i 360
     done
 
-    # asynchronous configue other VMs
+    # configue other VMs
     for i in $rubbos_benchmark $rubbos_client1 $rubbos_client2 \
              $rubbos_client3 $rubbos_client4 $rubbos_httpd $rubbos_mysql1 $rubbos_tomcat1
     do
           scp $ssh_args -r $SCRIPT_DIR ec2-user@$i:$SCRIPT_DIR
-          ssh $ssh_args ec2-user@$i "sudo bash $SCRIPT_DIR/vm_prepare_setup.sh" &
+          ssh $ssh_args ec2-user@$i "sudo bash $SCRIPT_DIR/vm_prepare_setup.sh"
     done
 
     # ugly use ssh execute script to fix ec2-user previlege issue
     ssh $ssh_args ec2-user@$rubbos_control "sudo bash $SCRIPT_DIR/vm_prepare_setup.sh"
+
+    # test root access
+    for i in $rubbos_control $rubbos_benchmark $rubbos_client1 $rubbos_client2 \
+             $rubbos_client3 $rubbos_client4 $rubbos_httpd $rubbos_mysql1 $rubbos_tomcat1
+    do
+          ssh $ssh_args root@$i "uname -a"
+    done
 }
 
 bottlenecks_download_repo()
@@ -84,10 +91,10 @@ bottlenecks_download_packages()
     echo "Bottlenecks: download rubbos dependent packages from artifacts"
 
     curl --connect-timeout 10 -o /tmp/app_tools.tar.gz $RUBBOS_APP_TOOLS_URL 2>/dev/null
-    sudo tar zxvf /tmp/app_tools.tar.gz -C $RUBBOS_DIR
+    sudo tar zxf /tmp/app_tools.tar.gz -C $RUBBOS_DIR
     rm -rf /tmp/app_tools.tar.gz
     curl --connect-timeout 10 -o /tmp/rubbosMulini6.tar.gz $RUBBOS_MULINI6_URL 2>/dev/null
-    sudo tar zxvf /tmp/rubbosMulini6.tar.gz -C $RUBBOS_MULINI6_DIR
+    sudo tar zxf /tmp/rubbosMulini6.tar.gz -C $RUBBOS_MULINI6_DIR
     rm -rf /tmp/rubbosMulini6.tar.gz
 }
 
diff --git a/utils/infra_setup/vm_dev_setup/vm_prepare_setup.sh b/utils/infra_setup/vm_dev_setup/vm_prepare_setup.sh
index cb7b6dfd..8fb995b8 100644
--- a/utils/infra_setup/vm_dev_setup/vm_prepare_setup.sh
+++ b/utils/infra_setup/vm_dev_setup/vm_prepare_setup.sh
@@ -10,8 +10,7 @@ main()
 
     generate_ssh_key
     configue_nameserver $nameserver_ip
-    install_packages $PreInstall_Packages
-
+    #install_packages $PreInstall_Packages
 }
 
 main