Add result handler & fix test run hanging bug

JIRA: BOTTLENECK-24

Change-Id: I8ef2c55d725af9258c34c3fd9b1161565851f7e0
Signed-off-by: QiLiang <liangqi1@huawei.com>

18 files changed, 98 insertions, 59 deletions

diff --git a/rubbos/rubbos_scripts/1-1-1/scripts/BENCHMARK_configure.sh b/rubbos/rubbos_scripts/1-1-1/scripts/BENCHMARK_configure.sh
index 6fb9d8b0..4b71136c 100755
--- a/rubbos/rubbos_scripts/1-1-1/scripts/BENCHMARK_configure.sh
+++ b/rubbos/rubbos_scripts/1-1-1/scripts/BENCHMARK_configure.sh
@@ -20,7 +20,7 @@ chmod ug+x $RUBBOS_HOME/bench/*.sh
 #build clients
 echo "COMPILING RUBBOS CLIENT on $HOSTNAME"
 cd $RUBBOS_HOME/Client
-make clean
-make
+make clean >/dev/null
+make >/dev/null 2>&1
 
 echo "DONE CONFIGURING RUBBOS CLIENT on $HOSTNAME"
diff --git a/rubbos/rubbos_scripts/1-1-1/scripts/BENCHMARK_rubbos_install.sh b/rubbos/rubbos_scripts/1-1-1/scripts/BENCHMARK_rubbos_install.sh
index 098df714..ccfbd716 100755
--- a/rubbos/rubbos_scripts/1-1-1/scripts/BENCHMARK_rubbos_install.sh
+++ b/rubbos/rubbos_scripts/1-1-1/scripts/BENCHMARK_rubbos_install.sh
@@ -19,9 +19,9 @@ cp $SOFTWARE_HOME/flush_cache $RUBBOS_HOME/bench/.
 tar xzf $SOFTWARE_HOME/$SYSSTAT_TARBALL --directory=$RUBBOS_APP
 
 cd $SYSSTAT_HOME
-./configure --prefix=$SYSSTAT_HOME
-make
-sudo make install
+./configure --prefix=$SYSSTAT_HOME >/dev/null
+make >/dev/null 2>&1
+sudo make install >/dev/null
 
 # install a script to collect statistics data
 cp $OUTPUT_HOME/rubbos_conf/cpu_mem.sh $RUBBOS_APP/.
diff --git a/rubbos/rubbos_scripts/1-1-1/scripts/CLIENT1_configure.sh b/rubbos/rubbos_scripts/1-1-1/scripts/CLIENT1_configure.sh
index 6fb9d8b0..4b71136c 100755
--- a/rubbos/rubbos_scripts/1-1-1/scripts/CLIENT1_configure.sh
+++ b/rubbos/rubbos_scripts/1-1-1/scripts/CLIENT1_configure.sh
@@ -20,7 +20,7 @@ chmod ug+x $RUBBOS_HOME/bench/*.sh
 #build clients
 echo "COMPILING RUBBOS CLIENT on $HOSTNAME"
 cd $RUBBOS_HOME/Client
-make clean
-make
+make clean >/dev/null
+make >/dev/null 2>&1
 
 echo "DONE CONFIGURING RUBBOS CLIENT on $HOSTNAME"
diff --git a/rubbos/rubbos_scripts/1-1-1/scripts/CLIENT1_rubbos_install.sh b/rubbos/rubbos_scripts/1-1-1/scripts/CLIENT1_rubbos_install.sh
index 098df714..ccfbd716 100755
--- a/rubbos/rubbos_scripts/1-1-1/scripts/CLIENT1_rubbos_install.sh
+++ b/rubbos/rubbos_scripts/1-1-1/scripts/CLIENT1_rubbos_install.sh
@@ -19,9 +19,9 @@ cp $SOFTWARE_HOME/flush_cache $RUBBOS_HOME/bench/.
 tar xzf $SOFTWARE_HOME/$SYSSTAT_TARBALL --directory=$RUBBOS_APP
 
 cd $SYSSTAT_HOME
-./configure --prefix=$SYSSTAT_HOME
-make
-sudo make install
+./configure --prefix=$SYSSTAT_HOME >/dev/null
+make >/dev/null 2>&1
+sudo make install >/dev/null
 
 # install a script to collect statistics data
 cp $OUTPUT_HOME/rubbos_conf/cpu_mem.sh $RUBBOS_APP/.
diff --git a/rubbos/rubbos_scripts/1-1-1/scripts/CLIENT2_configure.sh b/rubbos/rubbos_scripts/1-1-1/scripts/CLIENT2_configure.sh
index 6fb9d8b0..4b71136c 100755
--- a/rubbos/rubbos_scripts/1-1-1/scripts/CLIENT2_configure.sh
+++ b/rubbos/rubbos_scripts/1-1-1/scripts/CLIENT2_configure.sh
@@ -20,7 +20,7 @@ chmod ug+x $RUBBOS_HOME/bench/*.sh
 #build clients
 echo "COMPILING RUBBOS CLIENT on $HOSTNAME"
 cd $RUBBOS_HOME/Client
-make clean
-make
+make clean >/dev/null
+make >/dev/null 2>&1
 
 echo "DONE CONFIGURING RUBBOS CLIENT on $HOSTNAME"
diff --git a/rubbos/rubbos_scripts/1-1-1/scripts/CLIENT2_rubbos_install.sh b/rubbos/rubbos_scripts/1-1-1/scripts/CLIENT2_rubbos_install.sh
index 098df714..ccfbd716 100755
--- a/rubbos/rubbos_scripts/1-1-1/scripts/CLIENT2_rubbos_install.sh
+++ b/rubbos/rubbos_scripts/1-1-1/scripts/CLIENT2_rubbos_install.sh
@@ -19,9 +19,9 @@ cp $SOFTWARE_HOME/flush_cache $RUBBOS_HOME/bench/.
 tar xzf $SOFTWARE_HOME/$SYSSTAT_TARBALL --directory=$RUBBOS_APP
 
 cd $SYSSTAT_HOME
-./configure --prefix=$SYSSTAT_HOME
-make
-sudo make install
+./configure --prefix=$SYSSTAT_HOME >/dev/null
+make >/dev/null 2>&1
+sudo make install >/dev/null
 
 # install a script to collect statistics data
 cp $OUTPUT_HOME/rubbos_conf/cpu_mem.sh $RUBBOS_APP/.
diff --git a/rubbos/rubbos_scripts/1-1-1/scripts/CLIENT3_configure.sh b/rubbos/rubbos_scripts/1-1-1/scripts/CLIENT3_configure.sh
index 6fb9d8b0..4b71136c 100755
--- a/rubbos/rubbos_scripts/1-1-1/scripts/CLIENT3_configure.sh
+++ b/rubbos/rubbos_scripts/1-1-1/scripts/CLIENT3_configure.sh
@@ -20,7 +20,7 @@ chmod ug+x $RUBBOS_HOME/bench/*.sh
 #build clients
 echo "COMPILING RUBBOS CLIENT on $HOSTNAME"
 cd $RUBBOS_HOME/Client
-make clean
-make
+make clean >/dev/null
+make >/dev/null 2>&1
 
 echo "DONE CONFIGURING RUBBOS CLIENT on $HOSTNAME"
diff --git a/rubbos/rubbos_scripts/1-1-1/scripts/CLIENT3_rubbos_install.sh b/rubbos/rubbos_scripts/1-1-1/scripts/CLIENT3_rubbos_install.sh
index 098df714..ccfbd716 100755
--- a/rubbos/rubbos_scripts/1-1-1/scripts/CLIENT3_rubbos_install.sh
+++ b/rubbos/rubbos_scripts/1-1-1/scripts/CLIENT3_rubbos_install.sh
@@ -19,9 +19,9 @@ cp $SOFTWARE_HOME/flush_cache $RUBBOS_HOME/bench/.
 tar xzf $SOFTWARE_HOME/$SYSSTAT_TARBALL --directory=$RUBBOS_APP
 
 cd $SYSSTAT_HOME
-./configure --prefix=$SYSSTAT_HOME
-make
-sudo make install
+./configure --prefix=$SYSSTAT_HOME >/dev/null
+make >/dev/null 2>&1
+sudo make install >/dev/null
 
 # install a script to collect statistics data
 cp $OUTPUT_HOME/rubbos_conf/cpu_mem.sh $RUBBOS_APP/.
diff --git a/rubbos/rubbos_scripts/1-1-1/scripts/CLIENT4_configure.sh b/rubbos/rubbos_scripts/1-1-1/scripts/CLIENT4_configure.sh
index 6fb9d8b0..4b71136c 100755
--- a/rubbos/rubbos_scripts/1-1-1/scripts/CLIENT4_configure.sh
+++ b/rubbos/rubbos_scripts/1-1-1/scripts/CLIENT4_configure.sh
@@ -20,7 +20,7 @@ chmod ug+x $RUBBOS_HOME/bench/*.sh
 #build clients
 echo "COMPILING RUBBOS CLIENT on $HOSTNAME"
 cd $RUBBOS_HOME/Client
-make clean
-make
+make clean >/dev/null
+make >/dev/null 2>&1
 
 echo "DONE CONFIGURING RUBBOS CLIENT on $HOSTNAME"
diff --git a/rubbos/rubbos_scripts/1-1-1/scripts/CLIENT4_rubbos_install.sh b/rubbos/rubbos_scripts/1-1-1/scripts/CLIENT4_rubbos_install.sh
index 098df714..ccfbd716 100755
--- a/rubbos/rubbos_scripts/1-1-1/scripts/CLIENT4_rubbos_install.sh
+++ b/rubbos/rubbos_scripts/1-1-1/scripts/CLIENT4_rubbos_install.sh
@@ -19,9 +19,9 @@ cp $SOFTWARE_HOME/flush_cache $RUBBOS_HOME/bench/.
 tar xzf $SOFTWARE_HOME/$SYSSTAT_TARBALL --directory=$RUBBOS_APP
 
 cd $SYSSTAT_HOME
-./configure --prefix=$SYSSTAT_HOME
-make
-sudo make install
+./configure --prefix=$SYSSTAT_HOME >/dev/null
+make >/dev/null 2>&1
+sudo make install >/dev/null
 
 # install a script to collect statistics data
 cp $OUTPUT_HOME/rubbos_conf/cpu_mem.sh $RUBBOS_APP/.
diff --git a/rubbos/rubbos_scripts/1-1-1/scripts/CONTROL_rubbos_exec.sh b/rubbos/rubbos_scripts/1-1-1/scripts/CONTROL_rubbos_exec.sh
index 0bb298e6..32163d48 100755
--- a/rubbos/rubbos_scripts/1-1-1/scripts/CONTROL_rubbos_exec.sh
+++ b/rubbos/rubbos_scripts/1-1-1/scripts/CONTROL_rubbos_exec.sh
@@ -20,7 +20,6 @@ ssh $BENCHMARK_HOST "
 #TODO use for loop to genrate rubbos.properties file 200 ~ 1700
 for i in {2..2}
 do
-
   ssh $BENCHMARK_HOST "
     source /bottlenecks/rubbos/rubbos_scripts/1-1-1/set_bottlenecks_rubbos_env.sh
     rm -f $RUBBOS_HOME/Client/rubbos.properties
@@ -65,6 +64,7 @@ do
   #" &
 
   ssh $BENCHMARK_HOST "
+    set -x
     source /bottlenecks/rubbos/rubbos_scripts/1-1-1/set_bottlenecks_rubbos_env.sh
 
     cd $RUBBOS_HOME/bench
@@ -80,7 +80,7 @@ do
     for host in $BENCHMARK_HOST $CLIENT1_HOST $CLIENT2_HOST $CLIENT3_HOST \
                 $CLIENT4_HOST $HTTPD_HOST $TOMCAT1_HOST $MYSQL1_HOST
     do
-      for f in sar-* ps-* iostat-* mysql_mon-* postgres_lock-*
+      for f in 'sar-*' 'ps-*' 'iostat-*' 'mysql_mon-*' 'postgres_lock-*'
       do
         scp $scp_options \$host:$RUBBOS_APP/\$f ./
       done
@@ -91,11 +91,9 @@ do
     mv 20* $TMP_RESULTS_DIR_BASE/$RUBBOS_RESULTS_DIR_NAME/
   "
 
-  # TODO debug the rest of sciripts
-  exit 0
   #$OUTPUT_HOME/scripts/stop_all.sh
-  $OUTPUT_HOME/scripts/kill_all.sh
-  sleep 15
+  #$OUTPUT_HOME/scripts/kill_all.sh
+  #sleep 15
   echo "End Browsing Only with rubbos.properties_$((100*i))"
 
   # Read/Write
@@ -105,18 +103,31 @@ done
 echo "Processing the results..."
 ssh $BENCHMARK_HOST "
   cd $TMP_RESULTS_DIR_BASE
-  cd $RUBBOS_RESULTS_DIR_NAME
-  scp $RUBBOS_RESULTS_HOST:$RUBBOS_RESULTS_DIR_BASE/calc-sarSummary.prl ../
-  ../calc-sarSummary.prl
+  #cd $RUBBOS_RESULTS_DIR_NAME
+  #scp $RUBBOS_RESULTS_HOST:$RUBBOS_RESULTS_DIR_BASE/calc-sarSummary.prl ../
+  #../calc-sarSummary.prl
 
-  rm -f 20*/*.bin
+  #rm -f 20*/*.bin
 
-  cd ../
-  tar zcvf $RUBBOS_RESULTS_DIR_NAME.tgz $RUBBOS_RESULTS_DIR_NAME
-  scp $RUBBOS_RESULTS_DIR_NAME.tgz $RUBBOS_RESULTS_HOST:$RUBBOS_RESULTS_DIR_BASE/
+  #cd ../
+  tar zcf $RUBBOS_RESULTS_DIR_NAME.tgz $RUBBOS_RESULTS_DIR_NAME
+  scp $scp_options $RUBBOS_RESULTS_DIR_NAME.tgz $RUBBOS_RESULTS_HOST:$RUBBOS_RESULTS_DIR_BASE/
 "
 
+echo "Push the results to DB..."
+cd $RUBBOS_RESULTS_DIR_BASE
+
+ls $RUBBOS_RESULTS_DIR_NAME.tgz
+tar zxf $RUBBOS_RESULTS_DIR_NAME.tgz
+ls $RUBBOS_RESULTS_DIR_NAME
+
+python $BOTTLENECKS_TOP/utils/dashboard/process_data.py \
+           $RUBBOS_RESULTS_DIR_BASE/$RUBBOS_RESULTS_DIR_NAME \
+           $BOTTLENECKS_TOP/utils/dashboard/dashboard.yaml
+cd -
+
 echo "Finish RUBBoS"
+touch /tmp/rubbos_finished
 
 set +x
 
diff --git a/rubbos/rubbos_scripts/1-1-1/scripts/HTTPD_install.sh b/rubbos/rubbos_scripts/1-1-1/scripts/HTTPD_install.sh
index 1c388d2d..47e915b5 100755
--- a/rubbos/rubbos_scripts/1-1-1/scripts/HTTPD_install.sh
+++ b/rubbos/rubbos_scripts/1-1-1/scripts/HTTPD_install.sh
@@ -21,16 +21,16 @@ chmod 755 $RUBBOS_APP
 # apache
 tar zxf $SOFTWARE_HOME/$HTTPD_TARBALL --directory=$RUBBOS_APP
 cd $HTTPD_INSTALL_FILES 
-./configure --prefix=$HTTPD_HOME --enable-module=so --enable-so --with-mpm=worker
-make 
-make install 
+./configure --prefix=$HTTPD_HOME --enable-module=so --enable-so --with-mpm=worker >/dev/null
+make >/dev/null
+make install >/dev/null
 
 # mod jk
 tar zxf $SOFTWARE_HOME/$MOD_JK_TARBALL --directory=$RUBBOS_APP
 tar zxf $SOFTWARE_HOME/$JAVA_TARBALL --directory=$RUBBOS_APP
 cd $MOD_JK_INSTALL_CONFIGURE
-./configure --with-apxs=$HTTPD_HOME/bin/apxs --enable-jni --with-java-home=$JAVA_HOME
-make
-make install 
+./configure --with-apxs=$HTTPD_HOME/bin/apxs --enable-jni --with-java-home=$JAVA_HOME >/dev/null
+make >/dev/null
+make install >/dev/null
 
 echo "APACHE IS INSTALLED on $HOSTNAME"
diff --git a/rubbos/rubbos_scripts/1-1-1/scripts/HTTPD_rubbos_install.sh b/rubbos/rubbos_scripts/1-1-1/scripts/HTTPD_rubbos_install.sh
index 667a5d10..fd1f5dc7 100755
--- a/rubbos/rubbos_scripts/1-1-1/scripts/HTTPD_rubbos_install.sh
+++ b/rubbos/rubbos_scripts/1-1-1/scripts/HTTPD_rubbos_install.sh
@@ -15,9 +15,9 @@ cp $SOFTWARE_HOME/flush_cache $RUBBOS_HOME/bench/.
 tar xzf $SOFTWARE_HOME/$SYSSTAT_TARBALL --directory=$RUBBOS_APP
 
 cd $SYSSTAT_HOME
-./configure --prefix=$SYSSTAT_HOME
-make
-sudo make install
+./configure --prefix=$SYSSTAT_HOME >/dev/null
+make >/dev/null 2>&1
+sudo make install >/dev/null
 
 # install a script to collect statistics data
 cp $OUTPUT_HOME/rubbos_conf/cpu_mem.sh $RUBBOS_APP/.
diff --git a/rubbos/rubbos_scripts/1-1-1/scripts/MYSQL1_rubbos_install.sh b/rubbos/rubbos_scripts/1-1-1/scripts/MYSQL1_rubbos_install.sh
index 6014d22d..33205997 100755
--- a/rubbos/rubbos_scripts/1-1-1/scripts/MYSQL1_rubbos_install.sh
+++ b/rubbos/rubbos_scripts/1-1-1/scripts/MYSQL1_rubbos_install.sh
@@ -15,9 +15,9 @@ cp $SOFTWARE_HOME/flush_cache $RUBBOS_HOME/bench/.
 # install sysstat
 tar xzf $SOFTWARE_HOME/$SYSSTAT_TARBALL --directory=$RUBBOS_APP
 cd $SYSSTAT_HOME
-./configure --prefix=$SYSSTAT_HOME
-make
-sudo make install
+./configure --prefix=$SYSSTAT_HOME >/dev/null
+make >/dev/null 2>&1
+sudo make install >/dev/null
 
 # install a script to collect statistics data
 cp $OUTPUT_HOME/rubbos_conf/cpu_mem.sh $RUBBOS_APP/.
diff --git a/rubbos/rubbos_scripts/1-1-1/scripts/TOMCAT1_rubbosSL_configure.sh b/rubbos/rubbos_scripts/1-1-1/scripts/TOMCAT1_rubbosSL_configure.sh
index 91e4065f..fe344c8a 100755
--- a/rubbos/rubbos_scripts/1-1-1/scripts/TOMCAT1_rubbosSL_configure.sh
+++ b/rubbos/rubbos_scripts/1-1-1/scripts/TOMCAT1_rubbosSL_configure.sh
@@ -18,9 +18,9 @@ sed 's/public static final int    BrowseCategoriesPoolSize      = 6;/public stat
 mv Config.java.tmp Config.java
 
 cd $RUBBOS_HOME/Servlets
-/bottlenecks/rubbos/app/apache-ant-1.6.5/bin/ant clean
-/bottlenecks/rubbos/app/apache-ant-1.6.5/bin/ant dist
-make
+/bottlenecks/rubbos/app/apache-ant-1.6.5/bin/ant clean >/dev/null
+/bottlenecks/rubbos/app/apache-ant-1.6.5/bin/ant dist >/dev/null
+make >/dev/null 2>&1
 cp rubbos.war $CATALINA_HOME/webapps/
 
 echo "DONE CONFIGURING RUBBOS SERVLET on $HOSTNAME" 
diff --git a/rubbos/rubbos_scripts/1-1-1/scripts/TOMCAT1_rubbos_install.sh b/rubbos/rubbos_scripts/1-1-1/scripts/TOMCAT1_rubbos_install.sh
index 1027cc71..466fc933 100755
--- a/rubbos/rubbos_scripts/1-1-1/scripts/TOMCAT1_rubbos_install.sh
+++ b/rubbos/rubbos_scripts/1-1-1/scripts/TOMCAT1_rubbos_install.sh
@@ -16,9 +16,9 @@ mkdir -p $SYSSTAT_HOME
 tar xzf $SOFTWARE_HOME/$SYSSTAT_TARBALL --directory=$RUBBOS_APP
 
 cd $SYSSTAT_HOME
-./configure --prefix=$SYSSTAT_HOME
-make
-sudo make install
+./configure --prefix=$SYSSTAT_HOME >/dev/null
+make >/dev/null 2>&1
+sudo make install >/dev/null
 
 # install a script to collect statistics data
 cp $OUTPUT_HOME/rubbos_conf/cpu_mem.sh $RUBBOS_APP/.
diff --git a/utils/infra_setup/heat_template/HOT_create_instance.sh b/utils/infra_setup/heat_template/HOT_create_instance.sh
index 4de29b3a..8266229b 100755
--- a/utils/infra_setup/heat_template/HOT_create_instance.sh
+++ b/utils/infra_setup/heat_template/HOT_create_instance.sh
@@ -119,6 +119,34 @@ bottlenecks_create_instance()
          -P "image=$IMAGE_NAME;key_name=$KEY_NAME;public_net=$PUBLIC_NET_NAME;flavor=$FLAVOR_NAME"
 }
 
+bottlenecks_rubbos_wait_finish()
+{
+    echo "Start checking rubbos running status..."
+    retry=0
+    while true
+    do
+        ssh $ssh_args ec2-user@$control_ip "
+            FILE=/tmp/rubbos_finished
+            if [ -f \$FILE ]; then
+               exit 0
+            else
+               exit 1
+            fi
+        "
+        if [ $? = 0 ]; then
+            echo "Rubbos test case successfully finished :)"
+            return 0
+        fi
+        echo "Rubbos running $retry ..."
+        sleep 30
+        let retry+=1
+        if [[ $retry -ge $1 ]]; then
+            echo "Rubbos test case timeout :("
+            return 1
+        fi
+    done
+}
+
 bottlenecks_rubbos_run()
 {
     echo "Run Rubbos"
@@ -141,7 +169,9 @@ bottlenecks_rubbos_run()
         $BOTTLENECKS_REPO_DIR/utils/infra_setup/vm_dev_setup \
         ec2-user@$control_ip:/tmp
     ssh $ssh_args \
-        ec2-user@$control_ip "bash /tmp/vm_dev_setup/setup_env.sh"
+        ec2-user@$control_ip "bash /tmp/vm_dev_setup/setup_env.sh" &
+
+    bottlenecks_rubbos_wait_finish 240
 
     rm -rf $BOTTLENECKS_REPO_DIR/utils/infra_setup/vm_dev_setup/hosts.conf
 }
diff --git a/utils/infra_setup/vm_dev_setup/setup_env.sh b/utils/infra_setup/vm_dev_setup/setup_env.sh
index bd0ffb56..f1572e05 100755
--- a/utils/infra_setup/vm_dev_setup/setup_env.sh
+++ b/utils/infra_setup/vm_dev_setup/setup_env.sh
@@ -104,8 +104,6 @@ bottlenecks_rubbos_install_exe()
 
     cd $RUBBOS_RUN_DIR
     sudo ./run.sh
-    cd $RUBBOS_EXE_DIR
-    sudo ./CONTROL_rubbos_exec.sh
 }
 
 main()