improve data processing script for rubbos

JIRA: BOTTLENECK-63

Change-Id: Iddcb8f354fcf916b3057f677a1f7b0149b727f33
Signed-off-by: wangyaoguang <sunshine.wang@huawei.com>

2 files changed, 69 insertions, 22 deletions

diff --git a/utils/dashboard/process_data.py b/utils/dashboard/process_data.py
index 7a7144f8..d511a08d 100644
--- a/utils/dashboard/process_data.py
+++ b/utils/dashboard/process_data.py
@@ -10,22 +10,66 @@
 
 import subprocess as subp
 import sys
-from collector import Collector
+from rubbos_collector import RubbosCollector
 from uploader import Uploader
 
+def printUsage():
+    print "Usage: python process_data.py required_params(**) optional_params([])"
+    print "       ** -i|--input   input_data_dir"
+    print "       ** -s|--suite   suite_name"
+    print "       ** -c|--conf    conf_file"
+    print "       [] -o|--output  output_file"
+    print "       [] -u|--upload  yes|no"
 
-#process data
-if len(sys.argv)!=3:
-    print "Wrong arguments, please input 2 parameters, 1st as raw data path; "\
-    "2nd as config yaml!!"
-    exit (1)
-data_home = sys.argv[1]
-conf = sys.argv[2]
+def process(input_dir, suite_name):
+    result = dict()
+    if suite_name == "rubbos":
+        result = RubbosCollector().collect_data(input_dir)
+    return result
 
-#1collect result
-result = Collector().collect_data(data_home)
-print "Result collected:\n%s" % result
+def writeResult(output_file, result):
+    f = open(output_file, "w")
+    if isinstance(result, list):
+        for elem in result:
+            f.write(str(elem) + "\n")
+    f.close()
 
-#2upload result
-Uploader(conf).upload_result("rubbos", result)
+def uploadResult(conf, suite_name, result):
+    Uploader(conf).upload_result(suite_name, result)
 
+def main():
+    if len(sys.argv) < 7 or len(sys.argv) % 2 == 0:
+        printUsage()
+        exit (1)
+    i = 1
+    params = dict()
+    while (i < len(sys.argv)):
+        print sys.argv[i]
+        if sys.argv[i]=="-i" or sys.argv[i]=="--input":
+            params["input"] = sys.argv[i+1]
+        if sys.argv[i]=="-s" or sys.argv[i]=="--suite":
+            params["suite"] = sys.argv[i+1]
+        if sys.argv[i]=="-c" or sys.argv[i]=="--conf":
+            params["conf"] = sys.argv[i+1]
+        if sys.argv[i]=="-o" or sys.argv[i]=="--output":
+            params["output"] = sys.argv[i+1]
+        if sys.argv[i]=="-u" or sys.argv[i]=="--upload":
+            params["upload"] = sys.argv[i+1]
+        i = i+2
+    if not(params.has_key("input") and params.has_key("suite") and params.has_key("conf")):
+        print "Lack some required parameters."
+        exit (1)
+
+    result = process(params["input"], params["suite"])
+    print "Results:"
+    for elem in result:
+        print elem
+
+    if params.has_key("output"):
+        writeResult(params["output"],result)
+
+    if params.has_key("upload") and params["upload"].lower()=="yes":
+        uploadResult(params["conf"], params["suite"], result)
+
+if __name__=="__main__":
+    main()
diff --git a/utils/dashboard/collector.py b/utils/dashboard/rubbos_collector.py
index 1687f80f..c9851739 100755
--- a/utils/dashboard/collector.py
+++ b/utils/dashboard/rubbos_collector.py
@@ -14,33 +14,36 @@ def exec_shell(cmd):
     out,err = subp.Popen(cmd, stdout=subp.PIPE, shell=True).communicate()
     return out.strip()
 
-
 def get_onetime_data(dir_name):
     cmd = "grep -in 'remote client nodes' %s/index.html|awk '{print $5}'|awk -F '<' '{print $1}'" % dir_name
     client_node_num = int(exec_shell(cmd))
     cmd = "grep -n 'Number of clients' %s/index.html|awk '{print $5}'|awk -F '<' '{print $1}'" % dir_name
     each_client_num = int(exec_shell(cmd))
     total_client = (client_node_num+1) * each_client_num
-    cmd = 'grep -n "throughput" %s/stat_client*.html |awk -F "<B>" \'FNR%%4==0 {printf "%%s\\n", $3 }\'|awk \'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name
+
+    cmd = 'grep -n "throughput" %s/stat_client*.html |awk -F "<B>" \'{if (FNR%%2==0 && FNR%%4!=0) {printf "%%s\\n", $3}}\'|awk \'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name
     throughput = int(exec_shell(cmd))
 
-    return total_client, throughput
+    cmd = 'grep -n "Total" %s/stat_client*.html |awk -F "<B>" \'{if (FNR==4) {printf "%%s\\n", $4}}\'|awk -F "</B>" \'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name
+    request = int(exec_shell(cmd))
+
+    cmd = 'grep -n "Total" %s/stat_client*.html |awk -F "<B>" \'{if (FNR==4) {printf "%%s\\n", $5}}\'|awk -F "</B>" \'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name
+    error_request = int(exec_shell(cmd))
 
+    return total_client, throughput, request, error_request
 
-class Collector(object):
 
+class RubbosCollector(object):
 
     def __init__(self):
         pass
 
-
     def collect_data(self, data_home):
         cmd =  'ls -l %s |grep ^d|awk \'{print $9}\'' % data_home
         result = []
         for subdir in exec_shell(cmd).split('\n'):
-            total_client, throughput = get_onetime_data(data_home+'/'+subdir)
-            result.append({'client':total_client, 'throughput':throughput})
-            result.sort(key=lambda x:x['client'])
+            total_client, throughput, request, error_request = get_onetime_data(data_home+'/'+subdir)
+            result.append({'client':total_client, 'throughput':throughput, 'request':request, 'error_request':error_request})
+        result.sort(key=lambda x:x['client'])
 
         return result;
-