summaryrefslogtreecommitdiffstats
path: root/utils
diff options
context:
space:
mode:
authorwangyaoguang <sunshine.wang@huawei.com>2016-04-15 11:26:04 +0800
committerwangyaoguang <sunshine.wang@huawei.com>2016-04-15 11:43:20 +0800
commite2b73bf29779412d3b0ceb7742e6a4bdcecb7f74 (patch)
tree3acc602c41485bf570c8b2a5a277dabadd836ea7 /utils
parent7d25232df2f07b9bb3c2820a1e248a8a95cbee28 (diff)
improve data processing script for rubbos
JIRA: BOTTLENECK-63 Change-Id: Iddcb8f354fcf916b3057f677a1f7b0149b727f33 Signed-off-by: wangyaoguang <sunshine.wang@huawei.com>
Diffstat (limited to 'utils')
-rw-r--r--utils/dashboard/process_data.py70
-rwxr-xr-xutils/dashboard/rubbos_collector.py (renamed from utils/dashboard/collector.py)21
2 files changed, 69 insertions, 22 deletions
diff --git a/utils/dashboard/process_data.py b/utils/dashboard/process_data.py
index 7a7144f8..d511a08d 100644
--- a/utils/dashboard/process_data.py
+++ b/utils/dashboard/process_data.py
@@ -10,22 +10,66 @@
import subprocess as subp
import sys
-from collector import Collector
+from rubbos_collector import RubbosCollector
from uploader import Uploader
+def printUsage():
+ print "Usage: python process_data.py required_params(**) optional_params([])"
+ print " ** -i|--input input_data_dir"
+ print " ** -s|--suite suite_name"
+ print " ** -c|--conf conf_file"
+ print " [] -o|--output output_file"
+ print " [] -u|--upload yes|no"
-#process data
-if len(sys.argv)!=3:
- print "Wrong arguments, please input 2 parameters, 1st as raw data path; "\
- "2nd as config yaml!!"
- exit (1)
-data_home = sys.argv[1]
-conf = sys.argv[2]
+def process(input_dir, suite_name):
+ result = dict()
+ if suite_name == "rubbos":
+ result = RubbosCollector().collect_data(input_dir)
+ return result
-#1collect result
-result = Collector().collect_data(data_home)
-print "Result collected:\n%s" % result
+def writeResult(output_file, result):
+ f = open(output_file, "w")
+ if isinstance(result, list):
+ for elem in result:
+ f.write(str(elem) + "\n")
+ f.close()
-#2upload result
-Uploader(conf).upload_result("rubbos", result)
+def uploadResult(conf, suite_name, result):
+ Uploader(conf).upload_result(suite_name, result)
+def main():
+ if len(sys.argv) < 7 or len(sys.argv) % 2 == 0:
+ printUsage()
+ exit (1)
+ i = 1
+ params = dict()
+ while (i < len(sys.argv)):
+ print sys.argv[i]
+ if sys.argv[i]=="-i" or sys.argv[i]=="--input":
+ params["input"] = sys.argv[i+1]
+ if sys.argv[i]=="-s" or sys.argv[i]=="--suite":
+ params["suite"] = sys.argv[i+1]
+ if sys.argv[i]=="-c" or sys.argv[i]=="--conf":
+ params["conf"] = sys.argv[i+1]
+ if sys.argv[i]=="-o" or sys.argv[i]=="--output":
+ params["output"] = sys.argv[i+1]
+ if sys.argv[i]=="-u" or sys.argv[i]=="--upload":
+ params["upload"] = sys.argv[i+1]
+ i = i+2
+ if not(params.has_key("input") and params.has_key("suite") and params.has_key("conf")):
+ print "Lack some required parameters."
+ exit (1)
+
+ result = process(params["input"], params["suite"])
+ print "Results:"
+ for elem in result:
+ print elem
+
+ if params.has_key("output"):
+ writeResult(params["output"],result)
+
+ if params.has_key("upload") and params["upload"].lower()=="yes":
+ uploadResult(params["conf"], params["suite"], result)
+
+if __name__=="__main__":
+ main()
diff --git a/utils/dashboard/collector.py b/utils/dashboard/rubbos_collector.py
index 1687f80f..c9851739 100755
--- a/utils/dashboard/collector.py
+++ b/utils/dashboard/rubbos_collector.py
@@ -14,33 +14,36 @@ def exec_shell(cmd):
out,err = subp.Popen(cmd, stdout=subp.PIPE, shell=True).communicate()
return out.strip()
-
def get_onetime_data(dir_name):
cmd = "grep -in 'remote client nodes' %s/index.html|awk '{print $5}'|awk -F '<' '{print $1}'" % dir_name
client_node_num = int(exec_shell(cmd))
cmd = "grep -n 'Number of clients' %s/index.html|awk '{print $5}'|awk -F '<' '{print $1}'" % dir_name
each_client_num = int(exec_shell(cmd))
total_client = (client_node_num+1) * each_client_num
- cmd = 'grep -n "throughput" %s/stat_client*.html |awk -F "<B>" \'FNR%%4==0 {printf "%%s\\n", $3 }\'|awk \'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name
+
+ cmd = 'grep -n "throughput" %s/stat_client*.html |awk -F "<B>" \'{if (FNR%%2==0 && FNR%%4!=0) {printf "%%s\\n", $3}}\'|awk \'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name
throughput = int(exec_shell(cmd))
- return total_client, throughput
+ cmd = 'grep -n "Total" %s/stat_client*.html |awk -F "<B>" \'{if (FNR==4) {printf "%%s\\n", $4}}\'|awk -F "</B>" \'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name
+ request = int(exec_shell(cmd))
+
+ cmd = 'grep -n "Total" %s/stat_client*.html |awk -F "<B>" \'{if (FNR==4) {printf "%%s\\n", $5}}\'|awk -F "</B>" \'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name
+ error_request = int(exec_shell(cmd))
+ return total_client, throughput, request, error_request
-class Collector(object):
+class RubbosCollector(object):
def __init__(self):
pass
-
def collect_data(self, data_home):
cmd = 'ls -l %s |grep ^d|awk \'{print $9}\'' % data_home
result = []
for subdir in exec_shell(cmd).split('\n'):
- total_client, throughput = get_onetime_data(data_home+'/'+subdir)
- result.append({'client':total_client, 'throughput':throughput})
- result.sort(key=lambda x:x['client'])
+ total_client, throughput, request, error_request = get_onetime_data(data_home+'/'+subdir)
+ result.append({'client':total_client, 'throughput':throughput, 'request':request, 'error_request':error_request})
+ result.sort(key=lambda x:x['client'])
return result;
-