From e2b73bf29779412d3b0ceb7742e6a4bdcecb7f74 Mon Sep 17 00:00:00 2001 From: wangyaoguang Date: Fri, 15 Apr 2016 11:26:04 +0800 Subject: improve data processing script for rubbos JIRA: BOTTLENECK-63 Change-Id: Iddcb8f354fcf916b3057f677a1f7b0149b727f33 Signed-off-by: wangyaoguang --- utils/dashboard/collector.py | 46 ------------------------ utils/dashboard/process_data.py | 70 ++++++++++++++++++++++++++++++------- utils/dashboard/rubbos_collector.py | 49 ++++++++++++++++++++++++++ 3 files changed, 106 insertions(+), 59 deletions(-) delete mode 100755 utils/dashboard/collector.py create mode 100755 utils/dashboard/rubbos_collector.py diff --git a/utils/dashboard/collector.py b/utils/dashboard/collector.py deleted file mode 100755 index 1687f80f..00000000 --- a/utils/dashboard/collector.py +++ /dev/null @@ -1,46 +0,0 @@ -############################################################################## -# Copyright (c) 2015 Huawei Technologies Co.,Ltd. and others -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - -import subprocess as subp - -def exec_shell(cmd): - out,err = subp.Popen(cmd, stdout=subp.PIPE, shell=True).communicate() - return out.strip() - - -def get_onetime_data(dir_name): - cmd = "grep -in 'remote client nodes' %s/index.html|awk '{print $5}'|awk -F '<' '{print $1}'" % dir_name - client_node_num = int(exec_shell(cmd)) - cmd = "grep -n 'Number of clients' %s/index.html|awk '{print $5}'|awk -F '<' '{print $1}'" % dir_name - each_client_num = int(exec_shell(cmd)) - total_client = (client_node_num+1) * each_client_num - cmd = 'grep -n "throughput" %s/stat_client*.html |awk -F "" \'FNR%%4==0 {printf "%%s\\n", $3 }\'|awk \'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name - throughput = int(exec_shell(cmd)) - - return total_client, throughput - - -class Collector(object): - - - def __init__(self): - pass - - - def collect_data(self, data_home): - cmd = 'ls -l %s |grep ^d|awk \'{print $9}\'' % data_home - result = [] - for subdir in exec_shell(cmd).split('\n'): - total_client, throughput = get_onetime_data(data_home+'/'+subdir) - result.append({'client':total_client, 'throughput':throughput}) - result.sort(key=lambda x:x['client']) - - return result; - diff --git a/utils/dashboard/process_data.py b/utils/dashboard/process_data.py index 7a7144f8..d511a08d 100644 --- a/utils/dashboard/process_data.py +++ b/utils/dashboard/process_data.py @@ -10,22 +10,66 @@ import subprocess as subp import sys -from collector import Collector +from rubbos_collector import RubbosCollector from uploader import Uploader +def printUsage(): + print "Usage: python process_data.py required_params(**) optional_params([])" + print " ** -i|--input input_data_dir" + print " ** -s|--suite suite_name" + print " ** -c|--conf conf_file" + print " [] -o|--output output_file" + print " [] -u|--upload yes|no" -#process data -if len(sys.argv)!=3: - print "Wrong arguments, please input 2 parameters, 1st as raw data path; "\ - "2nd as config yaml!!" - exit (1) -data_home = sys.argv[1] -conf = sys.argv[2] +def process(input_dir, suite_name): + result = dict() + if suite_name == "rubbos": + result = RubbosCollector().collect_data(input_dir) + return result -#1collect result -result = Collector().collect_data(data_home) -print "Result collected:\n%s" % result +def writeResult(output_file, result): + f = open(output_file, "w") + if isinstance(result, list): + for elem in result: + f.write(str(elem) + "\n") + f.close() -#2upload result -Uploader(conf).upload_result("rubbos", result) +def uploadResult(conf, suite_name, result): + Uploader(conf).upload_result(suite_name, result) +def main(): + if len(sys.argv) < 7 or len(sys.argv) % 2 == 0: + printUsage() + exit (1) + i = 1 + params = dict() + while (i < len(sys.argv)): + print sys.argv[i] + if sys.argv[i]=="-i" or sys.argv[i]=="--input": + params["input"] = sys.argv[i+1] + if sys.argv[i]=="-s" or sys.argv[i]=="--suite": + params["suite"] = sys.argv[i+1] + if sys.argv[i]=="-c" or sys.argv[i]=="--conf": + params["conf"] = sys.argv[i+1] + if sys.argv[i]=="-o" or sys.argv[i]=="--output": + params["output"] = sys.argv[i+1] + if sys.argv[i]=="-u" or sys.argv[i]=="--upload": + params["upload"] = sys.argv[i+1] + i = i+2 + if not(params.has_key("input") and params.has_key("suite") and params.has_key("conf")): + print "Lack some required parameters." + exit (1) + + result = process(params["input"], params["suite"]) + print "Results:" + for elem in result: + print elem + + if params.has_key("output"): + writeResult(params["output"],result) + + if params.has_key("upload") and params["upload"].lower()=="yes": + uploadResult(params["conf"], params["suite"], result) + +if __name__=="__main__": + main() diff --git a/utils/dashboard/rubbos_collector.py b/utils/dashboard/rubbos_collector.py new file mode 100755 index 00000000..c9851739 --- /dev/null +++ b/utils/dashboard/rubbos_collector.py @@ -0,0 +1,49 @@ +############################################################################## +# Copyright (c) 2015 Huawei Technologies Co.,Ltd. and others +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +import subprocess as subp + +def exec_shell(cmd): + out,err = subp.Popen(cmd, stdout=subp.PIPE, shell=True).communicate() + return out.strip() + +def get_onetime_data(dir_name): + cmd = "grep -in 'remote client nodes' %s/index.html|awk '{print $5}'|awk -F '<' '{print $1}'" % dir_name + client_node_num = int(exec_shell(cmd)) + cmd = "grep -n 'Number of clients' %s/index.html|awk '{print $5}'|awk -F '<' '{print $1}'" % dir_name + each_client_num = int(exec_shell(cmd)) + total_client = (client_node_num+1) * each_client_num + + cmd = 'grep -n "throughput" %s/stat_client*.html |awk -F "" \'{if (FNR%%2==0 && FNR%%4!=0) {printf "%%s\\n", $3}}\'|awk \'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name + throughput = int(exec_shell(cmd)) + + cmd = 'grep -n "Total" %s/stat_client*.html |awk -F "" \'{if (FNR==4) {printf "%%s\\n", $4}}\'|awk -F "" \'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name + request = int(exec_shell(cmd)) + + cmd = 'grep -n "Total" %s/stat_client*.html |awk -F "" \'{if (FNR==4) {printf "%%s\\n", $5}}\'|awk -F "" \'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name + error_request = int(exec_shell(cmd)) + + return total_client, throughput, request, error_request + + +class RubbosCollector(object): + + def __init__(self): + pass + + def collect_data(self, data_home): + cmd = 'ls -l %s |grep ^d|awk \'{print $9}\'' % data_home + result = [] + for subdir in exec_shell(cmd).split('\n'): + total_client, throughput, request, error_request = get_onetime_data(data_home+'/'+subdir) + result.append({'client':total_client, 'throughput':throughput, 'request':request, 'error_request':error_request}) + result.sort(key=lambda x:x['client']) + + return result; -- cgit 1.2.3-korg