aboutsummaryrefslogtreecommitdiffstats
path: root/utils/transform
diff options
context:
space:
mode:
authorwu.zhihui <wu.zhihui1@zte.com.cn>2016-10-20 20:31:24 +0800
committerzhihui wu <wu.zhihui1@zte.com.cn>2016-10-21 07:05:29 +0000
commit36f6aa40ca02ef9ef1d24e61af337a960b8a76cd (patch)
tree71d102d0247b1f4f2bbfc236d79ebc27d476d03e /utils/transform
parent7f663c5d49b5cb619d48bb1b8656e9452c860121 (diff)
reorganize playbooks directory
In this patch, playbooks will not be deleted temporarily.Diectory playbooks will be reorganized like this benchmarks/ ├── fio_jobs │   └── test_job ├── perftest    ├── common    │   ├── git_proxy_pbook.yaml    │   ├── sys_info_pbook.yaml    │   └── sys_proxy_pbook.yaml    ├── dhrystone.yaml    ├── dpi.yaml    ├── etc    │   ├── dpi_average.sh    │   ├── info_collect.py    │   └── test_job    ├── fio.yaml    ├── iperf.yaml    ├── ramspeed.yaml    ├── ssl.yaml    └── whetstone.yaml utils/ └── transform ├── dpi_transform.py ├── final_report.py ├── fio_transform.py ├── __init__.py ├── iperf_transform.py ├── ramspeed_transform.py ├── ssl_transform.py └── ubench_transform.py JIRA: QTIP-131 Change-Id: I41003f2f1935efd15b6221ac05200f391fa8a6a9 Signed-off-by: wu.zhihui <wu.zhihui1@zte.com.cn>
Diffstat (limited to 'utils/transform')
-rw-r--r--utils/transform/__init__.py0
-rw-r--r--utils/transform/dpi_transform.py47
-rw-r--r--utils/transform/final_report.py24
-rwxr-xr-xutils/transform/fio_transform.py29
-rw-r--r--utils/transform/iperf_transform.py27
-rw-r--r--utils/transform/ramspeed_transform.py41
-rw-r--r--utils/transform/ssl_transform.py54
-rw-r--r--utils/transform/ubench_transform.py32
8 files changed, 254 insertions, 0 deletions
diff --git a/utils/transform/__init__.py b/utils/transform/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/utils/transform/__init__.py
diff --git a/utils/transform/dpi_transform.py b/utils/transform/dpi_transform.py
new file mode 100644
index 00000000..ee29d8e2
--- /dev/null
+++ b/utils/transform/dpi_transform.py
@@ -0,0 +1,47 @@
+import os
+import pickle
+import datetime
+
+sum_dpi_pps = float(0)
+sum_dpi_bps = float(0)
+
+for x in range(1, 11):
+ dpi_result_pps = float(
+ os.popen(
+ "cat $HOME/qtip_result/dpi_dump.txt | grep 'nDPI throughput:' | awk 'NR=='" +
+ str(x) +
+ " | awk '{print $3}'").read().lstrip())
+ dpi_result_bps = float(
+ os.popen(
+ "cat $HOME/qtip_result/dpi_dump.txt | grep 'nDPI throughput:' | awk 'NR=='" +
+ str(x) +
+ " | awk '{print $7}'").read().rstrip())
+
+ if (dpi_result_pps > 100):
+ dpi_result_pps = dpi_result_pps / 1000
+
+ if (dpi_result_bps > 100):
+ dpi_result_bps = dpi_result_bps / 1000
+
+ sum_dpi_pps += dpi_result_pps
+ sum_dpi_bps += dpi_result_bps
+
+dpi_result_pps = sum_dpi_pps / 10
+dpi_result_bps = sum_dpi_bps / 10
+
+host = os.popen("hostname").read().rstrip()
+log_time_stamp = str(datetime.datetime.utcnow().isoformat())
+
+os.popen(
+ "cat $HOME/qtip_result/dpi_dump.txt > $HOME/qtip_result/" +
+ host +
+ "-" +
+ log_time_stamp +
+ ".log")
+
+home_dir = str(os.popen("echo $HOME").read().rstrip())
+host = os.popen("echo $HOSTNAME")
+result = {'pps': round(dpi_result_pps, 3),
+ 'bps': round(dpi_result_bps, 3)}
+with open('./result_temp', 'w+') as result_file:
+ pickle.dump(result, result_file)
diff --git a/utils/transform/final_report.py b/utils/transform/final_report.py
new file mode 100644
index 00000000..274742d4
--- /dev/null
+++ b/utils/transform/final_report.py
@@ -0,0 +1,24 @@
+import pickle
+import json
+import datetime
+import os
+import sys
+
+home_dir = str((os.popen("echo $HOME").read().rstrip()))
+
+with open('./sys_info_temp', 'r') as sys_info_f:
+ sys_info_dict = pickle.load(sys_info_f)
+with open('./result_temp', 'r') as result_f:
+ result_dict = pickle.load(result_f)
+
+host_name = (os.popen("hostname").read().rstrip())
+benchmark_name = str(sys.argv[1])
+testcase_name = str(sys.argv[2])
+report_time_stamp = str(datetime.datetime.utcnow().isoformat())
+final_dict = {"name": testcase_name,
+ "time": report_time_stamp,
+ "system_information": sys_info_dict,
+ "details": result_dict}
+
+with open('./' + host_name + '-' + report_time_stamp + '.json', 'w+') as result_json:
+ json.dump(final_dict, result_json, indent=4, sort_keys=True)
diff --git a/utils/transform/fio_transform.py b/utils/transform/fio_transform.py
new file mode 100755
index 00000000..5ecac823
--- /dev/null
+++ b/utils/transform/fio_transform.py
@@ -0,0 +1,29 @@
+import json
+import pickle
+import os
+import datetime
+
+
+def get_fio_job_result(fio_job_data):
+ return {'read': {'io_bytes': fio_job_data["read"]["io_bytes"],
+ 'io_ps': fio_job_data["read"]["iops"],
+ 'io_runtime_millisec': fio_job_data["read"]["runtime"],
+ 'mean_io_latenchy_microsec': fio_job_data["read"]["lat"]["mean"]},
+ 'write': {'io_bytes': fio_job_data["write"]["io_bytes"],
+ 'io_ps': fio_job_data["write"]["iops"],
+ 'io_runtime_millisec': fio_job_data["write"]["runtime"],
+ 'mean_io_latenchy_microsec': fio_job_data["write"]["lat"]["mean"]}}
+
+
+with open("fio_result.json") as fio_raw:
+ fio_data = json.load(fio_raw)
+
+fio_result_dict = {}
+for x, result in enumerate(map(get_fio_job_result, fio_data["jobs"])):
+ fio_result_dict['job_{0}'.format(x)] = result
+
+host_name = (os.popen("hostname").read().rstrip())
+report_time = str(datetime.datetime.utcnow().isoformat())
+os.system("mv fio_result.json " + str(host_name) + "-" + report_time + ".log")
+with open('./result_temp', 'w + ')as out_fio_result:
+ pickle.dump(fio_result_dict, out_fio_result)
diff --git a/utils/transform/iperf_transform.py b/utils/transform/iperf_transform.py
new file mode 100644
index 00000000..b52e4634
--- /dev/null
+++ b/utils/transform/iperf_transform.py
@@ -0,0 +1,27 @@
+import json
+import datetime
+import pickle
+with open('iperf_raw.json', 'r') as ifile:
+ raw_iperf_data = json.loads(ifile.read().rstrip())
+
+bits_sent = raw_iperf_data['end']['sum_sent']['bits_per_second']
+bits_received = raw_iperf_data['end']['sum_received']['bits_per_second']
+total_byte_sent = raw_iperf_data['end']['sum_sent']['bytes']
+total_byte_received = raw_iperf_data['end']['sum_received']['bytes']
+cpu_host_total_percent = raw_iperf_data['end']['cpu_utilization_percent']['host_total']
+cpu_remote_total_percent = raw_iperf_data['end']['cpu_utilization_percent']['remote_total']
+
+time_stamp = str(datetime.datetime.utcnow().isoformat())
+
+result = {'version': raw_iperf_data['start']['version'],
+ 'bandwidth': {'sender_throughput': bits_sent,
+ 'received_throughput': bits_received},
+ 'cpu': {'cpu_host': cpu_host_total_percent,
+ 'cpu_remote': cpu_remote_total_percent}
+ }
+
+with open('iperf_raw-' + time_stamp + '.log', 'w+') as ofile:
+ ofile.write(json.dumps(raw_iperf_data))
+
+with open('./result_temp', 'w+') as result_file:
+ pickle.dump(result, result_file)
diff --git a/utils/transform/ramspeed_transform.py b/utils/transform/ramspeed_transform.py
new file mode 100644
index 00000000..960f84fc
--- /dev/null
+++ b/utils/transform/ramspeed_transform.py
@@ -0,0 +1,41 @@
+import os
+import pickle
+import datetime
+
+intmem_copy = os.popen("cat Intmem | grep 'BatchRun Copy' | awk '{print $4}'").read().rstrip()
+intmem_scale = os.popen("cat Intmem | grep 'BatchRun Scale' | awk '{print $4}'").read().rstrip()
+intmem_add = os.popen("cat Intmem | grep 'BatchRun Add' | awk '{print $4}'").read().rstrip()
+intmem_triad = os.popen("cat Intmem | grep 'BatchRun Triad' | awk '{print $4}'").read().rstrip()
+intmem_average = os.popen("cat Intmem | grep 'BatchRun AVERAGE' | awk '{print $4}'").read().rstrip()
+
+print intmem_copy
+print intmem_average
+
+floatmem_copy = os.popen("cat Floatmem | grep 'BatchRun Copy' | awk '{print $4}'").read().rstrip()
+floatmem_scale = os.popen("cat Floatmem | grep 'BatchRun Scale' | awk '{print $4}'").read().rstrip()
+floatmem_add = os.popen("cat Floatmem | grep 'BatchRun Add' | awk '{print $4}'").read().rstrip()
+floatmem_triad = os.popen("cat Floatmem | grep 'BatchRun Triad' | awk '{print $4}'").read().rstrip()
+floatmem_average = os.popen("cat Floatmem | grep 'BatchRun AVERAGE' | awk '{print $4}'").read().rstrip()
+
+print floatmem_copy
+print floatmem_average
+
+hostname = os.popen("hostname").read().rstrip()
+time_stamp = str(datetime.datetime.utcnow().isoformat())
+
+os.system("mv Intmem " + hostname + "-" + time_stamp + ".log")
+os.system("cp Floatmem >> " + hostname + "-" + time_stamp + ".log")
+
+result = {"int_bandwidth": {"copy": intmem_copy,
+ "add": intmem_add,
+ "scale": intmem_scale,
+ "triad": intmem_triad,
+ "average": intmem_average},
+ "float_bandwidth": {"copy": floatmem_copy,
+ "add": floatmem_add,
+ "scale": floatmem_scale,
+ "triad": floatmem_triad,
+ "average": floatmem_average}}
+
+with open('./result_temp', 'w+') as result_file:
+ pickle.dump(result, result_file)
diff --git a/utils/transform/ssl_transform.py b/utils/transform/ssl_transform.py
new file mode 100644
index 00000000..de84d24b
--- /dev/null
+++ b/utils/transform/ssl_transform.py
@@ -0,0 +1,54 @@
+import os
+import pickle
+import datetime
+
+openssl_version = os.popen("cat RSA_dump | head -1").read().rstrip()
+rsa_512_sps = os.popen(
+ "cat RSA_dump | grep '512 bits ' | awk '{print $6}' ").read().rstrip()
+rsa_512_vps = os.popen(
+ "cat RSA_dump | grep '512 bits ' | awk '{print $7}' ").read().rstrip()
+rsa_1024_sps = os.popen(
+ "cat RSA_dump | grep '1024 bits ' | awk '{print $6}' ").read().rstrip()
+rsa_1024_vps = os.popen(
+ "cat RSA_dump | grep '1024 bits ' | awk '{print $7}' ").read().rstrip()
+rsa_2048_sps = os.popen(
+ "cat RSA_dump | grep '2048 bits ' | awk '{print $6}' ").read().rstrip()
+rsa_2048_vps = os.popen(
+ "cat RSA_dump | grep '2048 bits ' | awk '{print $7}' ").read().rstrip()
+rsa_4096_sps = os.popen(
+ "cat RSA_dump | grep '4096 bits ' | awk '{print $6}' ").read().rstrip()
+rsa_4096_vps = os.popen(
+ "cat RSA_dump | grep '4096 bits ' | awk '{print $7}' ").read().rstrip()
+
+aes_16B = os.popen(
+ "cat AES-128-CBC_dump | grep 'aes-128-cbc ' | awk '{print $2}' ").read().rstrip()
+aes_64B = os.popen(
+ "cat AES-128-CBC_dump | grep 'aes-128-cbc ' | awk '{print $3}' ").read().rstrip()
+aes_256B = os.popen(
+ "cat AES-128-CBC_dump | grep 'aes-128-cbc ' | awk '{print $4}' ").read().rstrip()
+aes_1024B = os.popen(
+ "cat AES-128-CBC_dump | grep 'aes-128-cbc ' | awk '{print $5}' ").read().rstrip()
+aes_8192B = os.popen(
+ "cat AES-128-CBC_dump | grep 'aes-128-cbc ' | awk '{print $6}' ").read().rstrip()
+
+hostname = os.popen("hostname").read().rstrip()
+time_stamp = str(datetime.datetime.utcnow().isoformat())
+
+os.system("mv RSA_dump " + hostname + "-" + time_stamp + ".log")
+os.system("cat AES-128-CBC_dump >> " + hostname + "-" + time_stamp + ".log")
+
+result = {"version": [openssl_version],
+ "rsa_sig": {"512_bits": rsa_512_sps,
+ "1024_bits": rsa_1024_sps,
+ "2048_bits": rsa_2048_sps,
+ "4096_bits": rsa_4096_sps,
+ "unit": "sig/sec"},
+ "aes_128_cbc": {"16B_block": aes_16B,
+ "64B_block": aes_64B,
+ "256B_block": aes_256B,
+ "1024B_block": aes_1024B,
+ "8192B_block": aes_8192B,
+ "unit": "B/sec"}}
+
+with open('./result_temp', 'w+') as result_file:
+ pickle.dump(result, result_file)
diff --git a/utils/transform/ubench_transform.py b/utils/transform/ubench_transform.py
new file mode 100644
index 00000000..ab5fe171
--- /dev/null
+++ b/utils/transform/ubench_transform.py
@@ -0,0 +1,32 @@
+import os
+import json
+import pickle
+
+total_cpu = os.popen(
+ "cat $HOME/tempT/UnixBench/results/* | grep 'of tests' | awk '{print $1;}' | awk 'NR==1'").read().rstrip()
+
+cpu_1 = os.popen(
+ "cat $HOME/tempT/UnixBench/results/* | grep 'of tests' | awk '{print $6;}' | awk 'NR==1'").read().rstrip()
+
+
+cpu_2 = os.popen(
+ "cat $HOME/tempT/UnixBench/results/* | grep 'of tests' | awk '{print $6;}' | awk 'NR==2'").read().rstrip()
+
+
+index_1 = os.popen(
+ "cat $HOME/tempT/UnixBench/results/* | grep 'Index Score (Partial Only) ' | awk '{print $7;}' | awk 'NR==1'").read().rstrip()
+index_2 = os.popen(
+ "cat $HOME/tempT/UnixBench/results/* | grep 'Index Score (Partial Only) ' | awk '{print $7;}' | awk 'NR==2'").read().rstrip()
+
+
+result = {"n_cpu": total_cpu,
+ "single": {"n_para_test": cpu_1,
+ "score": index_1},
+ "multi": {"n_para_test": cpu_2,
+ "score": index_2}
+ }
+
+with open('result_temp', 'w+') as result_file:
+ pickle.dump(result, result_file)
+print json.dumps(result, indent=4, sort_keys=True)
+# print result.items()