reorganize playbooks directory

In this patch, playbooks will not be deleted temporarily.Diectory 
playbooks will be reorganized like this

benchmarks/
├── fio_jobs
│   └── test_job
├── perftest
    ├── common
    │   ├── git_proxy_pbook.yaml
    │   ├── sys_info_pbook.yaml
    │   └── sys_proxy_pbook.yaml
    ├── dhrystone.yaml
    ├── dpi.yaml
    ├── etc
    │   ├── dpi_average.sh
    │   ├── info_collect.py
    │   └── test_job
    ├── fio.yaml
    ├── iperf.yaml
    ├── ramspeed.yaml
    ├── ssl.yaml
    └── whetstone.yaml

utils/
└── transform
    ├── dpi_transform.py
    ├── final_report.py
    ├── fio_transform.py
    ├── __init__.py
    ├── iperf_transform.py
    ├── ramspeed_transform.py
    ├── ssl_transform.py
    └── ubench_transform.py

JIRA: QTIP-131

Change-Id: I41003f2f1935efd15b6221ac05200f391fa8a6a9
Signed-off-by: wu.zhihui <wu.zhihui1@zte.com.cn>

8 files changed, 254 insertions, 0 deletions

diff --git a/utils/transform/__init__.py b/utils/transform/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/utils/transform/__init__.py
diff --git a/utils/transform/dpi_transform.py b/utils/transform/dpi_transform.py
new file mode 100644
index 00000000..ee29d8e2
--- /dev/null
+++ b/utils/transform/dpi_transform.py
@@ -0,0 +1,47 @@
+import os
+import pickle
+import datetime
+
+sum_dpi_pps = float(0)
+sum_dpi_bps = float(0)
+
+for x in range(1, 11):
+    dpi_result_pps = float(
+        os.popen(
+            "cat $HOME/qtip_result/dpi_dump.txt | grep 'nDPI throughput:' | awk 'NR=='" +
+            str(x) +
+            " | awk '{print $3}'").read().lstrip())
+    dpi_result_bps = float(
+        os.popen(
+            "cat $HOME/qtip_result/dpi_dump.txt | grep 'nDPI throughput:' | awk 'NR=='" +
+            str(x) +
+            " | awk '{print $7}'").read().rstrip())
+
+    if (dpi_result_pps > 100):
+        dpi_result_pps = dpi_result_pps / 1000
+
+    if (dpi_result_bps > 100):
+        dpi_result_bps = dpi_result_bps / 1000
+
+    sum_dpi_pps += dpi_result_pps
+    sum_dpi_bps += dpi_result_bps
+
+dpi_result_pps = sum_dpi_pps / 10
+dpi_result_bps = sum_dpi_bps / 10
+
+host = os.popen("hostname").read().rstrip()
+log_time_stamp = str(datetime.datetime.utcnow().isoformat())
+
+os.popen(
+    "cat $HOME/qtip_result/dpi_dump.txt > $HOME/qtip_result/" +
+    host +
+    "-" +
+    log_time_stamp +
+    ".log")
+
+home_dir = str(os.popen("echo $HOME").read().rstrip())
+host = os.popen("echo $HOSTNAME")
+result = {'pps': round(dpi_result_pps, 3),
+          'bps': round(dpi_result_bps, 3)}
+with open('./result_temp', 'w+') as result_file:
+    pickle.dump(result, result_file)
diff --git a/utils/transform/final_report.py b/utils/transform/final_report.py
new file mode 100644
index 00000000..274742d4
--- /dev/null
+++ b/utils/transform/final_report.py
@@ -0,0 +1,24 @@
+import pickle
+import json
+import datetime
+import os
+import sys
+
+home_dir = str((os.popen("echo $HOME").read().rstrip()))
+
+with open('./sys_info_temp', 'r') as sys_info_f:
+    sys_info_dict = pickle.load(sys_info_f)
+with open('./result_temp', 'r') as result_f:
+    result_dict = pickle.load(result_f)
+
+host_name = (os.popen("hostname").read().rstrip())
+benchmark_name = str(sys.argv[1])
+testcase_name = str(sys.argv[2])
+report_time_stamp = str(datetime.datetime.utcnow().isoformat())
+final_dict = {"name": testcase_name,
+              "time": report_time_stamp,
+              "system_information": sys_info_dict,
+              "details": result_dict}
+
+with open('./' + host_name + '-' + report_time_stamp + '.json', 'w+') as result_json:
+    json.dump(final_dict, result_json, indent=4, sort_keys=True)
diff --git a/utils/transform/fio_transform.py b/utils/transform/fio_transform.py
new file mode 100755
index 00000000..5ecac823
--- /dev/null
+++ b/utils/transform/fio_transform.py
@@ -0,0 +1,29 @@
+import json
+import pickle
+import os
+import datetime
+
+
+def get_fio_job_result(fio_job_data):
+    return {'read': {'io_bytes': fio_job_data["read"]["io_bytes"],
+                     'io_ps': fio_job_data["read"]["iops"],
+                     'io_runtime_millisec': fio_job_data["read"]["runtime"],
+                     'mean_io_latenchy_microsec': fio_job_data["read"]["lat"]["mean"]},
+            'write': {'io_bytes': fio_job_data["write"]["io_bytes"],
+                      'io_ps': fio_job_data["write"]["iops"],
+                      'io_runtime_millisec': fio_job_data["write"]["runtime"],
+                      'mean_io_latenchy_microsec': fio_job_data["write"]["lat"]["mean"]}}
+
+
+with open("fio_result.json") as fio_raw:
+    fio_data = json.load(fio_raw)
+
+fio_result_dict = {}
+for x, result in enumerate(map(get_fio_job_result, fio_data["jobs"])):
+    fio_result_dict['job_{0}'.format(x)] = result
+
+host_name = (os.popen("hostname").read().rstrip())
+report_time = str(datetime.datetime.utcnow().isoformat())
+os.system("mv fio_result.json " + str(host_name) + "-" + report_time + ".log")
+with open('./result_temp', 'w + ')as out_fio_result:
+    pickle.dump(fio_result_dict, out_fio_result)
diff --git a/utils/transform/iperf_transform.py b/utils/transform/iperf_transform.py
new file mode 100644
index 00000000..b52e4634
--- /dev/null
+++ b/utils/transform/iperf_transform.py
@@ -0,0 +1,27 @@
+import json

+import datetime

+import pickle

+with open('iperf_raw.json', 'r') as ifile:

+    raw_iperf_data = json.loads(ifile.read().rstrip())

+

+bits_sent = raw_iperf_data['end']['sum_sent']['bits_per_second']

+bits_received = raw_iperf_data['end']['sum_received']['bits_per_second']

+total_byte_sent = raw_iperf_data['end']['sum_sent']['bytes']

+total_byte_received = raw_iperf_data['end']['sum_received']['bytes']

+cpu_host_total_percent = raw_iperf_data['end']['cpu_utilization_percent']['host_total']

+cpu_remote_total_percent = raw_iperf_data['end']['cpu_utilization_percent']['remote_total']

+

+time_stamp = str(datetime.datetime.utcnow().isoformat())

+

+result = {'version': raw_iperf_data['start']['version'],

+          'bandwidth': {'sender_throughput': bits_sent,

+                        'received_throughput': bits_received},

+          'cpu': {'cpu_host': cpu_host_total_percent,

+                  'cpu_remote': cpu_remote_total_percent}

+          }

+

+with open('iperf_raw-' + time_stamp + '.log', 'w+') as ofile:

+    ofile.write(json.dumps(raw_iperf_data))

+

+with open('./result_temp', 'w+') as result_file:

+    pickle.dump(result, result_file)

diff --git a/utils/transform/ramspeed_transform.py b/utils/transform/ramspeed_transform.py
new file mode 100644
index 00000000..960f84fc
--- /dev/null
+++ b/utils/transform/ramspeed_transform.py
@@ -0,0 +1,41 @@
+import os
+import pickle
+import datetime
+
+intmem_copy = os.popen("cat Intmem | grep 'BatchRun   Copy' | awk '{print $4}'").read().rstrip()
+intmem_scale = os.popen("cat Intmem | grep 'BatchRun   Scale' | awk '{print $4}'").read().rstrip()
+intmem_add = os.popen("cat Intmem | grep 'BatchRun   Add' | awk '{print $4}'").read().rstrip()
+intmem_triad = os.popen("cat Intmem | grep 'BatchRun   Triad' | awk '{print $4}'").read().rstrip()
+intmem_average = os.popen("cat Intmem | grep 'BatchRun   AVERAGE' | awk '{print $4}'").read().rstrip()
+
+print intmem_copy
+print intmem_average
+
+floatmem_copy = os.popen("cat Floatmem | grep 'BatchRun  Copy' | awk '{print $4}'").read().rstrip()
+floatmem_scale = os.popen("cat Floatmem | grep 'BatchRun  Scale' | awk '{print $4}'").read().rstrip()
+floatmem_add = os.popen("cat Floatmem | grep 'BatchRun  Add' | awk '{print $4}'").read().rstrip()
+floatmem_triad = os.popen("cat Floatmem | grep 'BatchRun  Triad' | awk '{print $4}'").read().rstrip()
+floatmem_average = os.popen("cat Floatmem | grep 'BatchRun  AVERAGE' | awk '{print $4}'").read().rstrip()
+
+print floatmem_copy
+print floatmem_average
+
+hostname = os.popen("hostname").read().rstrip()
+time_stamp = str(datetime.datetime.utcnow().isoformat())
+
+os.system("mv Intmem " + hostname + "-" + time_stamp + ".log")
+os.system("cp  Floatmem >> " + hostname + "-" + time_stamp + ".log")
+
+result = {"int_bandwidth": {"copy": intmem_copy,
+                            "add": intmem_add,
+                            "scale": intmem_scale,
+                            "triad": intmem_triad,
+                            "average": intmem_average},
+          "float_bandwidth": {"copy": floatmem_copy,
+                              "add": floatmem_add,
+                              "scale": floatmem_scale,
+                              "triad": floatmem_triad,
+                              "average": floatmem_average}}
+
+with open('./result_temp', 'w+') as result_file:
+    pickle.dump(result, result_file)
diff --git a/utils/transform/ssl_transform.py b/utils/transform/ssl_transform.py
new file mode 100644
index 00000000..de84d24b
--- /dev/null
+++ b/utils/transform/ssl_transform.py
@@ -0,0 +1,54 @@
+import os
+import pickle
+import datetime
+
+openssl_version = os.popen("cat RSA_dump | head -1").read().rstrip()
+rsa_512_sps = os.popen(
+    "cat RSA_dump | grep  '512 bits ' | awk '{print $6}' ").read().rstrip()
+rsa_512_vps = os.popen(
+    "cat RSA_dump | grep  '512 bits ' | awk '{print $7}' ").read().rstrip()
+rsa_1024_sps = os.popen(
+    "cat RSA_dump | grep  '1024 bits ' | awk '{print $6}' ").read().rstrip()
+rsa_1024_vps = os.popen(
+    "cat RSA_dump | grep  '1024 bits ' | awk '{print $7}' ").read().rstrip()
+rsa_2048_sps = os.popen(
+    "cat RSA_dump | grep  '2048 bits ' | awk '{print $6}' ").read().rstrip()
+rsa_2048_vps = os.popen(
+    "cat RSA_dump | grep  '2048 bits ' | awk '{print $7}' ").read().rstrip()
+rsa_4096_sps = os.popen(
+    "cat RSA_dump | grep  '4096 bits ' | awk '{print $6}' ").read().rstrip()
+rsa_4096_vps = os.popen(
+    "cat RSA_dump | grep  '4096 bits ' | awk '{print $7}' ").read().rstrip()
+
+aes_16B = os.popen(
+    "cat AES-128-CBC_dump | grep  'aes-128-cbc  ' | awk '{print $2}' ").read().rstrip()
+aes_64B = os.popen(
+    "cat AES-128-CBC_dump | grep  'aes-128-cbc  ' | awk '{print $3}' ").read().rstrip()
+aes_256B = os.popen(
+    "cat AES-128-CBC_dump | grep  'aes-128-cbc  ' | awk '{print $4}' ").read().rstrip()
+aes_1024B = os.popen(
+    "cat AES-128-CBC_dump | grep  'aes-128-cbc  ' | awk '{print $5}' ").read().rstrip()
+aes_8192B = os.popen(
+    "cat AES-128-CBC_dump | grep  'aes-128-cbc  ' | awk '{print $6}' ").read().rstrip()
+
+hostname = os.popen("hostname").read().rstrip()
+time_stamp = str(datetime.datetime.utcnow().isoformat())
+
+os.system("mv RSA_dump " + hostname + "-" + time_stamp + ".log")
+os.system("cat AES-128-CBC_dump >> " + hostname + "-" + time_stamp + ".log")
+
+result = {"version": [openssl_version],
+          "rsa_sig": {"512_bits": rsa_512_sps,
+                      "1024_bits": rsa_1024_sps,
+                      "2048_bits": rsa_2048_sps,
+                      "4096_bits": rsa_4096_sps,
+                      "unit": "sig/sec"},
+          "aes_128_cbc": {"16B_block": aes_16B,
+                          "64B_block": aes_64B,
+                          "256B_block": aes_256B,
+                          "1024B_block": aes_1024B,
+                          "8192B_block": aes_8192B,
+                          "unit": "B/sec"}}
+
+with open('./result_temp', 'w+') as result_file:
+    pickle.dump(result, result_file)
diff --git a/utils/transform/ubench_transform.py b/utils/transform/ubench_transform.py
new file mode 100644
index 00000000..ab5fe171
--- /dev/null
+++ b/utils/transform/ubench_transform.py
@@ -0,0 +1,32 @@
+import os
+import json
+import pickle
+
+total_cpu = os.popen(
+    "cat $HOME/tempT/UnixBench/results/* | grep 'of tests' | awk '{print $1;}' | awk 'NR==1'").read().rstrip()
+
+cpu_1 = os.popen(
+    "cat $HOME/tempT/UnixBench/results/* | grep 'of tests' | awk '{print $6;}' | awk 'NR==1'").read().rstrip()
+
+
+cpu_2 = os.popen(
+    "cat $HOME/tempT/UnixBench/results/* | grep 'of tests' | awk '{print $6;}' | awk 'NR==2'").read().rstrip()
+
+
+index_1 = os.popen(
+    "cat $HOME/tempT/UnixBench/results/* | grep 'Index Score (Partial Only)  ' | awk '{print $7;}' | awk 'NR==1'").read().rstrip()
+index_2 = os.popen(
+    "cat $HOME/tempT/UnixBench/results/* | grep 'Index Score (Partial Only)  ' | awk '{print $7;}' | awk 'NR==2'").read().rstrip()
+
+
+result = {"n_cpu": total_cpu,
+          "single": {"n_para_test": cpu_1,
+                     "score": index_1},
+          "multi": {"n_para_test": cpu_2,
+                    "score": index_2}
+          }
+
+with open('result_temp', 'w+') as result_file:
+    pickle.dump(result, result_file)
+print json.dumps(result, indent=4, sort_keys=True)
+# print result.items()