Minor changes to the code, to prep for code freeze.

the result format of iperf has been made consistent

Change-Id: Iede4b381ca2e5d7643bf6420f17684bbf2cc2921
Signed-off-by: Mofassir <Mofassir_arif@Dellteam.com>

4 files changed, 77 insertions, 19 deletions

diff --git a/benchmarks/playbooks/fio.yaml b/benchmarks/playbooks/fio.yaml
index 40fd805b..9f832ce1 100644
--- a/benchmarks/playbooks/fio.yaml
+++ b/benchmarks/playbooks/fio.yaml
@@ -2,9 +2,9 @@
    connection: local
    tasks:
     - name: making fio  directory
-      file: path={{Dest_dir}}/fio state=directory
+      file: path=../../{{Dest_dir}}/fio state=directory
     - name: making temporary fio directory
-      file: path={{Dest_dir}}/fio/fio_temp state=directory
+      file: path=../../{{Dest_dir}}/fio/fio_temp state=directory
 
 
  - hosts: "{{role}}"
@@ -54,29 +54,29 @@
       shell: (cd $HOME/qtip_result/; find . -maxdepth 1 -name "*.json") | cut -d'/' -f2
       register: files_to_copy
     - name: copy results
-      fetch:  src={{home_dir.stdout}}/qtip_result/{{item}} dest={{Dest_dir}}/fio/fio_temp
+      fetch:  src={{home_dir.stdout}}/qtip_result/{{item}} dest=../../{{Dest_dir}}/fio/fio_temp
       with_items: files_to_copy.stdout_lines
     - name: registering log files
       shell: (cd $HOME/qtip_result/; find . -maxdepth 1 -name "*.log") | cut -d'/' -f2
       register: copy_log_results
     - name: copying log results
-      fetch:  src={{home_dir.stdout}}/qtip_result/{{item}} dest={{Dest_dir}}/fio/fio_temp
+      fetch:  src={{home_dir.stdout}}/qtip_result/{{item}} dest=../../{{Dest_dir}}/fio/fio_temp
       with_items: copy_log_results.stdout_lines
 
 
     - name: copy results
-      fetch:  src=/root/results/{{item}}  dest={{Dest_dir}}/fio
+      fetch:  src=/root/results/{{item}}  dest=../../{{Dest_dir}}/fio
       with_items: files_to_copy.stdout_lines
 
  - hosts: 127.0.0.1
    connection: local
    tasks:
     - name: extracting_json
-      shell: ( find  {{Dest_dir}}/fio/fio_temp/ -name "*.json" | xargs cp -t {{Dest_dir}}/fio/)
+      shell: ( find  ../../{{Dest_dir}}/fio/fio_temp/ -name "*.json" | xargs cp -t ../../{{Dest_dir}}/fio/)
     - name: making_logs_folder
-      shell: mkdir -p {{Dest_dir}}/fio/logs
+      shell: mkdir -p ../../{{Dest_dir}}/fio/logs
     - name: extracting_log
-      shell: ( find {{Dest_dir}}/fio/fio_temp/ -name "*.log"  | xargs cp -t {{Dest_dir}}/fio/logs)
+      shell: ( find ../../{{Dest_dir}}/fio/fio_temp/ -name "*.log"  | xargs cp -t ../../{{Dest_dir}}/fio/logs)
     - name: removing fio_log
-      shell: rm -rf {{Dest_dir}}/fio/fio_temp
+      shell: rm -rf ../../{{Dest_dir}}/fio/fio_temp
 
diff --git a/benchmarks/playbooks/iperf.yaml b/benchmarks/playbooks/iperf.yaml
index dd5c2b0f..11d427f3 100644
--- a/benchmarks/playbooks/iperf.yaml
+++ b/benchmarks/playbooks/iperf.yaml
@@ -59,24 +59,40 @@
       poll: 0
       when: rolename == "1-server"
     - name: Running Iperf on Host
-      shell: iperf3  -Z --{{protocol}} --time {{duration}} -b {{bandwidthGbps}}G  -c {{item}} -J >> ./qtip_result/server{{hostID.stdout}}-{{item}}.json
-      ignore_errors: yes
-      with_items:
-        - "{{ip1}}"
-        - "{{ip2}}"
-      when: rolename == "2-host" and "{{ip2}}" != ''
-    - name: Running Iperf on Host
-      shell: iperf3  -Z --{{protocol}} --time {{duration}} -b {{bandwidthGbps}}G -c {{item}} -J >> ./qtip_result/server{{hostID.stdout}}-{{item}}.json
+      shell: iperf3  --time {{duration}} -b {{bandwidthGbps}}G -c {{privateip1}} -J -O10>> ./qtip_result/iperf_raw.json
       ignore_errors: yes
       with_items: 
         - "{{ip1}}"
       when: rolename == "2-host" and "{{ip2}}" == ''
+    - name: Fetching result transformation script
+      copy: src=./result_transform/iperf/iperf_transform.py dest={{home_dir.stdout}}/qtip_result
+    - name: Transforming result
+      shell: cd $HOME/qtip_result  &&  python iperf_transform.py
+      when: rolename =="2-host" and "{{ip2}}" == ''
+    - name: copy report formation script
+      copy: src=./result_transform/final_report.py dest={{home_dir.stdout}}/qtip_result
+      when: rolename =="2-host" and "{{ip2}}" == ''
+    - name: consolidating report
+      shell: cd $HOME/qtip_result &&  python final_report.py IPERF      
+      when: rolename =="2-host" and "{{ip2}}" == ''
     - name: Files to Copy  
       shell: (cd $HOME/qtip_result/; find . -maxdepth 1 -name "*.json") | cut -d'/' -f2
       register: files_to_copy
+      when: rolename =="2-host" and "{{ip2}}" == ''
     - name: copy results
       fetch:  src={{home_dir.stdout}}/qtip_result/{{item}} dest=../../{{Dest_dir}}/iperf/iperf_temp
       with_items: files_to_copy.stdout_lines
+      when: rolename =="2-host" and "{{ip2}}" == ''
+    - name: registering log files
+      shell: (cd $HOME/qtip_result/; find . -maxdepth 1 -name "*.log") | cut -d'/' -f2
+      register: copy_log_results
+      when: rolename =="2-host" and "{{ip2}}" == ''
+    - name: copying log results
+      fetch:  src={{home_dir.stdout}}/qtip_result/{{item}} dest=../../{{Dest_dir}}/iperf/iperf_temp
+      with_items: copy_log_results.stdout_lines
+      when: rolename =="2-host" and "{{ip2}}" == ''
+
+
  - hosts: 127.0.0.1
    connection: local
    tasks:
@@ -86,4 +102,14 @@
         when: role is defined
     - name: extracting_json
       shell: ( find  ../../{{Dest_dir}}/iperf/iperf_temp/ -name "*.json" | xargs cp -t ../../{{Dest_dir}}/iperf/)
-      when: rolename == "2-host"
\ No newline at end of file
+      when: rolename == "2-host"
+    - name: making_logs_folder
+      shell: mkdir -p ../../{{Dest_dir}}/iperf/logs
+    - name: extracting_log
+      shell: ( find ../../{{Dest_dir}}/iperf/iperf_temp/ -name "*.log"  | xargs cp -t ../../{{Dest_dir}}/iperf/logs)
+      when: rolename == "2-host"
+    - name: removing iperf_raw file
+      shell: rm -rf ../../{{Dest_dir}}/iperf/iperf_raw.json
+      when: rolename == "2-host"
+    - name: removing iperf_temp
+      shell: rm -rf ../../{{Dest_dir}}/iperf/iperf_temp
\ No newline at end of file
diff --git a/benchmarks/playbooks/netperf.yaml b/benchmarks/playbooks/netperf.yaml
index fb6fd18a..3292f060 100644
--- a/benchmarks/playbooks/netperf.yaml
+++ b/benchmarks/playbooks/netperf.yaml
@@ -71,4 +71,6 @@
         when: role is defined
     - name: extracting_json
       shell: ( find  ../../{{Dest_dir}}/netperf/netperf_temp/ -name "*.json" | xargs cp -t ../../{{Dest_dir}}/netperf/)
-      when: rolename == "2-host"
\ No newline at end of file
+      when: rolename == "2-host"
+    - name: removing netperf_temp
+      shell: rm -rf ../../{{Dest_dir}}/netperf/netperf_temp
\ No newline at end of file
diff --git a/benchmarks/playbooks/result_transform/iperf/iperf_transform.py b/benchmarks/playbooks/result_transform/iperf/iperf_transform.py
new file mode 100644
index 00000000..39c5956c
--- /dev/null
+++ b/benchmarks/playbooks/result_transform/iperf/iperf_transform.py
@@ -0,0 +1,30 @@
+import json

+import datetime

+import pickle

+with open('iperf_raw.json','r') as ifile:

+    raw_iperf_data=json.loads(ifile.read().rstrip())

+    

+    

+bits_sent= raw_iperf_data['end']['sum_sent']['bits_per_second']

+bits_received= raw_iperf_data['end']['sum_received']['bits_per_second']

+total_byte_sent=raw_iperf_data['end']['sum_sent']['bytes']

+total_byte_received=raw_iperf_data['end']['sum_received']['bytes']

+cpu_host_total_percent=raw_iperf_data['end']['cpu_utilization_percent']['host_total']

+cpu_remote_total_percent=raw_iperf_data['end']['cpu_utilization_percent']['remote_total']

+

+result={}

+time_stamp = str(datetime.datetime.utcnow().isoformat())

+

+result['1. Version']=raw_iperf_data['start']['version']

+result['2. Bandwidth']={}

+result['2. Bandwidth']['1. throughput Sender (b/s)'] = bits_sent

+result['2. Bandwidth']['2. throughput Received (b/s)'] = bits_received

+result['3. CPU']={}

+result['3. CPU']['1. CPU host total (%)']=cpu_host_total_percent

+result['3. CPU']['2. CPU remote total (%)']=cpu_remote_total_percent

+

+with open('iperf_raw-'+time_stamp+'.log','w+') as ofile:

+    ofile.write(json.dumps(raw_iperf_data))

+    

+with open('./result_temp', 'w+') as result_file:

+    pickle.dump(result,result_file)
\ No newline at end of file