From 7d705a294ed7c9fb4931ffd2bc0555dfb35231e1 Mon Sep 17 00:00:00 2001 From: Mofassir Date: Wed, 30 Dec 2015 23:58:38 -0800 Subject: Minor changes to the code, to prep for code freeze. the result format of iperf has been made consistent Change-Id: Iede4b381ca2e5d7643bf6420f17684bbf2cc2921 Signed-off-by: Mofassir --- benchmarks/playbooks/fio.yaml | 18 ++++----- benchmarks/playbooks/iperf.yaml | 44 +++++++++++++++++----- benchmarks/playbooks/netperf.yaml | 4 +- .../result_transform/iperf/iperf_transform.py | 30 +++++++++++++++ 4 files changed, 77 insertions(+), 19 deletions(-) create mode 100644 benchmarks/playbooks/result_transform/iperf/iperf_transform.py (limited to 'benchmarks') diff --git a/benchmarks/playbooks/fio.yaml b/benchmarks/playbooks/fio.yaml index 40fd805b..9f832ce1 100644 --- a/benchmarks/playbooks/fio.yaml +++ b/benchmarks/playbooks/fio.yaml @@ -2,9 +2,9 @@ connection: local tasks: - name: making fio directory - file: path={{Dest_dir}}/fio state=directory + file: path=../../{{Dest_dir}}/fio state=directory - name: making temporary fio directory - file: path={{Dest_dir}}/fio/fio_temp state=directory + file: path=../../{{Dest_dir}}/fio/fio_temp state=directory - hosts: "{{role}}" @@ -54,29 +54,29 @@ shell: (cd $HOME/qtip_result/; find . -maxdepth 1 -name "*.json") | cut -d'/' -f2 register: files_to_copy - name: copy results - fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest={{Dest_dir}}/fio/fio_temp + fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest=../../{{Dest_dir}}/fio/fio_temp with_items: files_to_copy.stdout_lines - name: registering log files shell: (cd $HOME/qtip_result/; find . -maxdepth 1 -name "*.log") | cut -d'/' -f2 register: copy_log_results - name: copying log results - fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest={{Dest_dir}}/fio/fio_temp + fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest=../../{{Dest_dir}}/fio/fio_temp with_items: copy_log_results.stdout_lines - name: copy results - fetch: src=/root/results/{{item}} dest={{Dest_dir}}/fio + fetch: src=/root/results/{{item}} dest=../../{{Dest_dir}}/fio with_items: files_to_copy.stdout_lines - hosts: 127.0.0.1 connection: local tasks: - name: extracting_json - shell: ( find {{Dest_dir}}/fio/fio_temp/ -name "*.json" | xargs cp -t {{Dest_dir}}/fio/) + shell: ( find ../../{{Dest_dir}}/fio/fio_temp/ -name "*.json" | xargs cp -t ../../{{Dest_dir}}/fio/) - name: making_logs_folder - shell: mkdir -p {{Dest_dir}}/fio/logs + shell: mkdir -p ../../{{Dest_dir}}/fio/logs - name: extracting_log - shell: ( find {{Dest_dir}}/fio/fio_temp/ -name "*.log" | xargs cp -t {{Dest_dir}}/fio/logs) + shell: ( find ../../{{Dest_dir}}/fio/fio_temp/ -name "*.log" | xargs cp -t ../../{{Dest_dir}}/fio/logs) - name: removing fio_log - shell: rm -rf {{Dest_dir}}/fio/fio_temp + shell: rm -rf ../../{{Dest_dir}}/fio/fio_temp diff --git a/benchmarks/playbooks/iperf.yaml b/benchmarks/playbooks/iperf.yaml index dd5c2b0f..11d427f3 100644 --- a/benchmarks/playbooks/iperf.yaml +++ b/benchmarks/playbooks/iperf.yaml @@ -59,24 +59,40 @@ poll: 0 when: rolename == "1-server" - name: Running Iperf on Host - shell: iperf3 -Z --{{protocol}} --time {{duration}} -b {{bandwidthGbps}}G -c {{item}} -J >> ./qtip_result/server{{hostID.stdout}}-{{item}}.json - ignore_errors: yes - with_items: - - "{{ip1}}" - - "{{ip2}}" - when: rolename == "2-host" and "{{ip2}}" != '' - - name: Running Iperf on Host - shell: iperf3 -Z --{{protocol}} --time {{duration}} -b {{bandwidthGbps}}G -c {{item}} -J >> ./qtip_result/server{{hostID.stdout}}-{{item}}.json + shell: iperf3 --time {{duration}} -b {{bandwidthGbps}}G -c {{privateip1}} -J -O10>> ./qtip_result/iperf_raw.json ignore_errors: yes with_items: - "{{ip1}}" when: rolename == "2-host" and "{{ip2}}" == '' + - name: Fetching result transformation script + copy: src=./result_transform/iperf/iperf_transform.py dest={{home_dir.stdout}}/qtip_result + - name: Transforming result + shell: cd $HOME/qtip_result && python iperf_transform.py + when: rolename =="2-host" and "{{ip2}}" == '' + - name: copy report formation script + copy: src=./result_transform/final_report.py dest={{home_dir.stdout}}/qtip_result + when: rolename =="2-host" and "{{ip2}}" == '' + - name: consolidating report + shell: cd $HOME/qtip_result && python final_report.py IPERF + when: rolename =="2-host" and "{{ip2}}" == '' - name: Files to Copy shell: (cd $HOME/qtip_result/; find . -maxdepth 1 -name "*.json") | cut -d'/' -f2 register: files_to_copy + when: rolename =="2-host" and "{{ip2}}" == '' - name: copy results fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest=../../{{Dest_dir}}/iperf/iperf_temp with_items: files_to_copy.stdout_lines + when: rolename =="2-host" and "{{ip2}}" == '' + - name: registering log files + shell: (cd $HOME/qtip_result/; find . -maxdepth 1 -name "*.log") | cut -d'/' -f2 + register: copy_log_results + when: rolename =="2-host" and "{{ip2}}" == '' + - name: copying log results + fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest=../../{{Dest_dir}}/iperf/iperf_temp + with_items: copy_log_results.stdout_lines + when: rolename =="2-host" and "{{ip2}}" == '' + + - hosts: 127.0.0.1 connection: local tasks: @@ -86,4 +102,14 @@ when: role is defined - name: extracting_json shell: ( find ../../{{Dest_dir}}/iperf/iperf_temp/ -name "*.json" | xargs cp -t ../../{{Dest_dir}}/iperf/) - when: rolename == "2-host" \ No newline at end of file + when: rolename == "2-host" + - name: making_logs_folder + shell: mkdir -p ../../{{Dest_dir}}/iperf/logs + - name: extracting_log + shell: ( find ../../{{Dest_dir}}/iperf/iperf_temp/ -name "*.log" | xargs cp -t ../../{{Dest_dir}}/iperf/logs) + when: rolename == "2-host" + - name: removing iperf_raw file + shell: rm -rf ../../{{Dest_dir}}/iperf/iperf_raw.json + when: rolename == "2-host" + - name: removing iperf_temp + shell: rm -rf ../../{{Dest_dir}}/iperf/iperf_temp \ No newline at end of file diff --git a/benchmarks/playbooks/netperf.yaml b/benchmarks/playbooks/netperf.yaml index fb6fd18a..3292f060 100644 --- a/benchmarks/playbooks/netperf.yaml +++ b/benchmarks/playbooks/netperf.yaml @@ -71,4 +71,6 @@ when: role is defined - name: extracting_json shell: ( find ../../{{Dest_dir}}/netperf/netperf_temp/ -name "*.json" | xargs cp -t ../../{{Dest_dir}}/netperf/) - when: rolename == "2-host" \ No newline at end of file + when: rolename == "2-host" + - name: removing netperf_temp + shell: rm -rf ../../{{Dest_dir}}/netperf/netperf_temp \ No newline at end of file diff --git a/benchmarks/playbooks/result_transform/iperf/iperf_transform.py b/benchmarks/playbooks/result_transform/iperf/iperf_transform.py new file mode 100644 index 00000000..39c5956c --- /dev/null +++ b/benchmarks/playbooks/result_transform/iperf/iperf_transform.py @@ -0,0 +1,30 @@ +import json +import datetime +import pickle +with open('iperf_raw.json','r') as ifile: + raw_iperf_data=json.loads(ifile.read().rstrip()) + + +bits_sent= raw_iperf_data['end']['sum_sent']['bits_per_second'] +bits_received= raw_iperf_data['end']['sum_received']['bits_per_second'] +total_byte_sent=raw_iperf_data['end']['sum_sent']['bytes'] +total_byte_received=raw_iperf_data['end']['sum_received']['bytes'] +cpu_host_total_percent=raw_iperf_data['end']['cpu_utilization_percent']['host_total'] +cpu_remote_total_percent=raw_iperf_data['end']['cpu_utilization_percent']['remote_total'] + +result={} +time_stamp = str(datetime.datetime.utcnow().isoformat()) + +result['1. Version']=raw_iperf_data['start']['version'] +result['2. Bandwidth']={} +result['2. Bandwidth']['1. throughput Sender (b/s)'] = bits_sent +result['2. Bandwidth']['2. throughput Received (b/s)'] = bits_received +result['3. CPU']={} +result['3. CPU']['1. CPU host total (%)']=cpu_host_total_percent +result['3. CPU']['2. CPU remote total (%)']=cpu_remote_total_percent + +with open('iperf_raw-'+time_stamp+'.log','w+') as ofile: + ofile.write(json.dumps(raw_iperf_data)) + +with open('./result_temp', 'w+') as result_file: + pickle.dump(result,result_file) \ No newline at end of file -- cgit 1.2.3-korg