diff options
28 files changed, 2004 insertions, 42 deletions
diff --git a/docs/release/results/euphrates_fraser_comparison.rst b/docs/release/results/euphrates_fraser_comparison.rst new file mode 100644 index 000000000..53dfb994f --- /dev/null +++ b/docs/release/results/euphrates_fraser_comparison.rst @@ -0,0 +1,602 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International +.. License. +.. http://creativecommons.org/licenses/by/4.0 + +======================================================= +Test results analysis for Euphrates and Fraser releases +======================================================= + +TC002 +----- + +The round-trip-time (RTT) between 2 VMs on different blades is measured using +ping. + +Most test run measurements result on average between 0.39 and 4.00 ms. +Compared with Euphrates release, the average RTT result of the same pod experiences +a slight decline in Fraser release. For example, the average RTT of arm-pod5 is +1.518 in Ehphrates and 1.714 in Fraser. The average RTT of intel-pod18 is 1.6575 +ms in Ehphrates and 1.856 ms in Fraser. + +{ + + "huawei-pod2:stable/euphrates": [0.3925], + + "lf-pod2:stable/euphrates": [0.5315], + + "lf-pod1:stable/euphrates": [0.62], + + "huawei-pod2:stable/fraser": [0.677], + + "lf-pod1:stable/fraser": [0.725], + + "flex-pod2:stable/euphrates": [0.795], + + "huawei-pod12:stable/euphrates": [0.87], + + "ericsson-pod1:stable/fraser": [0.9165], + + "huawei-pod12:stable/fraser": [1.0465], + + "lf-pod2:stable/fraser": [1.2325], + + "intel-pod5:stable/euphrates": [1.25], + + "ericsson-virtual3:stable/euphrates": [1.2655], + + "ericsson-pod1:stable/euphrates": [1.372], + + "zte-pod2:stable/fraser": [1.395], + + "arm-pod5:stable/euphrates": [1.518], + + "huawei-virtual4:stable/euphrates": [1.5355], + + "ericsson-virtual4:stable/fraser": [1.582], + + "huawei-virtual3:stable/euphrates": [1.606], + + "intel-pod18:stable/euphrates": [1.6575], + + "huawei-virtual4:stable/fraser": [1.697], + + "huawei-virtual8:stable/euphrates": [1.709], + + "arm-pod5:stable/fraser": [1.714], + + "huawei-virtual3:stable/fraser": [1.716], + + "intel-pod18:stable/fraser": [1.856], + + "huawei-virtual2:stable/euphrates": [1.872], + + "arm-pod6:stable/euphrates": [1.895], + + "huawei-virtual2:stable/fraser": [1.964], + + "huawei-virtual1:stable/fraser": [1.9765], + + "huawei-virtual9:stable/euphrates": [2.0745], + + "arm-pod6:stable/fraser": [2.209], + + "huawei-virtual1:stable/euphrates": [2.495], + + "ericsson-virtual2:stable/euphrates": [2.7895], + + "ericsson-virtual4:stable/euphrates": [3.768], + + "ericsson-virtual1:stable/euphrates": [3.8035], + + "ericsson-virtual3:stable/fraser": [3.9175], + + "ericsson-virtual2:stable/fraser": [4.004] + +} + +TC010 +----- + +The tool we use to measure memory read latency is lmbench, which is a series of +micro benchmarks intended to measure basic operating system and hardware system +metrics. Compared with Euphrates release, the memory read latency of the same pod +also experience a slight decline. Virtual pods seem to have a higher memory read +latency than physical pods. Compared with X86 pods, the memory read latency of +arm pods is significant higher. + +{ + + "ericsson-pod1:stable/euphrates": [5.7785], + + "flex-pod2:stable/euphrates": [5.908], + + "ericsson-virtual1:stable/euphrates": [6.412], + + "intel-pod18:stable/euphrates": [6.5905], + + "intel-pod5:stable/euphrates": [6.6975], + + "ericsson-pod1:stable/fraser": [7.0645], + + "ericsson-virtual4:stable/euphrates": [7.183], + + "intel-pod18:stable/fraser": [7.4465], + + "zte-pod2:stable/fraser": [8.1865], + + "ericsson-virtual2:stable/euphrates": [8.4985], + + "huawei-pod2:stable/euphrates": [8.877], + + "huawei-pod12:stable/euphrates": [9.091], + + "huawei-pod2:stable/fraser": [9.236], + + "huawei-pod12:stable/fraser": [9.615], + + "ericsson-virtual3:stable/euphrates": [9.719], + + "ericsson-virtual2:stable/fraser": [9.8925], + + "huawei-virtual4:stable/euphrates": [10.1195], + + "huawei-virtual3:stable/euphrates": [10.19], + + "huawei-virtual2:stable/fraser": [10.22], + + "huawei-virtual1:stable/euphrates": [10.3045], + + "huawei-virtual9:stable/euphrates": [10.318], + + "ericsson-virtual4:stable/fraser": [10.5465], + + "ericsson-virtual3:stable/fraser": [10.9355], + + "huawei-virtual3:stable/fraser": [10.95], + + "huawei-virtual2:stable/euphrates": [11.274], + + "huawei-virtual4:stable/fraser": [11.557], + + "lf-pod1:stable/euphrates": [15.7025], + + "lf-pod2:stable/euphrates": [15.8495], + + "lf-pod2:stable/fraser": [16.5595], + + "lf-pod1:stable/fraser": [16.8395], + + "arm-pod5:stable/euphrates": [18.092], + + "arm-pod5:stable/fraser": [18.744], + + "huawei-virtual1:stable/fraser": [19.8235], + + "huawei-virtual8:stable/euphrates": [33.999], + + "arm-pod6:stable/euphrates": [41.5605], + + "arm-pod6:stable/fraser": [55.804] + +} + +TC011 +----- + +Iperf3 is a tool for evaluating the packet delay variation between 2 VMs on +different blades. In general, the packet delay variation of the two releases +look similar. + +{ + + "arm-pod6:stable/fraser": [1], + + "ericsson-pod1:stable/fraser": [1], + + "ericsson-virtual2:stable/fraser": [1], + + "ericsson-virtual3:stable/fraser": [1], + + "lf-pod2:stable/fraser": [1], + + "huawei-virtual1:stable/fraser": [2997], + + "huawei-virtual2:stable/euphrates": [2997], + + "flex-pod2:stable/euphrates": [2997.5], + + "huawei-virtual3:stable/euphrates": [2998], + + "huawei-virtual3:stable/fraser": [2999], + + "huawei-virtual9:stable/euphrates": [3000], + + "huawei-virtual8:stable/euphrates": [3001], + + "huawei-virtual4:stable/euphrates": [3002], + + "huawei-virtual4:stable/fraser": [3002], + + "ericsson-virtual3:stable/euphrates": [3006], + + "huawei-virtual1:stable/euphrates": [3007], + + "ericsson-virtual2:stable/euphrates": [3009], + + "intel-pod18:stable/euphrates": [3010], + + "ericsson-virtual4:stable/euphrates": [3017], + + "lf-pod2:stable/euphrates": [3021], + + "arm-pod5:stable/euphrates": [3022], + + "arm-pod6:stable/euphrates": [3022], + + "ericsson-pod1:stable/euphrates": [3022], + + "huawei-pod12:stable/euphrates": [3022], + + "huawei-pod12:stable/fraser": [3022], + + "huawei-pod2:stable/euphrates": [3022], + + "huawei-pod2:stable/fraser": [3022], + + "intel-pod18:stable/fraser": [3022], + + "intel-pod5:stable/euphrates": [3022], + + "lf-pod1:stable/euphrates": [3022], + + "lf-pod1:stable/fraser": [3022], + + "zte-pod2:stable/fraser": [3022], + + "huawei-virtual2:stable/fraser": [3025] + +} + +TC012 +----- + +Lmbench is also used to measure the memory read and write bandwidth. +Like TC010, compared with Euphrates release, the memory read and write bandwidth +of the same pod also experience a slight decline. And compared with X86 pods, the memory +read and write bandwidth of arm pods is significant lower. + +{ + + "lf-pod1:stable/euphrates": [22912.39], + + "lf-pod2:stable/euphrates": [22637.67], + + "lf-pod1:stable/fraser": [20552.9], + + "flex-pod2:stable/euphrates": [20229.99], + + "lf-pod2:stable/fraser": [20058.925], + + "ericsson-pod1:stable/fraser": [18930.78], + + "intel-pod18:stable/fraser": [18757.545], + + "ericsson-virtual1:stable/euphrates": [17474.965], + + "ericsson-pod1:stable/euphrates": [17127.38], + + "ericsson-virtual4:stable/euphrates": [16219.97], + + "ericsson-virtual2:stable/euphrates": [15652.28], + + "ericsson-virtual3:stable/euphrates": [15551.26], + + "ericsson-virtual4:stable/fraser": [15389.465], + + "ericsson-virtual2:stable/fraser": [15343.79], + + "huawei-pod2:stable/euphrates": [15017.2], + + "huawei-pod2:stable/fraser": [14870.78], + + "huawei-virtual4:stable/euphrates": [14266.34], + + "huawei-virtual1:stable/euphrates": [14233.035], + + "huawei-virtual3:stable/euphrates": [14227.63], + + "zte-pod2:stable/fraser": [14157.99], + + "huawei-pod12:stable/euphrates": [14147.245], + + "huawei-pod12:stable/fraser": [14126.99], + + "intel-pod18:stable/euphrates": [14058.33], + + "huawei-virtual3:stable/fraser": [13929.67], + + "huawei-virtual2:stable/euphrates": [13862.85], + + "huawei-virtual4:stable/fraser": [13847.155], + + "huawei-virtual2:stable/fraser": [13702.92], + + "huawei-virtual1:stable/fraser": [13496.45], + + "intel-pod5:stable/euphrates": [13280.32], + + "ericsson-virtual3:stable/fraser": [12733.19], + + "huawei-virtual9:stable/euphrates": [12559.445], + + "huawei-virtual8:stable/euphrates": [8998.02], + + "arm-pod5:stable/euphrates": [4388.875], + + "arm-pod5:stable/fraser": [4326.11], + + "arm-pod6:stable/euphrates": [4260.2], + + "arm-pod6:stable/fraser": [3809.885] + +} + +TC014 +----- + +The Unixbench is used to evaluate the IaaS processing speed with regards to +score of single CPU running and parallel running. Below are the single CPU running +scores. It can be seen that the processing test results vary from scores 715 to 3737. +In general, the single CPU score of the two releases look similar. + +{ + + "lf-pod2:stable/fraser": [3737.6], + + "lf-pod2:stable/euphrates": [3723.95], + + "lf-pod1:stable/fraser": [3702.7], + + "lf-pod1:stable/euphrates": [3669], + + "intel-pod5:stable/euphrates": [3388.6], + + "intel-pod18:stable/euphrates": [3298.4], + + "flex-pod2:stable/euphrates": [3208.6], + + "ericsson-pod1:stable/fraser": [3131.6], + + "intel-pod18:stable/fraser": [3098.1], + + "ericsson-virtual1:stable/euphrates": [2988.9], + + "zte-pod2:stable/fraser": [2831.4], + + "ericsson-pod1:stable/euphrates": [2669.1], + + "ericsson-virtual4:stable/euphrates": [2598.5], + + "ericsson-virtual2:stable/fraser": [2559.7], + + "ericsson-virtual3:stable/euphrates": [2553.15], + + "huawei-pod2:stable/euphrates": [2531.2], + + "huawei-pod2:stable/fraser": [2528.9], + + "ericsson-virtual4:stable/fraser": [2527.8], + + "ericsson-virtual2:stable/euphrates": [2526.9], + + "huawei-virtual4:stable/euphrates": [2407.4], + + "huawei-virtual3:stable/fraser": [2379.1], + + "huawei-virtual3:stable/euphrates": [2374.6], + + "huawei-virtual4:stable/fraser": [2362.1], + + "huawei-virtual2:stable/euphrates": [2326.4], + + "huawei-virtual9:stable/euphrates": [2324.95], + + "huawei-virtual1:stable/euphrates": [2302.6], + + "huawei-virtual2:stable/fraser": [2299.3], + + "huawei-pod12:stable/euphrates": [2232.2], + + "huawei-pod12:stable/fraser": [2229], + + "huawei-virtual1:stable/fraser": [2171.3], + + "ericsson-virtual3:stable/fraser": [2104.8], + + "huawei-virtual8:stable/euphrates": [2085.3], + + "arm-pod5:stable/fraser": [1764.2], + + "arm-pod5:stable/euphrates": [1754.4], + + "arm-pod6:stable/euphrates": [716.15], + + "arm-pod6:stable/fraser": [715.4] + +} + +TC069 +----- + +With the block size changing from 1 kb to 512 kb, the memory write bandwidth +tends to become larger first and then smaller within every run test. Below are +the scores for 32mb block array. + +{ + + "intel-pod18:stable/euphrates": [18871.79], + + "intel-pod18:stable/fraser": [16939.24], + + "intel-pod5:stable/euphrates": [16055.79], + + "arm-pod6:stable/euphrates": [13327.02], + + "arm-pod6:stable/fraser": [11895.71], + + "flex-pod2:stable/euphrates": [9384.585], + + "zte-pod2:stable/fraser": [9375.33], + + "ericsson-pod1:stable/euphrates": [9331.535], + + "huawei-pod12:stable/euphrates": [9164.88], + + "ericsson-pod1:stable/fraser": [9140.42], + + "huawei-pod2:stable/euphrates": [9026.52], + + "huawei-pod12:stable/fraser": [8993.37], + + "huawei-virtual9:stable/euphrates": [8825.805], + + "huawei-pod2:stable/fraser": [8794.01], + + "huawei-virtual2:stable/fraser": [7670.21], + + "ericsson-virtual1:stable/euphrates": [7615.97], + + "ericsson-virtual4:stable/euphrates": [7539.23], + + "arm-pod5:stable/fraser": [7479.32], + + "arm-pod5:stable/euphrates": [7403.38], + + "huawei-virtual3:stable/euphrates": [7247.89], + + "ericsson-virtual2:stable/fraser": [7219.21], + + "huawei-virtual2:stable/euphrates": [7205.35], + + "huawei-virtual1:stable/euphrates": [7196.405], + + "ericsson-virtual3:stable/euphrates": [7173.72], + + "huawei-virtual4:stable/euphrates": [7131.47], + + "ericsson-virtual2:stable/euphrates": [7129.08], + + "huawei-virtual4:stable/fraser": [7059.045], + + "huawei-virtual3:stable/fraser": [7023.57], + + "lf-pod1:stable/euphrates": [6928.18], + + "lf-pod2:stable/euphrates": [6875.88], + + "lf-pod2:stable/fraser": [6834.7], + + "lf-pod1:stable/fraser": [6775.27], + + "ericsson-virtual4:stable/fraser": [6522.86], + + "ericsson-virtual3:stable/fraser": [5835.59], + + "huawei-virtual8:stable/euphrates": [5729.705], + + "huawei-virtual1:stable/fraser": [5617.12] + +} + +TC082 +----- + +For this test case, we use perf to measure context-switches under load. +High context switch rates are not themselves an issue, but they may point the +way to a more significant problem. + +{ + + "zte-pod2:stable/fraser": [306.5], + + "huawei-pod12:stable/euphrates": [316], + + "lf-pod2:stable/fraser": [337.5], + + "intel-pod18:stable/euphrates": [340], + + "intel-pod18:stable/fraser": [343.5], + + "intel-pod5:stable/euphrates": [357.5], + + "ericsson-pod1:stable/euphrates": [384], + + "lf-pod2:stable/euphrates": [394.5], + + "huawei-pod12:stable/fraser": [399], + + "lf-pod1:stable/euphrates": [435], + + "lf-pod1:stable/fraser": [454], + + "flex-pod2:stable/euphrates": [476], + + "huawei-pod2:stable/euphrates": [518], + + "huawei-pod2:stable/fraser": [544.5], + + "arm-pod5:stable/euphrates": [869.5], + + "huawei-virtual9:stable/euphrates": [1002], + + "huawei-virtual4:stable/fraser": [1138], + + "huawei-virtual4:stable/euphrates": [1174], + + "huawei-virtual3:stable/euphrates": [1239], + + "ericsson-pod1:stable/fraser": [1305], + + "huawei-virtual2:stable/euphrates": [1430], + + "huawei-virtual3:stable/fraser": [1433], + + "huawei-virtual1:stable/fraser": [1470], + + "huawei-virtual1:stable/euphrates": [1489], + + "arm-pod6:stable/fraser": [1738.5], + + "arm-pod6:stable/euphrates": [1883.5] + +} + +TC083 +----- + +TC083 measures network latency and throughput between VMs using netperf. +The test results shown below are for UDP throughout. + +{ + + "lf-pod1:stable/euphrates": [2204.42], + + "lf-pod2:stable/fraser": [1893.39], + + "intel-pod18:stable/euphrates": [1835.55], + + "lf-pod2:stable/euphrates": [1676.705], + + "intel-pod5:stable/euphrates": [1612.555], + + "zte-pod2:stable/fraser": [1543.995], + + "lf-pod1:stable/fraser": [1480.86], + + "intel-pod18:stable/fraser": [1417.015], + + "flex-pod2:stable/euphrates": [1370.23], + + "huawei-pod12:stable/euphrates": [1300.12] + +} diff --git a/docs/release/results/euphrates_fraser_comparsion.rst b/docs/release/results/euphrates_fraser_comparsion.rst deleted file mode 100644 index 222dc8bb0..000000000 --- a/docs/release/results/euphrates_fraser_comparsion.rst +++ /dev/null @@ -1,8 +0,0 @@ -.. This work is licensed under a Creative Commons Attribution 4.0 International -.. License. -.. http://creativecommons.org/licenses/by/4.0 - -======================================================= -Test results analysis for Euphrates and Fraser releases -======================================================= - diff --git a/docs/release/results/images/tc002_pod_fraser.png b/docs/release/results/images/tc002_pod_fraser.png Binary files differnew file mode 100644 index 000000000..797dc3136 --- /dev/null +++ b/docs/release/results/images/tc002_pod_fraser.png diff --git a/docs/release/results/images/tc002_scenario_fraser.png b/docs/release/results/images/tc002_scenario_fraser.png Binary files differnew file mode 100644 index 000000000..ff42e6516 --- /dev/null +++ b/docs/release/results/images/tc002_scenario_fraser.png diff --git a/docs/release/results/images/tc010_pod_fraser.png b/docs/release/results/images/tc010_pod_fraser.png Binary files differnew file mode 100644 index 000000000..23367d34a --- /dev/null +++ b/docs/release/results/images/tc010_pod_fraser.png diff --git a/docs/release/results/images/tc010_scenario_fraser.png b/docs/release/results/images/tc010_scenario_fraser.png Binary files differnew file mode 100644 index 000000000..a481a595f --- /dev/null +++ b/docs/release/results/images/tc010_scenario_fraser.png diff --git a/docs/release/results/images/tc011_pod_fraser.png b/docs/release/results/images/tc011_pod_fraser.png Binary files differnew file mode 100644 index 000000000..82dc9c763 --- /dev/null +++ b/docs/release/results/images/tc011_pod_fraser.png diff --git a/docs/release/results/images/tc011_scenario_fraser.png b/docs/release/results/images/tc011_scenario_fraser.png Binary files differnew file mode 100644 index 000000000..226d0b856 --- /dev/null +++ b/docs/release/results/images/tc011_scenario_fraser.png diff --git a/docs/release/results/images/tc012_pod_fraser.png b/docs/release/results/images/tc012_pod_fraser.png Binary files differnew file mode 100644 index 000000000..66e79be85 --- /dev/null +++ b/docs/release/results/images/tc012_pod_fraser.png diff --git a/docs/release/results/images/tc012_scenario_fraser.png b/docs/release/results/images/tc012_scenario_fraser.png Binary files differnew file mode 100644 index 000000000..4ef44119a --- /dev/null +++ b/docs/release/results/images/tc012_scenario_fraser.png diff --git a/docs/release/results/images/tc014_pod_fraseer.png b/docs/release/results/images/tc014_pod_fraseer.png Binary files differnew file mode 100644 index 000000000..697201d76 --- /dev/null +++ b/docs/release/results/images/tc014_pod_fraseer.png diff --git a/docs/release/results/images/tc014_scenario_fraser.png b/docs/release/results/images/tc014_scenario_fraser.png Binary files differnew file mode 100644 index 000000000..f7865dcdc --- /dev/null +++ b/docs/release/results/images/tc014_scenario_fraser.png diff --git a/docs/release/results/images/tc069_pod_fraser.png b/docs/release/results/images/tc069_pod_fraser.png Binary files differnew file mode 100644 index 000000000..1cba192d7 --- /dev/null +++ b/docs/release/results/images/tc069_pod_fraser.png diff --git a/docs/release/results/images/tc069_scenario_fraser.png b/docs/release/results/images/tc069_scenario_fraser.png Binary files differnew file mode 100644 index 000000000..f988b90c6 --- /dev/null +++ b/docs/release/results/images/tc069_scenario_fraser.png diff --git a/docs/release/results/images/tc082_pod_fraser.png b/docs/release/results/images/tc082_pod_fraser.png Binary files differnew file mode 100644 index 000000000..d54ab901a --- /dev/null +++ b/docs/release/results/images/tc082_pod_fraser.png diff --git a/docs/release/results/images/tc083_pod_fraser.png b/docs/release/results/images/tc083_pod_fraser.png Binary files differnew file mode 100644 index 000000000..942cc2074 --- /dev/null +++ b/docs/release/results/images/tc083_pod_fraser.png diff --git a/docs/release/results/index.rst b/docs/release/results/index.rst index 3ec9e1cff..63445fd1a 100644 --- a/docs/release/results/index.rst +++ b/docs/release/results/index.rst @@ -14,4 +14,4 @@ Yardstick test results .. include:: ./overview.rst .. include:: ./results.rst -.. include:: ./euphrates_fraser_comparsion.rst +.. include:: ./euphrates_fraser_comparison.rst diff --git a/docs/release/results/results.rst b/docs/release/results/results.rst index c75f5ae94..0ed92f867 100644 --- a/docs/release/results/results.rst +++ b/docs/release/results/results.rst @@ -23,6 +23,7 @@ Scenario Results The following documents contain results of Yardstick test cases executed on OPNFV labs, triggered by OPNFV CI pipeline, documented per test case. +For hardware details of OPNFV labs, please visit: https://wiki.opnfv.org/display/pharos/Community+Labs .. toctree:: :maxdepth: 1 @@ -38,6 +39,7 @@ OPNFV labs, triggered by OPNFV CI pipeline, documented per test case. Test results of executed tests are avilable in Dashboard_ and logs in Jenkins_. +Test results for Fraser release are collected from April 10, 2018 to May 13, 2018. Feature Test Results ==================== diff --git a/docs/release/results/tc002-network-latency.rst b/docs/release/results/tc002-network-latency.rst index 722423473..064983bec 100644 --- a/docs/release/results/tc002-network-latency.rst +++ b/docs/release/results/tc002-network-latency.rst @@ -315,3 +315,211 @@ The influence of the POD Fraser release -------------- + +Test results per scenario and pod (lower is better): + +{ + + "os-odl_l3-nofeature-ha:huawei-pod2:compass": [0.42], + + "os-odl-sfc-ha:huawei-pod2:compass": [0.557], + + "os-nosdn-ovs-ha:ericsson-pod1:fuel": [0.5765], + + "os-nosdn-kvm-ha:huawei-pod2:compass": [0.582], + + "os-odl-bgpvpn-ha:lf-pod1:apex": [0.678], + + "os-nosdn-nofeature-ha:lf-pod1:apex": [0.7075], + + "os-nosdn-calipso-noha:lf-pod1:apex": [0.713], + + "os-nosdn-nofeature-noha:lf-pod1:apex": [0.7155], + + "os-nosdn-bar-ha:lf-pod1:apex": [0.732], + + "os-nosdn-bar-noha:lf-pod1:apex": [0.7415], + + "os-odl-nofeature-noha:lf-pod1:apex": [0.7565], + + "os-nosdn-ovs-ha:arm-pod6:fuel": [0.8015], + + "os-nosdn-nofeature-ha:huawei-pod2:compass": [0.908], + + "os-odl-nofeature-ha:ericsson-pod1:fuel": [0.9165], + + "os-nosdn-bar-ha:huawei-pod2:compass": [0.969], + + "os-nosdn-ovs-noha:ericsson-virtual2:fuel": [0.9765], + + "os-nosdn-nofeature-noha:huawei-pod12:joid": [1.0245], + + "os-nosdn-nofeature-ha:huawei-pod12:joid": [1.0495], + + "os-odl-sfc-noha:huawei-virtual4:compass": [1.1645], + + "os-nosdn-nofeature-ha:lf-pod2:fuel": [1.206], + + "os-odl-sfc-noha:huawei-virtual3:compass": [1.236], + + "os-nosdn-ovs-noha:ericsson-virtual4:fuel": [1.241], + + "os-nosdn-nofeature-ha:zte-pod2:daisy": [1.2805], + + "os-odl-nofeature-ha:lf-pod2:fuel": [1.286], + + "os-odl_l3-nofeature-noha:huawei-virtual3:compass": [1.299], + + "os-odl-sfc-ha:huawei-virtual4:compass": [1.305], + + "os-odl_l3-nofeature-noha:huawei-virtual4:compass": [1.309], + + "os-nosdn-kvm-noha:huawei-virtual4:compass": [1.314], + + "os-nosdn-nofeature-noha:huawei-virtual4:compass": [1.431], + + "os-nosdn-nofeature-ha:ericsson-pod1:fuel": [1.457], + + "os-odl-nofeature-ha:zte-pod2:daisy": [1.517], + + "os-nosdn-kvm-noha:huawei-virtual3:compass": [1.576], + + "os-nosdn-nofeature-noha:huawei-virtual3:compass": [1.592], + + "os-odl-nofeature-ha:arm-pod5:fuel": [1.714], + + "os-nosdn-nofeature-noha:intel-pod18:joid": [1.809], + + "os-nosdn-bar-noha:huawei-virtual4:compass": [1.81], + + "os-nosdn-nofeature-ha:intel-pod18:joid": [1.8505], + + "os-nosdn-nofeature-ha:huawei-virtual4:compass": [1.8895], + + "os-nosdn-nofeature-ha:huawei-virtual3:compass": [1.909], + + "os-odl_l3-nofeature-ha:huawei-virtual4:compass": [1.925], + + "os-nosdn-nofeature-noha:huawei-virtual2:compass": [1.964], + + "os-nosdn-openbaton-ha:intel-pod18:joid": [1.9755], + + "os-nosdn-nofeature-ha:huawei-virtual1:compass": [1.9765], + + "os-nosdn-bar-noha:huawei-virtual3:compass": [1.9915], + + "os-odl_l3-nofeature-ha:huawei-virtual3:compass": [1.9925], + + "os-nosdn-kvm-ha:huawei-virtual4:compass": [2.0265], + + "os-odl-nofeature-ha:arm-pod6:fuel": [2.106], + + "os-odl-sfc-ha:huawei-virtual3:compass": [2.124], + + "os-nosdn-kvm-ha:huawei-virtual3:compass": [2.185], + + "os-nosdn-nofeature-ha:arm-pod6:fuel": [2.281], + + "os-nosdn-bar-ha:huawei-virtual4:compass": [2.432], + + "os-odl-nofeature-noha:ericsson-virtual4:fuel": [2.483], + + "os-nosdn-bar-ha:huawei-virtual3:compass": [2.524], + + "os-nosdn-nofeature-noha:ericsson-virtual3:fuel": [3.9175], + + "os-odl-nofeature-noha:ericsson-virtual2:fuel": [4.338], + + "os-nosdn-nofeature-noha:ericsson-virtual2:fuel": [4.641] + +} + + +The influence of the scenario +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. image:: images/tc002_scenario_fraser.png + :width: 800px + :alt: TC002 influence of scenario + +{ + + "os-odl-bgpvpn-ha": [0.678], + + "os-nosdn-calipso-noha": [0.713], + + "os-nosdn-ovs-ha": [0.7245], + + "os-odl_l3-nofeature-ha": [0.7435], + + "os-odl-sfc-ha": [0.796], + + "os-nosdn-kvm-ha": [1.059], + + "os-nosdn-bar-ha": [1.083], + + "os-nosdn-ovs-noha": [1.09], + + "os-odl-sfc-noha": [1.196], + + "os-nosdn-nofeature-noha": [1.26], + + "os-nosdn-nofeature-ha": [1.291], + + "os-odl_l3-nofeature-noha": [1.308], + + "os-nosdn-bar-noha": [1.4125], + + "os-nosdn-kvm-noha": [1.4475], + + "os-odl-nofeature-ha": [1.508], + + "os-odl-nofeature-noha": [1.914], + + "os-nosdn-openbaton-ha": [1.9755] + +} + + +The influence of the POD +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. image:: images/tc002_pod_fraser.png + :width: 800px + :alt: TC002 influence of the POD + +{ + + "huawei-pod2": [0.677], + + "lf-pod1": [0.725], + + "ericsson-pod1": [0.9165], + + "huawei-pod12": [1.0465], + + "lf-pod2": [1.2325], + + "zte-pod2": [1.395], + + "ericsson-virtual4": [1.582], + + "huawei-virtual4": [1.697], + + "arm-pod5": [1.714], + + "huawei-virtual3": [1.716], + + "intel-pod18": [1.856], + + "huawei-virtual2": [1.964], + + "huawei-virtual1": [1.9765], + + "arm-pod6": [2.209], + + "ericsson-virtual3": [3.9175], + + "ericsson-virtual2": [4.004] + +} diff --git a/docs/release/results/tc010-memory-read-latency.rst b/docs/release/results/tc010-memory-read-latency.rst index 9a296b7a0..81559d647 100644 --- a/docs/release/results/tc010-memory-read-latency.rst +++ b/docs/release/results/tc010-memory-read-latency.rst @@ -297,3 +297,214 @@ The influence of the POD Fraser release -------------- + +Test results per scenario and pod (lower is better): + +{ + + "os-odl-nofeature-ha:ericsson-pod1:fuel": [6.8675], + + "os-nosdn-nofeature-noha:intel-pod18:joid": [6.991], + + "os-nosdn-openbaton-ha:intel-pod18:joid": [7.5535], + + "os-nosdn-nofeature-ha:intel-pod18:joid": [7.571], + "os-nosdn-ovs-ha:ericsson-pod1:fuel": [7.635], + + "os-nosdn-nofeature-ha:zte-pod2:daisy": [8.153], + + "os-odl-nofeature-ha:zte-pod2:daisy": [8.1935], + + "os-nosdn-bar-ha:huawei-pod2:compass": [9.1715], + + "os-odl-sfc-ha:huawei-pod2:compass": [9.1875], + + "os-odl_l3-nofeature-ha:huawei-pod2:compass": [9.241], + + "os-nosdn-nofeature-ha:huawei-pod2:compass": [9.255], + + "os-nosdn-kvm-ha:huawei-pod2:compass": [9.388], + + "os-nosdn-nofeature-noha:huawei-virtual4:compass": [9.5825], + + "os-nosdn-nofeature-noha:huawei-pod12:joid": [9.5875], + + "os-nosdn-nofeature-ha:huawei-pod12:joid": [9.6345], + + "os-odl-sfc-noha:huawei-virtual4:compass": [9.6535], + + "os-nosdn-nofeature-noha:ericsson-virtual2:fuel": [9.743], + + "os-odl-sfc-noha:huawei-virtual3:compass": [9.82], + + "os-odl-nofeature-noha:ericsson-virtual2:fuel": [9.8715], + + "os-odl_l3-nofeature-noha:huawei-virtual3:compass": [9.982], + + "os-nosdn-bar-noha:huawei-virtual4:compass": [10.0195], + + "os-odl_l3-nofeature-noha:huawei-virtual4:compass": [10.1285], + + "os-nosdn-nofeature-noha:huawei-virtual3:compass": [10.1335], + + "os-nosdn-nofeature-noha:huawei-virtual2:compass": [10.22], + + "os-nosdn-bar-noha:huawei-virtual3:compass": [10.2845], + + "os-nosdn-ovs-noha:ericsson-virtual4:fuel": [10.4185], + + "os-nosdn-ovs-noha:ericsson-virtual2:fuel": [10.4555], + + "os-nosdn-kvm-noha:huawei-virtual3:compass": [10.5635], + + "os-nosdn-kvm-noha:huawei-virtual4:compass": [10.6515], + + "os-nosdn-nofeature-noha:ericsson-virtual3:fuel": [10.9355], + + "os-odl-nofeature-noha:ericsson-virtual4:fuel": [11.2015], + + "os-odl_l3-nofeature-ha:huawei-virtual3:compass": [12.984], + + "os-nosdn-bar-ha:huawei-virtual3:compass": [13.306], + + "os-nosdn-nofeature-ha:huawei-virtual3:compass": [13.721], + + "os-nosdn-bar-ha:huawei-virtual4:compass": [14.133], + + "os-nosdn-nofeature-ha:ericsson-pod1:fuel": [14.158], + + "os-odl_l3-nofeature-ha:huawei-virtual4:compass": [14.375], + + "os-nosdn-nofeature-ha:huawei-virtual4:compass": [14.396], + + "os-nosdn-kvm-ha:huawei-virtual4:compass": [14.9375], + + "os-odl-sfc-ha:huawei-virtual3:compass": [14.957], + + "os-nosdn-calipso-noha:lf-pod1:apex": [16.3445], + + "os-nosdn-ovs-ha:lf-pod2:fuel": [16.478], + + "os-nosdn-nofeature-ha:lf-pod2:fuel": [16.4895], + + "os-odl-nofeature-noha:lf-pod1:apex": [16.55], + + "os-nosdn-nofeature-noha:lf-pod1:apex": [16.5665], + + "os-odl-sfc-noha:lf-pod1:apex": [16.598], + + "os-ovn-nofeature-noha:lf-pod1:apex": [16.805], + + "os-odl-nofeature-ha:lf-pod1:apex": [16.9095], + + "os-nosdn-bar-ha:lf-pod1:apex": [17.494], + + "os-nosdn-bar-noha:lf-pod1:apex": [17.4995], + + "os-nosdn-nofeature-ha:lf-pod1:apex": [18.094], + + "os-odl-nofeature-ha:arm-pod5:fuel": [18.744], + + "os-nosdn-nofeature-ha:huawei-virtual1:compass": [19.8235], + + "os-odl-nofeature-ha:lf-pod2:fuel": [20.758], + + "os-nosdn-kvm-ha:huawei-virtual3:compass": [26.5245], + + "os-nosdn-nofeature-ha:arm-pod6:fuel": [55.667], + + "os-odl-nofeature-ha:arm-pod6:fuel": [56.175], + + "os-nosdn-ovs-ha:arm-pod6:fuel": [57.86] + +} + + +The influence of the scenario +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. image:: images/tc010_scenario_fraser.png + :width: 800px + :alt: TC010 influence of scenario + +{ + + "os-nosdn-openbaton-ha": [7.5535], + + "os-odl-nofeature-ha": [8.2535], + + "os-odl-sfc-ha": [9.251], + + "os-nosdn-nofeature-ha": [9.464], + + "os-odl-sfc-noha": [9.8265], + + "os-odl_l3-nofeature-ha": [9.836], + + "os-odl_l3-nofeature-noha": [10.0565], + + "os-nosdn-nofeature-noha": [10.079], + + "os-nosdn-kvm-ha": [10.418], + + "os-nosdn-ovs-noha": [10.43], + + "os-nosdn-kvm-noha": [10.603], + + "os-nosdn-bar-noha": [11.067], + + "os-nosdn-bar-ha": [13.911], + + "os-odl-nofeature-noha": [14.046], + + "os-nosdn-calipso-noha": [16.3445], + + "os-nosdn-ovs-ha": [16.478], + + "os-ovn-nofeature-noha": [16.805] + +} + + +The influence of the POD +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. image:: images/tc010_pod_fraser.png + :width: 800px + :alt: TC010 influence of the POD + +{ + + "ericsson-pod1": [7.0645], + + "intel-pod18": [7.4465], + + "zte-pod2": [8.1865], + + "huawei-pod2": [9.236], + + "huawei-pod12": [9.615], + + "ericsson-virtual2": [9.8925], + + "huawei-virtual2": [10.22], + + "ericsson-virtual4": [10.5465], + + "ericsson-virtual3": [10.9355], + + "huawei-virtual3": [10.95], + + "huawei-virtual4": [11.557], + + "lf-pod2": [16.5595], + + "lf-pod1": [16.8395], + + "arm-pod5": [18.744], + + "huawei-virtual1": [19.8235], + + "arm-pod6": [55.804] + +} diff --git a/docs/release/results/tc011-packet-delay-variation.rst b/docs/release/results/tc011-packet-delay-variation.rst index b07ea8980..f255b50ca 100644 --- a/docs/release/results/tc011-packet-delay-variation.rst +++ b/docs/release/results/tc011-packet-delay-variation.rst @@ -260,3 +260,173 @@ The influence of the POD Fraser release -------------- + +Test results per scenario and pod (lower is better): + +{ + + "os-nosdn-nofeature-ha:arm-pod6:fuel": [1], + + "os-nosdn-nofeature-ha:ericsson-pod1:fuel": [1], + + "os-nosdn-nofeature-ha:lf-pod2:fuel": [1], + + "os-nosdn-nofeature-noha:ericsson-virtual2:fuel": [1], + + "os-nosdn-nofeature-noha:ericsson-virtual3:fuel": [1], + + "os-ovn-nofeature-noha:lf-pod1:apex": [1511.5], + + "os-nosdn-kvm-noha:huawei-virtual3:compass": [2996], + + "os-nosdn-bar-ha:huawei-virtual4:compass": [2997], + + "os-nosdn-bar-noha:huawei-virtual4:compass": [2997], + + "os-nosdn-kvm-ha:huawei-virtual4:compass": [2997], + + "os-nosdn-nofeature-ha:huawei-virtual1:compass": [2997], + + "os-nosdn-nofeature-ha:huawei-virtual3:compass": [2997], + + "os-nosdn-nofeature-ha:huawei-virtual4:compass": [2997], + + "os-odl-sfc-ha:huawei-virtual3:compass": [2997], + + "os-odl_l3-nofeature-ha:huawei-virtual3:compass": [2997], + + "os-odl_l3-nofeature-noha:huawei-virtual3:compass": [3000], + + "os-odl_l3-nofeature-ha:huawei-virtual4:compass": [3003], + + "os-nosdn-bar-noha:huawei-virtual3:compass": [3011], + + "os-nosdn-bar-ha:huawei-virtual3:compass": [3015.5], + + "os-nosdn-kvm-noha:huawei-virtual4:compass": [3019], + + "os-nosdn-nofeature-noha:huawei-virtual4:compass": [3021], + + "os-odl-sfc-ha:huawei-virtual4:compass": [3021], + + "os-nosdn-bar-ha:huawei-pod2:compass": [3022], + + "os-nosdn-bar-ha:lf-pod1:apex": [3022], + + "os-nosdn-bar-noha:lf-pod1:apex": [3022], + + "os-nosdn-calipso-noha:lf-pod1:apex": [3022], + + "os-nosdn-kvm-ha:huawei-pod2:compass": [3022], + + "os-nosdn-nofeature-ha:huawei-pod12:joid": [3022], + + "os-nosdn-nofeature-ha:huawei-pod2:compass": [3022], + + "os-nosdn-nofeature-ha:intel-pod18:joid": [3022], + + "os-nosdn-nofeature-ha:lf-pod1:apex": [3022], + + "os-nosdn-nofeature-ha:zte-pod2:daisy": [3022], + + "os-nosdn-nofeature-noha:huawei-pod12:joid": [3022], + + "os-nosdn-nofeature-noha:intel-pod18:joid": [3022], + + "os-nosdn-nofeature-noha:lf-pod1:apex": [3022], + + "os-nosdn-openbaton-ha:intel-pod18:joid": [3022], + + "os-odl-sfc-ha:huawei-pod2:compass": [3022], + + "os-odl_l3-nofeature-ha:huawei-pod2:compass": [3022], + + "os-odl_l3-nofeature-noha:huawei-virtual4:compass": [3022], + + "os-odl-sfc-noha:huawei-virtual4:compass": [3022.5], + + "os-nosdn-nofeature-noha:huawei-virtual3:compass": [3023], + + "os-odl-sfc-noha:huawei-virtual3:compass": [3023], + + "os-nosdn-nofeature-noha:huawei-virtual2:compass": [3025] + +} + + +The influence of the scenario +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. image:: images/tc011_scenario_fraser.png + :width: 800px + :alt: TC011 influence of scenario + +{ + + "os-ovn-nofeature-noha": [1511.5], + + "os-nosdn-kvm-noha": [2997], + + "os-odl-sfc-ha": [3021], + + "os-nosdn-bar-ha": [3022], + + "os-nosdn-bar-noha": [3022], + + "os-nosdn-calipso-noha": [3022], + + "os-nosdn-kvm-ha": [3022], + + "os-nosdn-nofeature-ha": [3022], + + "os-nosdn-nofeature-noha": [3022], + + "os-nosdn-openbaton-ha": [3022], + + "os-odl_l3-nofeature-ha": [3022], + + "os-odl_l3-nofeature-noha": [3022], + + "os-odl-sfc-noha": [3023] + +} + + +The influence of the POD +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. image:: images/tc011_pod_fraser.png + :width: 800px + :alt: TC011 influence of the POD + +{ + + "arm-pod6": [1], + + "ericsson-pod1": [1], + + "ericsson-virtual2": [1], + + "ericsson-virtual3": [1], + + "lf-pod2": [1], + + "huawei-virtual1": [2997], + + "huawei-virtual3": [2999], + + "huawei-virtual4": [3002], + + "huawei-pod12": [3022], + + "huawei-pod2": [3022], + + "intel-pod18": [3022], + + "lf-pod1": [3022], + + "zte-pod2": [3022], + + "huawei-virtual2": [3025] + +} diff --git a/docs/release/results/tc012-memory-read-write-bandwidth.rst b/docs/release/results/tc012-memory-read-write-bandwidth.rst index c28eb1f3c..71d69cde9 100644 --- a/docs/release/results/tc012-memory-read-write-bandwidth.rst +++ b/docs/release/results/tc012-memory-read-write-bandwidth.rst @@ -297,3 +297,217 @@ The influence of the POD Fraser release -------------- + +Test results per scenario and pod (higher is better): + +{ + + "os-nosdn-nofeature-ha:lf-pod2:fuel": [21421.795], + + "os-odl-sfc-noha:lf-pod1:apex": [21075], + + "os-odl-sfc-ha:lf-pod1:apex": [21017.44], + + "os-nosdn-bar-noha:lf-pod1:apex": [20991.46], + + "os-nosdn-bar-ha:lf-pod1:apex": [20812.405], + + "os-ovn-nofeature-noha:lf-pod1:apex": [20694.035], + + "os-nosdn-nofeature-noha:lf-pod1:apex": [20672.765], + + "os-odl-nofeature-ha:lf-pod2:fuel": [20269.65], + + "os-nosdn-calipso-noha:lf-pod1:apex": [20186.32], + + "os-odl-nofeature-noha:lf-pod1:apex": [19959.915], + + "os-nosdn-ovs-ha:lf-pod2:fuel": [19719.38], + + "os-odl-nofeature-ha:lf-pod1:apex": [19654.505], + + "os-nosdn-nofeature-ha:lf-pod1:apex": [19391.145], + + "os-nosdn-nofeature-noha:intel-pod18:joid": [19378.64], + + "os-odl-nofeature-ha:ericsson-pod1:fuel": [19103.43], + + "os-nosdn-nofeature-ha:intel-pod18:joid": [18688.695], + + "os-nosdn-openbaton-ha:intel-pod18:joid": [18557.95], + + "os-nosdn-nofeature-ha:ericsson-pod1:fuel": [17088.61], + + "os-nosdn-ovs-ha:ericsson-pod1:fuel": [17040.78], + + "os-nosdn-ovs-noha:ericsson-virtual2:fuel": [16057.235], + + "os-odl-nofeature-noha:ericsson-virtual4:fuel": [15622.355], + + "os-nosdn-nofeature-noha:ericsson-virtual2:fuel": [15422.235], + + "os-odl-sfc-ha:huawei-pod2:compass": [15403.09], + + "os-odl-nofeature-noha:ericsson-virtual2:fuel": [15141.58], + + "os-nosdn-bar-ha:huawei-pod2:compass": [14922.37], + + "os-odl_l3-nofeature-ha:huawei-pod2:compass": [14864.195], + + "os-nosdn-nofeature-ha:huawei-pod2:compass": [14856.295], + + "os-nosdn-kvm-ha:huawei-pod2:compass": [14796.035], + + "os-odl-sfc-noha:huawei-virtual4:compass": [14484.375], + + "os-nosdn-nofeature-ha:huawei-pod12:joid": [14441.955], + + "os-odl-sfc-noha:huawei-virtual3:compass": [14373], + + "os-nosdn-nofeature-noha:huawei-virtual4:compass": [14330.44], + + "os-nosdn-ovs-noha:ericsson-virtual4:fuel": [14320.305], + + "os-odl_l3-nofeature-noha:huawei-virtual3:compass": [14253.715], + + "os-nosdn-nofeature-ha:huawei-virtual4:compass": [14203.655], + + "os-nosdn-nofeature-noha:huawei-virtual3:compass": [14179.93], + + "os-odl-nofeature-ha:zte-pod2:daisy": [14177.135], + + "os-nosdn-nofeature-ha:zte-pod2:daisy": [14150.825], + + "os-nosdn-nofeature-noha:huawei-pod12:joid": [14100.87], + + "os-nosdn-bar-noha:huawei-virtual4:compass": [14033.36], + + "os-odl_l3-nofeature-noha:huawei-virtual4:compass": [13963.73], + + "os-nosdn-kvm-noha:huawei-virtual3:compass": [13874.775], + + "os-nosdn-kvm-noha:huawei-virtual4:compass": [13805.65], + + "os-odl_l3-nofeature-ha:huawei-virtual3:compass": [13754.63], + + "os-nosdn-nofeature-noha:huawei-virtual2:compass": [13702.92], + + "os-nosdn-bar-ha:huawei-virtual3:compass": [13638.115], + + "os-odl-sfc-ha:huawei-virtual3:compass": [13637.83], + + "os-odl_l3-nofeature-ha:huawei-virtual4:compass": [13635.66], + + "os-nosdn-bar-noha:huawei-virtual3:compass": [13635.58], + + "os-nosdn-bar-ha:huawei-virtual4:compass": [13544.95], + + "os-nosdn-nofeature-ha:huawei-virtual3:compass": [13514.27], + + "os-nosdn-nofeature-ha:huawei-virtual1:compass": [13496.45], + + "os-odl-sfc-ha:huawei-virtual4:compass": [13475.38], + + "os-nosdn-nofeature-noha:ericsson-virtual3:fuel": [12733.19], + + "os-nosdn-kvm-ha:huawei-virtual4:compass": [12682.805], + + "os-odl-nofeature-ha:arm-pod5:fuel": [4326.11], + + "os-nosdn-nofeature-ha:arm-pod6:fuel": [3824.13], + + "os-odl-nofeature-ha:arm-pod6:fuel": [3797.795], + + "os-nosdn-ovs-ha:arm-pod6:fuel": [3749.91] + +} + + +The influence of the scenario +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. image:: images/tc012_scenario_fraser.png + :width: 800px + :alt: TC012 influence of scenario + +{ + + "os-ovn-nofeature-noha": [20694.035], + + "os-nosdn-calipso-noha": [20186.32], + + "os-nosdn-openbaton-ha": [18557.95], + + "os-nosdn-ovs-ha": [17048.17], + + "os-odl-nofeature-noha": [16191.125], + + "os-nosdn-ovs-noha": [15790.32], + + "os-nosdn-bar-ha": [14833.97], + + "os-odl-sfc-ha": [14828.72], + + "os-odl_l3-nofeature-ha": [14801.25], + + "os-nosdn-kvm-ha": [14700.1], + + "os-nosdn-nofeature-ha": [14610.48], + + "os-nosdn-nofeature-noha": [14555.975], + + "os-odl-sfc-noha": [14508.14], + + "os-nosdn-bar-noha": [14395.22], + + "os-odl-nofeature-ha": [14231.245], + + "os-odl_l3-nofeature-noha": [14161.58], + + "os-nosdn-kvm-noha": [13845.685] + +} + + +The influence of the POD +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. image:: images/tc012_pod_fraser.png + :width: 800px + :alt: TC012 influence of the POD + +{ + + "lf-pod1": [20552.9], + + "lf-pod2": [20058.925], + + "ericsson-pod1": [18930.78], + + "intel-pod18": [18757.545], + + "ericsson-virtual4": [15389.465], + + "ericsson-virtual2": [15343.79], + + "huawei-pod2": [14870.78], + + "zte-pod2": [14157.99], + + "huawei-pod12": [14126.99], + + "huawei-virtual3": [13929.67], + + "huawei-virtual4": [13847.155], + + "huawei-virtual2": [13702.92], + + "huawei-virtual1": [13496.45], + + "ericsson-virtual3": [12733.19], + + "arm-pod5": [4326.11], + + "arm-pod6": [3809.885] + +} diff --git a/docs/release/results/tc014-cpu-processing-speed.rst b/docs/release/results/tc014-cpu-processing-speed.rst index 34d4ad0f9..a2eeb6302 100644 --- a/docs/release/results/tc014-cpu-processing-speed.rst +++ b/docs/release/results/tc014-cpu-processing-speed.rst @@ -296,3 +296,217 @@ The influence of the POD Fraser release -------------- + +Test results per scenario and pod (higher is better): + +{ + + "os-nosdn-nofeature-ha:lf-pod2:fuel": [3747.3], + + "os-nosdn-calipso-noha:lf-pod1:apex": [3727.2], + + "os-odl-nofeature-ha:lf-pod1:apex": [3726.5], + + "os-ovn-nofeature-noha:lf-pod1:apex": [3723.8], + + "os-odl-nofeature-noha:lf-pod1:apex": [3718.9], + + "os-nosdn-nofeature-noha:lf-pod1:apex": [3717.75], + + "os-nosdn-nofeature-ha:lf-pod1:apex": [3706.5], + + "os-odl-nofeature-ha:lf-pod2:fuel": [3704.9], + + "os-nosdn-ovs-ha:lf-pod2:fuel": [3687.7], + + "os-nosdn-bar-noha:lf-pod1:apex": [3635.4], + + "os-nosdn-bar-ha:lf-pod1:apex": [3632.55], + + "os-odl-sfc-noha:lf-pod1:apex": [3569], + + "os-nosdn-nofeature-noha:intel-pod18:joid": [3432.1], + + "os-odl-nofeature-ha:ericsson-pod1:fuel": [3133.85], + + "os-nosdn-ovs-ha:ericsson-pod1:fuel": [3079.8], + + "os-nosdn-nofeature-ha:intel-pod18:joid": [3074.75], + + "os-nosdn-openbaton-ha:intel-pod18:joid": [2976.2], + + "os-nosdn-nofeature-ha:zte-pod2:daisy": [2910.95], + + "os-odl-nofeature-ha:zte-pod2:daisy": [2801.1], + + "os-nosdn-ovs-noha:ericsson-virtual2:fuel": [2603], + + "os-odl-nofeature-noha:ericsson-virtual2:fuel": [2559.7], + + "os-nosdn-nofeature-noha:ericsson-virtual2:fuel": [2539.1], + + "os-odl_l3-nofeature-ha:huawei-pod2:compass": [2530.5], + + "os-nosdn-nofeature-ha:huawei-pod2:compass": [2529.4], + + "os-odl-sfc-ha:huawei-pod2:compass": [2528.9], + + "os-odl-nofeature-noha:ericsson-virtual4:fuel": [2527.8], + + "os-nosdn-bar-ha:huawei-pod2:compass": [2527.4], + + "os-nosdn-kvm-ha:huawei-pod2:compass": [2517.8], + + "os-nosdn-nofeature-noha:huawei-virtual4:compass": [2472.4], + + "os-nosdn-nofeature-ha:huawei-virtual4:compass": [2469.1], + + "os-odl-sfc-noha:huawei-virtual3:compass": [2452.05], + + "os-odl-sfc-noha:huawei-virtual4:compass": [2438.7], + + "os-odl_l3-nofeature-noha:huawei-virtual3:compass": [2418.4], + + "os-nosdn-ovs-noha:ericsson-virtual4:fuel": [2404.35], + + "os-nosdn-kvm-noha:huawei-virtual3:compass": [2391], + + "os-nosdn-kvm-noha:huawei-virtual4:compass": [2376.75], + + "os-odl_l3-nofeature-noha:huawei-virtual4:compass": [2376.2], + + "os-nosdn-nofeature-noha:huawei-virtual3:compass": [2359.45], + + "os-nosdn-bar-noha:huawei-virtual4:compass": [2353.3], + + "os-odl-sfc-ha:huawei-virtual3:compass": [2351.9], + + "os-nosdn-bar-ha:huawei-virtual3:compass": [2339.4], + + "os-odl-sfc-ha:huawei-virtual4:compass": [2335.6], + + "os-nosdn-bar-ha:huawei-virtual4:compass": [2328], + + "os-odl_l3-nofeature-ha:huawei-virtual3:compass": [2324.5], + + "os-nosdn-bar-noha:huawei-virtual3:compass": [2317.3], + + "os-nosdn-nofeature-ha:huawei-virtual3:compass": [2313.95], + + "os-odl_l3-nofeature-ha:huawei-virtual4:compass": [2308.1], + + "os-nosdn-nofeature-noha:huawei-virtual2:compass": [2299.3], + + "os-nosdn-kvm-ha:huawei-virtual4:compass": [2250.4], + + "os-nosdn-nofeature-noha:huawei-pod12:joid": [2229.7], + + "os-nosdn-nofeature-ha:huawei-pod12:joid": [2228.8], + + "os-nosdn-nofeature-ha:huawei-virtual1:compass": [2171.3], + + "os-nosdn-nofeature-noha:ericsson-virtual3:fuel": [2104.8], + + "os-nosdn-nofeature-ha:ericsson-pod1:fuel": [1961.35], + + "os-nosdn-ovs-ha:arm-pod5:fuel": [1764.2], + + "os-odl-nofeature-ha:arm-pod5:fuel": [1730.95], + + "os-nosdn-ovs-ha:arm-pod6:fuel": [715.55], + + "os-odl-nofeature-ha:arm-pod6:fuel": [715.4], + + "os-nosdn-nofeature-ha:arm-pod6:fuel": [715.25] + +} + + +The influence of the scenario +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. image:: images/tc014_scenario_fraser.png + :width: 800px + :alt: TC014 influence of scenario + +{ + + "os-nosdn-calipso-noha": [3727.2], + + "os-ovn-nofeature-noha": [3723.8], + + "os-odl-nofeature-noha": [3128.05], + + "os-nosdn-openbaton-ha": [2976.2], + + "os-nosdn-ovs-ha": [2814.5], + + "os-odl-nofeature-ha": [2801.4], + + "os-nosdn-nofeature-ha": [2649.7], + + "os-nosdn-ovs-noha": [2587.3], + + "os-odl_l3-nofeature-ha": [2528.45], + + "os-odl-sfc-ha": [2527.6], + + "os-nosdn-bar-ha": [2526.55], + + "os-nosdn-kvm-ha": [2516.95], + + "os-odl-sfc-noha": [2453.65], + + "os-nosdn-bar-noha": [2447.7], + + "os-nosdn-nofeature-noha": [2443.85], + + "os-odl_l3-nofeature-noha": [2394.3], + + "os-nosdn-kvm-noha": [2379.7] + +} + + +The influence of the POD +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. image:: images/tc014_pod_fraser.png + :width: 800px + :alt: TC014 influence of the POD + +{ + + "lf-pod2": [3737.6], + + "lf-pod1": [3702.7], + + "ericsson-pod1": [3131.6], + + "intel-pod18": [3098.1], + + "zte-pod2": [2831.4], + + "ericsson-virtual2": [2559.7], + + "huawei-pod2": [2528.9], + + "ericsson-virtual4": [2527.8], + + "huawei-virtual3": [2379.1], + + "huawei-virtual4": [2362.1], + + "huawei-virtual2": [2299.3], + + "huawei-pod12": [2229], + + "huawei-virtual1": [2171.3], + + "ericsson-virtual3": [2104.8], + + "arm-pod5": [1764.2], + + "arm-pod6": [715.4] + +} diff --git a/docs/release/results/tc069-memory-write-bandwidth.rst b/docs/release/results/tc069-memory-write-bandwidth.rst index 06e2ec922..4cd3be3b0 100644 --- a/docs/release/results/tc069-memory-write-bandwidth.rst +++ b/docs/release/results/tc069-memory-write-bandwidth.rst @@ -298,3 +298,219 @@ The influence of the POD Fraser release -------------- + +Test results per scenario and pod (higher is better): + +{ + + "os-nosdn-nofeature-noha:intel-pod18:joid": [18382.49], + + "os-nosdn-openbaton-ha:intel-pod18:joid": [16774.52], + + "os-nosdn-nofeature-ha:intel-pod18:joid": [16680.305], + + "os-nosdn-ovs-ha:arm-pod6:fuel": [11925.22], + + "os-nosdn-nofeature-ha:arm-pod6:fuel": [11895.71], + + "os-odl-nofeature-ha:arm-pod6:fuel": [11880.7], + + "os-nosdn-nofeature-ha:ericsson-pod1:fuel": [9471.095], + + "os-odl-nofeature-ha:zte-pod2:daisy": [9375.33], + + "os-nosdn-nofeature-ha:zte-pod2:daisy": [9372.95], + + "os-odl-nofeature-ha:ericsson-pod1:fuel": [9174.36], + + "os-nosdn-nofeature-noha:huawei-pod12:joid": [9051.57], + + "os-nosdn-nofeature-ha:huawei-pod12:joid": [8894.74], + + "os-odl_l3-nofeature-ha:huawei-pod2:compass": [8857.23], + + "os-nosdn-nofeature-ha:huawei-pod2:compass": [8855.8], + + "os-nosdn-bar-ha:huawei-pod2:compass": [8840.94], + + "os-odl-sfc-ha:huawei-pod2:compass": [8826.23], + + "os-nosdn-nofeature-noha:huawei-virtual4:compass": [8039.48], + + "os-nosdn-nofeature-noha:huawei-virtual2:compass": [7670.21], + + "os-nosdn-ovs-ha:arm-pod5:fuel": [7590.9], + + "os-odl-sfc-noha:huawei-virtual4:compass": [7579.625], + + "os-nosdn-bar-noha:huawei-virtual3:compass": [7511.775], + + "os-odl-nofeature-ha:arm-pod5:fuel": [7475.16], + + "os-nosdn-bar-noha:huawei-virtual4:compass": [7435.08], + + "os-nosdn-nofeature-noha:ericsson-virtual2:fuel": [7426.79], + + "os-nosdn-nofeature-ha:huawei-virtual4:compass": [7362.8], + + "os-nosdn-kvm-noha:huawei-virtual4:compass": [7263.45], + + "os-nosdn-nofeature-noha:huawei-virtual3:compass": [7262.72], + + "os-odl_l3-nofeature-noha:huawei-virtual3:compass": [7241.07], + + "os-odl-nofeature-noha:ericsson-virtual2:fuel": [7219.21], + + "os-nosdn-kvm-noha:huawei-virtual3:compass": [7174.33], + + "os-odl-sfc-noha:huawei-virtual3:compass": [7170.795], + + "os-odl-nofeature-noha:lf-pod1:apex": [7158.335], + + "os-nosdn-kvm-ha:huawei-pod2:compass": [7122.45], + + "os-odl-sfc-ha:huawei-virtual4:compass": [7104.9], + + "os-nosdn-ovs-noha:ericsson-virtual2:fuel": [7044.37], + + "os-nosdn-bar-ha:huawei-virtual3:compass": [7011.075], + + "os-nosdn-ovs-ha:ericsson-pod1:fuel": [6950.28], + + "os-nosdn-ovs-noha:ericsson-virtual4:fuel": [6918.31], + + "os-nosdn-bar-ha:huawei-virtual4:compass": [6903.11], + + "os-nosdn-nofeature-ha:lf-pod2:fuel": [6880.98], + + "os-odl-sfc-ha:lf-pod1:apex": [6863.39], + + "os-odl_l3-nofeature-ha:huawei-virtual3:compass": [6851.54], + + "os-nosdn-nofeature-noha:lf-pod1:apex": [6834.75], + + "os-nosdn-calipso-noha:lf-pod1:apex": [6833.92], + + "os-nosdn-ovs-ha:lf-pod2:fuel": [6814.68], + + "os-ovn-nofeature-noha:lf-pod1:apex": [6809.44], + + "os-odl_l3-nofeature-ha:huawei-virtual4:compass": [6784.48], + + "os-nosdn-nofeature-ha:lf-pod1:apex": [6737.64], + + "os-nosdn-bar-noha:lf-pod1:apex": [6708.61], + + "os-nosdn-bar-ha:lf-pod1:apex": [6697.2], + + "os-odl-nofeature-ha:lf-pod1:apex": [6626.51], + + "os-odl-sfc-noha:lf-pod1:apex": [6609.57], + + "os-odl-sfc-ha:huawei-virtual3:compass": [6606.87], + + "os-odl_l3-nofeature-noha:huawei-virtual4:compass": [6547.39], + + "os-odl-nofeature-ha:lf-pod2:fuel": [6465.48], + + "os-odl-nofeature-noha:ericsson-virtual4:fuel": [6413], + + "os-nosdn-kvm-ha:huawei-virtual4:compass": [6409.075], + + "os-nosdn-nofeature-ha:huawei-virtual3:compass": [6128.79], + + "os-nosdn-nofeature-noha:ericsson-virtual3:fuel": [5835.59], + + "os-nosdn-nofeature-ha:huawei-virtual1:compass": [5617.12] + +} + + +The influence of the scenario +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. image:: images/tc069_scenario_fraser.png + :width: 800px + :alt: TC069 influence of scenario + +{ + + "os-nosdn-openbaton-ha": [16774.52], + + "os-odl-nofeature-ha": [9363.69], + + "os-nosdn-nofeature-ha": [8878.01], + + "os-odl_l3-nofeature-ha": [8748.4], + + "os-odl-sfc-ha": [8708.045], + + "os-nosdn-nofeature-noha": [7426.79], + + "os-nosdn-kvm-noha": [7230.79], + + "os-odl-sfc-noha": [7224.11], + + "os-odl-nofeature-noha": [7187.84], + + "os-nosdn-ovs-noha": [7044.37], + + "os-nosdn-bar-ha": [6947.87], + + "os-odl_l3-nofeature-noha": [6895.96], + + "os-nosdn-kvm-ha": [6890.92], + + "os-nosdn-calipso-noha": [6833.92], + + "os-nosdn-ovs-ha": [6833.495], + + "os-nosdn-bar-noha": [6811.66], + + "os-ovn-nofeature-noha": [6809.44] + +} + + +The influence of the POD +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. image:: images/tc069_pod_fraser.png + :width: 800px + :alt: TC069 influence of the POD + +{ + + "intel-pod18": [16939.24], + + "arm-pod6": [11895.71], + + "zte-pod2": [9375.33], + + "ericsson-pod1": [9140.42], + + "huawei-pod12": [8993.37], + + "huawei-pod2": [8794.01], + + "huawei-virtual2": [7670.21], + + "arm-pod5": [7479.32], + + "ericsson-virtual2": [7219.21], + + "huawei-virtual4": [7059.045], + + "huawei-virtual3": [7023.57], + + "lf-pod2": [6834.7], + + "lf-pod1": [6775.27], + + "ericsson-virtual4": [6522.86], + + "ericsson-virtual3": [5835.59], + + "huawei-virtual1": [5617.12] + +} diff --git a/docs/release/results/tc082-context-switches-under-load.rst b/docs/release/results/tc082-context-switches-under-load.rst index d8a9f5493..92bc69907 100644 --- a/docs/release/results/tc082-context-switches-under-load.rst +++ b/docs/release/results/tc082-context-switches-under-load.rst @@ -70,8 +70,6 @@ The influence of the scenario :width: 800px :alt: TC082 influence of scenario -the influence of the scenario - { "os-nosdn-nofeature-ha": [505], @@ -88,8 +86,6 @@ The influence of the POD :width: 800px :alt: TC082 influence of the POD -the influence of the POD - { "huawei-pod12": [316], @@ -127,3 +123,65 @@ the influence of the POD Fraser release -------------- + +Test results per scenario and pod (lower is better): + +{ + + "os-nosdn-nofeature-ha:zte-pod2:daisy": [306.5], + + "os-nosdn-nofeature-ha:lf-pod2:fuel": [337.5], + + "os-nosdn-nofeature-ha:intel-pod18:joid": [343.5], + + "os-nosdn-nofeature-ha:huawei-pod12:joid": [399], + + "os-nosdn-nofeature-ha:lf-pod1:apex": [454], + + "os-nosdn-nofeature-ha:huawei-pod2:compass": [544.5], + + "os-nosdn-nofeature-ha:huawei-virtual4:compass": [1138], + + "os-nosdn-nofeature-ha:ericsson-pod1:fuel": [1305], + + "os-nosdn-nofeature-ha:huawei-virtual3:compass": [1433], + + "os-nosdn-nofeature-ha:huawei-virtual1:compass": [1470], + + "os-nosdn-nofeature-ha:arm-pod6:fuel": [1738.5] + +} + + +The influence of the POD +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. image:: images/tc082_pod_fraser.png + :width: 800px + :alt: TC082 influence of the POD + +{ + + "zte-pod2": [306.5], + + "lf-pod2": [337.5], + + "intel-pod18": [343.5], + + "huawei-pod12": [399], + + "lf-pod1": [454], + + "huawei-pod2": [544.5], + + "huawei-virtual4": [1138], + + "ericsson-pod1": [1305], + + "huawei-virtual3": [1433], + + "huawei-virtual1": [1470], + + "arm-pod6": [1738.5] + +} diff --git a/docs/release/results/tc083-network-throughput-between-vm.rst b/docs/release/results/tc083-network-throughput-between-vm.rst index f846571a5..0389eaafe 100644 --- a/docs/release/results/tc083-network-throughput-between-vm.rst +++ b/docs/release/results/tc083-network-throughput-between-vm.rst @@ -70,8 +70,6 @@ The influence of the scenario :width: 800px :alt: TC083 influence of scenario -the influence of the scenario - { "os-nosdn-nofeature-ha": [1109.12], @@ -88,8 +86,6 @@ The influence of the POD :width: 800px :alt: TC083 influence of the POD -the influence of the POD - { "lf-pod1": [2204.42], @@ -127,3 +123,65 @@ the influence of the POD Fraser release -------------- + +Test results per scenario and pod (higher is better): + +{ + + "os-nosdn-nofeature-ha:lf-pod2:fuel": [1893.39], + + "os-nosdn-nofeature-ha:zte-pod2:daisy": [1543.995], + + "os-nosdn-nofeature-ha:lf-pod1:apex": [1480.86], + + "os-nosdn-nofeature-ha:intel-pod18:joid": [1417.015], + + "os-nosdn-nofeature-ha:huawei-pod12:joid": [1028.55], + + "os-nosdn-nofeature-ha:huawei-pod2:compass": [1007.65], + + "os-nosdn-nofeature-ha:ericsson-pod1:fuel": [811.795], + + "os-nosdn-nofeature-ha:huawei-virtual4:compass": [552.95], + + "os-nosdn-nofeature-ha:arm-pod6:fuel": [227.655], + + "os-nosdn-nofeature-ha:huawei-virtual1:compass": [216.63], + + "os-nosdn-nofeature-ha:huawei-virtual3:compass": [59.255] + +} + + +The influence of the POD +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. image:: images/tc083_pod_fraser.png + :width: 800px + :alt: TC083 influence of the POD + +{ + + "lf-pod2": [1893.39], + + "zte-pod2": [1543.995], + + "lf-pod1": [1480.86], + + "intel-pod18": [1417.015], + + "huawei-pod12": [1028.55], + + "huawei-pod2": [1007.65], + + "ericsson-pod1": [811.795], + + "huawei-virtual4": [552.95], + + "arm-pod6": [227.655], + + "huawei-virtual1": [216.63], + + "huawei-virtual3": [59.255] + +} diff --git a/yardstick/tests/unit/common/test_utils.py b/yardstick/tests/unit/common/test_utils.py index 87995ac49..5fd91c87f 100644 --- a/yardstick/tests/unit/common/test_utils.py +++ b/yardstick/tests/unit/common/test_utils.py @@ -894,7 +894,7 @@ class TestUtils(unittest.TestCase): os.environ.clear() os.environ.update(base_env) - @mock.patch('yardstick.common.utils.configparser.ConfigParser') + @mock.patch.object(configparser, 'ConfigParser') def test_parse_ini_file(self, mock_config_parser_type): defaults = { 'default1': 'value1', @@ -926,23 +926,26 @@ class TestUtils(unittest.TestCase): result = utils.parse_ini_file('my_path') self.assertDictEqual(result, expected) - @mock.patch('yardstick.common.utils.configparser.ConfigParser') - def test_parse_ini_file_missing_section_header(self, mock_config_parser_type): + @mock.patch.object(utils, 'logger') + @mock.patch.object(configparser, 'ConfigParser') + def test_parse_ini_file_missing_section_header( + self, mock_config_parser_type, *args): mock_config_parser = mock_config_parser_type() - mock_config_parser.read.side_effect = \ - configparser.MissingSectionHeaderError(mock.Mock(), 321, mock.Mock()) + mock_config_parser.read.side_effect = ( + configparser.MissingSectionHeaderError(mock.Mock(), 321, + mock.Mock())) with self.assertRaises(configparser.MissingSectionHeaderError): utils.parse_ini_file('my_path') - @mock.patch('yardstick.common.utils.configparser.ConfigParser') + @mock.patch.object(configparser, 'ConfigParser') def test_parse_ini_file_no_file(self, mock_config_parser_type): mock_config_parser = mock_config_parser_type() mock_config_parser.read.return_value = False with self.assertRaises(RuntimeError): utils.parse_ini_file('my_path') - @mock.patch('yardstick.common.utils.configparser.ConfigParser') + @mock.patch.object(configparser, 'ConfigParser') def test_parse_ini_file_no_default_section_header(self, mock_config_parser_type): s1 = { 'key1': 'value11', diff --git a/yardstick/tests/unit/test_cmd/test_NSBperf.py b/yardstick/tests/unit/test_cmd/test_NSBperf.py index d64b0c551..5de892212 100644 --- a/yardstick/tests/unit/test_cmd/test_NSBperf.py +++ b/yardstick/tests/unit/test_cmd/test_NSBperf.py @@ -11,15 +11,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -from __future__ import absolute_import -import unittest -import mock -import subprocess +import argparse import os +import subprocess + +import mock +from six.moves import builtins +import unittest -from yardstick.cmd.NSBperf import YardstickNSCli from yardstick.cmd import NSBperf @@ -32,30 +32,39 @@ class TestHandler(unittest.TestCase): class TestYardstickNSCli(unittest.TestCase): + + def setUp(self): + self._mock_print = mock.patch.object(builtins, 'print') + self.mock_print = self._mock_print.start() + self.addCleanup(self._stop_mocks) + + def _stop_mocks(self): + self._mock_print.stop() + def test___init__(self): - yardstick_ns_cli = YardstickNSCli() + yardstick_ns_cli = NSBperf.YardstickNSCli() self.assertIsNotNone(yardstick_ns_cli) def test_generate_final_report(self): - yardstick_ns_cli = YardstickNSCli() + yardstick_ns_cli = NSBperf.YardstickNSCli() test_case = "tc_baremetal_rfc2544_ipv4_1flow_1518B.yaml" if os.path.isfile("/tmp/yardstick.out"): os.remove('/tmp/yardstick.out') self.assertIsNone(yardstick_ns_cli.generate_final_report(test_case)) def test_generate_kpi_results(self): - yardstick_ns_cli = YardstickNSCli() + yardstick_ns_cli = NSBperf.YardstickNSCli() tkey = "cpu" tgen = {"cpu": {"ipc": 0}} self.assertIsNone(yardstick_ns_cli.generate_kpi_results(tkey, tgen)) def test_generate_nfvi_results(self): - yardstick_ns_cli = YardstickNSCli() + yardstick_ns_cli = NSBperf.YardstickNSCli() nfvi = {"collect_stats": {"cpu": {"ipc": 0, "Hz": 2.6}}} self.assertIsNone(yardstick_ns_cli.generate_nfvi_results(nfvi)) def test_handle_list_options(self): - yardstick_ns_cli = YardstickNSCli() + yardstick_ns_cli = NSBperf.YardstickNSCli() CLI_PATH = os.path.dirname(os.path.realpath(__file__)) repo_dir = CLI_PATH + "/../../../" test_path = os.path.join(repo_dir, "../samples/vnf_samples/nsut/") @@ -68,16 +77,21 @@ class TestYardstickNSCli(unittest.TestCase): args, test_path) def test_main(self): - yardstick_ns_cli = YardstickNSCli() + yardstick_ns_cli = NSBperf.YardstickNSCli() yardstick_ns_cli.parse_arguments = mock.Mock(return_value=0) yardstick_ns_cli.handle_list_options = mock.Mock(return_value=0) yardstick_ns_cli.terminate_if_less_options = mock.Mock(return_value=0) yardstick_ns_cli.run_test = mock.Mock(return_value=0) self.assertIsNone(yardstick_ns_cli.main()) - def test_parse_arguments(self): - yardstick_ns_cli = YardstickNSCli() - self.assertRaises(SystemExit, yardstick_ns_cli.parse_arguments) + @mock.patch.object(argparse.ArgumentParser, 'parse_args') + def test_parse_arguments(self, mock_parse): + class DummyArgs(object): + var1 = 'value1' + + mock_parse.return_value = DummyArgs + yardstick_ns_cli = NSBperf.YardstickNSCli() + self.assertIn('var1', yardstick_ns_cli.parse_arguments()) def test_run_test(self): cur_dir = os.getcwd() @@ -85,7 +99,7 @@ class TestYardstickNSCli(unittest.TestCase): YARDSTICK_REPOS_DIR = os.path.join(CLI_PATH + "/../../") test_path = os.path.join(YARDSTICK_REPOS_DIR, "../samples/vnf_samples/nsut/") - yardstick_ns_cli = YardstickNSCli() + yardstick_ns_cli = NSBperf.YardstickNSCli() subprocess.check_output = mock.Mock(return_value=0) args = {"vnf": "vpe", "test": "tc_baremetal_rfc2544_ipv4_1flow_1518B.yaml"} @@ -103,13 +117,13 @@ class TestYardstickNSCli(unittest.TestCase): os.chdir(cur_dir) def test_terminate_if_less_options(self): - yardstick_ns_cli = YardstickNSCli() + yardstick_ns_cli = NSBperf.YardstickNSCli() args = {"vnf": False} self.assertRaises(SystemExit, yardstick_ns_cli.terminate_if_less_options, args) def test_validate_input(self): - yardstick_ns_cli = YardstickNSCli() + yardstick_ns_cli = NSBperf.YardstickNSCli() self.assertEqual(1, yardstick_ns_cli.validate_input("", 4)) NSBperf.input = lambda _: 'yes' self.assertEqual(1, yardstick_ns_cli.validate_input(5, 4)) |
