diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | docs/testing/developer/devguide/design/vswitchperf_design.rst | 2 | ||||
| -rw-r--r-- | docs/testing/developer/devguide/index.rst | 6 | ||||
| -rw-r--r-- | docs/testing/user/configguide/index.rst | 1 | ||||
| -rw-r--r-- | docs/testing/user/userguide/index.rst | 15 | ||||
| -rw-r--r-- | docs/testing/user/userguide/testlist.rst | 109 | ||||
| -rw-r--r-- | src/trex/Makefile | 3 | ||||
| -rw-r--r-- | tools/opnfvdashboard/opnfvdashboard.py | 77 | ||||
| -rwxr-xr-x | vsperf | 10 |
9 files changed, 192 insertions, 32 deletions
@@ -67,6 +67,7 @@ target/ /src/vpp/vpp/ /src_cuse/ /src_vanilla/ +/src/trex/trex .vagrant tags diff --git a/docs/testing/developer/devguide/design/vswitchperf_design.rst b/docs/testing/developer/devguide/design/vswitchperf_design.rst index 8111b513..58563845 100644 --- a/docs/testing/developer/devguide/design/vswitchperf_design.rst +++ b/docs/testing/developer/devguide/design/vswitchperf_design.rst @@ -23,7 +23,7 @@ Example Connectivity to DUT Establish connectivity to the VSPERF DUT Linux host. If this is in an OPNFV lab following the steps provided by `Pharos <https://www.opnfv.org/community/projects/pharos>`_ - to `access the POD <https://wiki.opnfv.org/display/pharos/Pharos+Lab+Support>`_ +to `access the POD <https://wiki.opnfv.org/display/pharos/Pharos+Lab+Support>`_ The followign steps establish the VSPERF environment. diff --git a/docs/testing/developer/devguide/index.rst b/docs/testing/developer/devguide/index.rst index 4ae155f9..9af8eb81 100644 --- a/docs/testing/developer/devguide/index.rst +++ b/docs/testing/developer/devguide/index.rst @@ -56,9 +56,9 @@ VSPERF IETF RFC 8204 :maxdepth: 2 :numbered: -The VSPERF test specification was summarized in a 23 page Internet Draft ... `Benchmarking Virtual Switches in OPNFV -<https://tools.ietf.org/html/draft-ietf-bmwg-vswitch-opnfv-01>`_ - which was contributed to the IETF Benchmarking Methodology Working Group (BMWG). The BMWG was re-chartered in 2014 +The VSPERF test specification was summarized in a 23 page Internet Draft ... +`Benchmarking Virtual Switches in OPNFV <https://tools.ietf.org/html/draft-ietf-bmwg-vswitch-opnfv-01>`_ +which was contributed to the IETF Benchmarking Methodology Working Group (BMWG). The BMWG was re-chartered in 2014 to include benchmarking for Virtualized Network Functions (VNFs) and their infrastructure. The Internet Engineering Steering Group of the IETF has approved the most recent version for publication as RFC 8204. diff --git a/docs/testing/user/configguide/index.rst b/docs/testing/user/configguide/index.rst index bfeb0d2b..b5a37692 100644 --- a/docs/testing/user/configguide/index.rst +++ b/docs/testing/user/configguide/index.rst @@ -58,3 +58,4 @@ VSPERF Test Guide ../userguide/teststeps.rst ../userguide/integration.rst ../userguide/yardstick.rst + ../userguide/testlist.rst diff --git a/docs/testing/user/userguide/index.rst b/docs/testing/user/userguide/index.rst index 4443a393..64d91657 100644 --- a/docs/testing/user/userguide/index.rst +++ b/docs/testing/user/userguide/index.rst @@ -3,3 +3,18 @@ .. (c) OPNFV, Intel Corporation, AT&T, Red Hat, Spirent, Ixia and others. .. OPNFV VSPERF Documentation master file. + +================= +VSPERF Test Guide +================= + +.. toctree:: + :caption: VSPERF Test Execution + :maxdepth: 2 + :numbered: + + ./testusage.rst + ./teststeps.rst + ./integration.rst + ./yardstick.rst + ./testlist.rst diff --git a/docs/testing/user/userguide/testlist.rst b/docs/testing/user/userguide/testlist.rst new file mode 100644 index 00000000..4b535494 --- /dev/null +++ b/docs/testing/user/userguide/testlist.rst @@ -0,0 +1,109 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) OPNFV, Intel Corporation, AT&T and others. + +List of vswitchperf testcases +----------------------------- + +Performance testcases +^^^^^^^^^^^^^^^^^^^^^ + +============================= ==================================================================== +Testcase Name Description +============================= ==================================================================== +phy2phy_tput LTD.Throughput.RFC2544.PacketLossRatio +phy2phy_forwarding LTD.Forwarding.RFC2889.MaxForwardingRate +phy2phy_learning LTD.AddrLearning.RFC2889.AddrLearningRate +phy2phy_caching LTD.AddrCaching.RFC2889.AddrCachingCapacity +back2back LTD.Throughput.RFC2544.BackToBackFrames +phy2phy_tput_mod_vlan LTD.Throughput.RFC2544.PacketLossRatioFrameModification +phy2phy_cont Phy2Phy Continuous Stream +pvp_cont PVP Continuous Stream +pvvp_cont PVVP Continuous Stream +pvpv_cont Two VMs in parallel with Continuous Stream +phy2phy_scalability LTD.Scalability.Flows.RFC2544.0PacketLoss +pvp_tput LTD.Throughput.RFC2544.PacketLossRatio +pvp_back2back LTD.Throughput.RFC2544.BackToBackFrames +pvvp_tput LTD.Throughput.RFC2544.PacketLossRatio +pvvp_back2back LTD.Throughput.RFC2544.BackToBackFrames +phy2phy_cpu_load LTD.CPU.RFC2544.0PacketLoss +phy2phy_mem_load LTD.Memory.RFC2544.0PacketLoss +phy2phy_tput_vpp VPP: LTD.Throughput.RFC2544.PacketLossRatio +phy2phy_cont_vpp VPP: Phy2Phy Continuous Stream +phy2phy_back2back_vpp VPP: LTD.Throughput.RFC2544.BackToBackFrames +pvp_tput_vpp VPP: LTD.Throughput.RFC2544.PacketLossRatio +pvp_cont_vpp VPP: PVP Continuous Stream +pvp_back2back_vpp VPP: LTD.Throughput.RFC2544.BackToBackFrames +pvvp_tput_vpp VPP: LTD.Throughput.RFC2544.PacketLossRatio +pvvp_cont_vpp VPP: PVP Continuous Stream +pvvp_back2back_vpp VPP: LTD.Throughput.RFC2544.BackToBackFrames +============================= ==================================================================== + +List of performance testcases above can be obtained by execution of: + +.. code-block:: bash + + $ ./vsperf --list + + +Integration testcases +^^^^^^^^^^^^^^^^^^^^^ + +====================================== ======================================================================================== +Testcase Name Description +====================================== ======================================================================================== +vswitch_vports_add_del_flow vSwitch - configure switch with vports, add and delete flow +vswitch_add_del_flows vSwitch - add and delete flows +vswitch_p2p_tput vSwitch - configure switch and execute RFC2544 throughput test +vswitch_p2p_back2back vSwitch - configure switch and execute RFC2544 back2back test +vswitch_p2p_cont vSwitch - configure switch and execute RFC2544 continuous stream test +vswitch_pvp vSwitch - configure switch and one vnf +vswitch_vports_pvp vSwitch - configure switch with vports and one vnf +vswitch_pvp_tput vSwitch - configure switch, vnf and execute RFC2544 throughput test +vswitch_pvp_back2back vSwitch - configure switch, vnf and execute RFC2544 back2back test +vswitch_pvp_cont vSwitch - configure switch, vnf and execute RFC2544 continuous stream test +vswitch_pvp_all vSwitch - configure switch, vnf and execute all test types +vswitch_pvvp vSwitch - configure switch and two vnfs +vswitch_pvvp_tput vSwitch - configure switch, two chained vnfs and execute RFC2544 throughput test +vswitch_pvvp_back2back vSwitch - configure switch, two chained vnfs and execute RFC2544 back2back test +vswitch_pvvp_cont vSwitch - configure switch, two chained vnfs and execute RFC2544 continuous stream test +vswitch_pvvp_all vSwitch - configure switch, two chained vnfs and execute all test types +vswitch_p4vp Just configure 4 chained vnfs +vswitch_p4vp_tput 4 chained vnfs, execute RFC2544 throughput test +vswitch_p4vp_back2back 4 chained vnfs, execute RFC2544 back2back test +vswitch_p4vp_cont 4 chained vnfs, execute RFC2544 continuous stream test +vswitch_p4vp_all 4 chained vnfs, execute RFC2544 throughput test +2pvp_udp_dest_flows RFC2544 Continuous TC with 2 Parallel VMs, flows on UDP Dest Port +4pvp_udp_dest_flows RFC2544 Continuous TC with 4 Parallel VMs, flows on UDP Dest Port +6pvp_udp_dest_flows RFC2544 Continuous TC with 6 Parallel VMs, flows on UDP Dest Port +vhost_numa_awareness vSwitch DPDK - verify that PMD threads are served by the same NUMA slot as QEMU instances +ixnet_pvp_tput_1nic PVP Scenario with 1 port towards IXIA +vswitch_vports_add_del_connection_vpp VPP: vSwitch - configure switch with vports, add and delete connection +p2p_l3_multi_IP_ovs OVS: P2P L3 multistream with unique flow for each IP stream +p2p_l3_multi_IP_mask_ovs OVS: P2P L3 multistream with 1 flow for /8 net mask +pvp_l3_multi_IP_mask_ovs OVS: PVP L3 multistream with 1 flow for /8 net mask +pvvp_l3_multi_IP_mask_ovs OVS: PVVP L3 multistream with 1 flow for /8 net mask +p2p_l4_multi_PORT_ovs OVS: P2P L4 multistream with unique flow for each IP stream +p2p_l4_multi_PORT_mask_ovs OVS: P2P L4 multistream with 1 flow for /8 net and port mask +pvp_l4_multi_PORT_mask_ovs OVS: PVP L4 multistream flows for /8 net and port mask +pvvp_l4_multi_PORT_mask_ovs OVS: PVVP L4 multistream with flows for /8 net and port mask +p2p_l3_multi_IP_arp_vpp VPP: P2P L3 multistream with unique ARP entry for each IP stream +p2p_l3_multi_IP_mask_vpp VPP: P2P L3 multistream with 1 route for /8 net mask +p2p_l3_multi_IP_routes_vpp VPP: P2P L3 multistream with unique route for each IP stream +pvp_l3_multi_IP_mask_vpp VPP: PVP L3 multistream with route for /8 netmask +pvvp_l3_multi_IP_mask_vpp VPP: PVVP L3 multistream with route for /8 netmask +p2p_l4_multi_PORT_arp_vpp VPP: P2P L4 multistream with unique ARP entry for each IP stream and port check +p2p_l4_multi_PORT_mask_vpp VPP: P2P L4 multistream with 1 route for /8 net mask and port check +p2p_l4_multi_PORT_routes_vpp VPP: P2P L4 multistream with unique route for each IP stream and port check +pvp_l4_multi_PORT_mask_vpp VPP: PVP L4 multistream with route for /8 net and port mask +pvvp_l4_multi_PORT_mask_vpp VPP: PVVP L4 multistream with route for /8 net and port mask +vxlan_multi_IP_mask_ovs OVS: VxLAN L3 multistream +vxlan_multi_IP_arp_vpp VPP: VxLAN L3 multistream with unique ARP entry for each IP stream +vxlan_multi_IP_mask_vpp VPP: VxLAN L3 multistream with 1 route for /8 netmask +====================================== ======================================================================================== + +List of integration testcases above can be obtained by execution of: + +.. code-block:: bash + + $ ./vsperf --integration --list diff --git a/src/trex/Makefile b/src/trex/Makefile index 9aaaa203..41eb52ab 100644 --- a/src/trex/Makefile +++ b/src/trex/Makefile @@ -26,7 +26,7 @@ TAG_DONE_FLAG = $(WORK_DIR)/.$(TREX_TAG).done all: force_pull @echo "Finished pulling $(WORK_DIR) " -force_pull: $(WORK_DIR) Makefile +force_pull: $(TAG_DONE_FLAG) $(AT)cd $(WORK_DIR) && git pull $(TREX_URL) $(TREX_TAG) @echo "git pull done" @@ -35,6 +35,7 @@ $(WORK_DIR): $(TAG_DONE_FLAG): $(WORK_DIR) $(AT)cd $(WORK_DIR); git checkout $(TREX_TAG) + $(AT)touch $@ install: @echo "Make install in $(WORK_DIR) (stub) " diff --git a/tools/opnfvdashboard/opnfvdashboard.py b/tools/opnfvdashboard/opnfvdashboard.py index cf2fd1b0..a77a9c3a 100644 --- a/tools/opnfvdashboard/opnfvdashboard.py +++ b/tools/opnfvdashboard/opnfvdashboard.py @@ -17,10 +17,14 @@ vsperf2dashboard import os import csv +import copy import logging +from datetime import datetime as dt import requests -def results2opnfv_dashboard(results_path, int_data): +_DETAILS = {"64": '', "128": '', "512": '', "1024": '', "1518": ''} + +def results2opnfv_dashboard(tc_names, results_path, int_data): """ the method open the csv file with results and calls json encoder """ @@ -31,23 +35,33 @@ def results2opnfv_dashboard(results_path, int_data): resfile = results_path + '/' + test with open(resfile, 'r') as in_file: reader = csv.DictReader(in_file) - _push_results(reader, int_data) + tc_data = _prepare_results(reader, int_data) + _push_results(tc_data) + tc_names.remove(tc_data['id']) + + # report TCs without results as FAIL + if tc_names: + tc_fail = copy.deepcopy(int_data) + tc_fail['start_time'] = dt.now().strftime('%Y-%m-%d %H:%M:%S') + tc_fail['stop_time'] = tc_fail['start_time'] + tc_fail['criteria'] = 'FAIL' + tc_fail['version'] = 'N/A' + tc_fail['details'] = copy.deepcopy(_DETAILS) + for tc_name in tc_names: + tc_fail['dashboard_id'] = "{}_{}".format(tc_name, tc_fail['vswitch']) + _push_results(tc_fail) -def _push_results(reader, int_data): +def _prepare_results(reader, int_data): """ - the method encodes results and sends them into opnfv dashboard + the method prepares dashboard details for passed testcases """ - db_url = int_data['db_url'] - url = db_url + "/results" - casename = "" version_vswitch = "" version_dpdk = "" - version = "" allowed_pkt = ["64", "128", "512", "1024", "1518"] - details = {"64": '', "128": '', "512": '', "1024": '', "1518": ''} - test_start = None - test_stop = None vswitch = None + details = copy.deepcopy(_DETAILS) + tc_data = copy.deepcopy(int_data) + tc_data['criteria'] = 'PASS' for row_reader in reader: if allowed_pkt.count(row_reader['packet_size']) == 0: @@ -56,20 +70,25 @@ def _push_results(reader, int_data): # test execution time includes all frame sizes, so start & stop time # is the same (repeated) for every framesize in CSV file - if test_start is None: - test_start = row_reader['start_time'] - test_stop = row_reader['stop_time'] + if not 'test_start' in tc_data: + tc_data['start_time'] = row_reader['start_time'] + tc_data['stop_time'] = row_reader['stop_time'] + tc_data['id'] = row_reader['id'] # CI job executes/reports TCs per vswitch type vswitch = row_reader['vswitch'] - casename = "{}_{}".format(row_reader['id'], row_reader['vswitch'].lower()) + tc_data['dashboard_id'] = "{}_{}".format(row_reader['id'], row_reader['vswitch'].lower()) if "back2back" in row_reader['id']: + # 0 B2B frames is quite common, so we can't mark such TC as FAIL details[row_reader['packet_size']] = row_reader['b2b_frames'] else: details[row_reader['packet_size']] = row_reader['throughput_rx_fps'] + # 0 PPS is definitelly a failure + if float(row_reader['throughput_rx_fps']) == 0: + tc_data['criteria'] = 'FAIL' # Create version field - with open(int_data['pkg_list'], 'r') as pkg_file: + with open(tc_data['pkg_list'], 'r') as pkg_file: for line in pkg_file: if "OVS_TAG" in line and vswitch.startswith('Ovs'): version_vswitch = line.replace(' ', '') @@ -83,23 +102,33 @@ def _push_results(reader, int_data): version_dpdk = line.replace(' ', '') version_dpdk = " DPDK {}".format( version_dpdk.replace('DPDK_TAG?=', '')) - version = version_vswitch.replace('\n', '') + version_dpdk.replace('\n', '') + + tc_data['details'] = details + tc_data['version'] = version_vswitch.replace('\n', '') + version_dpdk.replace('\n', '') + + return tc_data + +def _push_results(int_data): + """ + the method sends testcase details into dashboard database + """ + url = int_data['db_url'] + "/results" # Build body body = {"project_name": "vsperf", "scenario": "vsperf", - "start_date": test_start, - "stop_date": test_stop, - "case_name": casename, + "start_date": int_data['start_time'], + "stop_date": int_data['stop_time'], + "case_name": int_data['dashboard_id'], "pod_name": int_data['pod'], "installer": int_data['installer'], - "version": version, + "version": int_data['version'], "build_tag": int_data['build_tag'], "criteria": int_data['criteria'], - "details": details} + "details": int_data['details']} my_data = requests.post(url, json=body) - logging.info("Results for %s sent to opnfv, http response: %s", casename, my_data) - logging.debug("opnfv url: %s", db_url) + logging.info("Results for %s sent to opnfv, http response: %s", int_data['dashboard_id'], my_data) + logging.debug("opnfv url: %s", int_data['db_url']) logging.debug("the body sent to opnfv") logging.debug(body) @@ -708,12 +708,16 @@ def main(): pkg_list = settings.getValue('PACKAGE_LIST') int_data = {'pod': pod_name, - 'criteria': "PASS", 'build_tag': get_build_tag(), 'installer': installer_name, 'pkg_list': pkg_list, - 'db_url': opnfv_url} - opnfvdashboard.results2opnfv_dashboard(results_path, int_data) + 'db_url': opnfv_url, + # pass vswitch name from configuration to be used for failed + # TCs; In case of successful TCs it is safer to use vswitch + # name from CSV as TC can override global configuration + 'vswitch': str(settings.getValue('VSWITCH')).lower()} + tc_names = [tc['Name'] for tc in selected_tests] + opnfvdashboard.results2opnfv_dashboard(tc_names, results_path, int_data) # cleanup before exit vsperf_finalize() |