From c17fe61eba519ca1c49aee2214438f1a248fe9f8 Mon Sep 17 00:00:00 2001 From: Trevor Cooper <tcooper@localhost.localdomain> Date: Sun, 1 Oct 2017 21:51:47 -0700 Subject: Removed drafts of test specificaiton now published as RFC8204. Also removed IETF license notice and added a file with link to RFC8204 Change-Id: I44b4cb75afeff74a58f402a7cb8816ddfbe52b87 Signed-off-by: Trevor Cooper <trevor.cooper@intel.com> --- .../devguide/requirements/ietf_draft/LICENSE | 12 - .../draft-ietf-bmwg-vswitch-opnfv-00.xml | 1016 ------------------- .../draft-ietf-bmwg-vswitch-opnfv-01.xml | 1027 -------------------- .../draft-vsperf-bmwg-vswitch-opnfv-00.xml | 964 ------------------ .../draft-vsperf-bmwg-vswitch-opnfv-01.xml | 1016 ------------------- .../draft-vsperf-bmwg-vswitch-opnfv-02.xml | 1016 ------------------- .../rfc8204-vsperf-bmwg-vswitch-opnfv.rst | 19 + 7 files changed, 19 insertions(+), 5051 deletions(-) delete mode 100644 docs/testing/developer/devguide/requirements/ietf_draft/LICENSE delete mode 100644 docs/testing/developer/devguide/requirements/ietf_draft/draft-ietf-bmwg-vswitch-opnfv-00.xml delete mode 100644 docs/testing/developer/devguide/requirements/ietf_draft/draft-ietf-bmwg-vswitch-opnfv-01.xml delete mode 100644 docs/testing/developer/devguide/requirements/ietf_draft/draft-vsperf-bmwg-vswitch-opnfv-00.xml delete mode 100644 docs/testing/developer/devguide/requirements/ietf_draft/draft-vsperf-bmwg-vswitch-opnfv-01.xml delete mode 100644 docs/testing/developer/devguide/requirements/ietf_draft/draft-vsperf-bmwg-vswitch-opnfv-02.xml create mode 100644 docs/testing/developer/devguide/requirements/ietf_draft/rfc8204-vsperf-bmwg-vswitch-opnfv.rst diff --git a/docs/testing/developer/devguide/requirements/ietf_draft/LICENSE b/docs/testing/developer/devguide/requirements/ietf_draft/LICENSE deleted file mode 100644 index 7fc9ae14..00000000 --- a/docs/testing/developer/devguide/requirements/ietf_draft/LICENSE +++ /dev/null @@ -1,12 +0,0 @@ -Copyright (c) 2016 IETF Trust and the persons identified as the -document authors. All rights reserved. - -This document is subject to BCP 78 and the IETF Trust's Legal -Provisions Relating to IETF Documents -(http://trustee.ietf.org/license-info) in effect on the date of -publication of this document. Please review these documents -carefully, as they describe your rights and restrictions with respect -to this document. Code Components extracted from this document must -include Simplified BSD License text as described in Section 4.e of -the Trust Legal Provisions and are provided without warranty as -described in the Simplified BSD License. diff --git a/docs/testing/developer/devguide/requirements/ietf_draft/draft-ietf-bmwg-vswitch-opnfv-00.xml b/docs/testing/developer/devguide/requirements/ietf_draft/draft-ietf-bmwg-vswitch-opnfv-00.xml deleted file mode 100644 index 2259b23c..00000000 --- a/docs/testing/developer/devguide/requirements/ietf_draft/draft-ietf-bmwg-vswitch-opnfv-00.xml +++ /dev/null @@ -1,1016 +0,0 @@ -<?xml version="1.0" encoding="US-ASCII"?> -<!DOCTYPE rfc SYSTEM "rfc2629.dtd"> -<?rfc toc="yes"?> -<?rfc tocompact="yes"?> -<?rfc tocdepth="3"?> -<?rfc tocindent="yes"?> -<?rfc symrefs="yes"?> -<?rfc sortrefs="yes"?> -<?rfc comments="yes"?> -<?rfc inline="yes"?> -<?rfc compact="yes"?> -<?rfc subcompact="no"?> -<rfc category="info" docName="draft-ietf-bmwg-vswitch-opnfv-00" - ipr="trust200902"> - <front> - <title abbrev="Benchmarking vSwitches">Benchmarking Virtual Switches in - OPNFV</title> - - <author fullname="Maryam Tahhan" initials="M." surname="Tahhan"> - <organization>Intel</organization> - - <address> - <postal> - <street/> - - <city/> - - <region/> - - <code/> - - <country/> - </postal> - - <phone/> - - <facsimile/> - - <email>maryam.tahhan@intel.com</email> - - <uri/> - </address> - </author> - - <author fullname="Billy O'Mahony" initials="B." surname="O'Mahony"> - <organization>Intel</organization> - - <address> - <postal> - <street/> - - <city/> - - <region/> - - <code/> - - <country/> - </postal> - - <phone/> - - <facsimile/> - - <email>billy.o.mahony@intel.com</email> - - <uri/> - </address> - </author> - - <author fullname="Al Morton" initials="A." surname="Morton"> - <organization>AT&T Labs</organization> - - <address> - <postal> - <street>200 Laurel Avenue South</street> - - <city>Middletown,</city> - - <region>NJ</region> - - <code>07748</code> - - <country>USA</country> - </postal> - - <phone>+1 732 420 1571</phone> - - <facsimile>+1 732 368 1192</facsimile> - - <email>acmorton@att.com</email> - - <uri>http://home.comcast.net/~acmacm/</uri> - </address> - </author> - - <date day="8" month="July" year="2016"/> - - <abstract> - <t>This memo describes the progress of the Open Platform for NFV (OPNFV) - project on virtual switch performance "VSWITCHPERF". This project - intends to build on the current and completed work of the Benchmarking - Methodology Working Group in IETF, by referencing existing literature. - The Benchmarking Methodology Working Group has traditionally conducted - laboratory characterization of dedicated physical implementations of - internetworking functions. Therefore, this memo begins to describe the - additional considerations when virtual switches are implemented in - general-purpose hardware. The expanded tests and benchmarks are also - influenced by the OPNFV mission to support virtualization of the "telco" - infrastructure.</t> - </abstract> - - <note title="Requirements Language"> - <t>The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", - "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this - document are to be interpreted as described in <xref - target="RFC2119">RFC 2119</xref>.</t> - - <t/> - </note> - </front> - - <middle> - <section title="Introduction"> - <t>Benchmarking Methodology Working Group (BMWG) has traditionally - conducted laboratory characterization of dedicated physical - implementations of internetworking functions. The Black-box Benchmarks - of Throughput, Latency, Forwarding Rates and others have served our - industry for many years. Now, Network Function Virtualization (NFV) has - the goal to transform how internetwork functions are implemented, and - therefore has garnered much attention.</t> - - <t>This memo summarizes the progress of the Open Platform for NFV - (OPNFV) project on virtual switch performance characterization, - "VSWITCHPERF", through the Brahmaputra (second) release <xref - target="BrahRel"/>. This project intends to build on the current and - completed work of the Benchmarking Methodology Working Group in IETF, by - referencing existing literature. For example, currently the most often - referenced RFC is <xref target="RFC2544"/> (which depends on <xref - target="RFC1242"/>) and foundation of the benchmarking work in OPNFV is - common and strong.</t> - - <t>See - https://wiki.opnfv.org/characterize_vswitch_performance_for_telco_nfv_use_cases - for more background, and the OPNFV website for general information: - https://www.opnfv.org/</t> - - <t>The authors note that OPNFV distinguishes itself from other open - source compute and networking projects through its emphasis on existing - "telco" services as opposed to cloud-computing. There are many ways in - which telco requirements have different emphasis on performance - dimensions when compared to cloud computing: support for and transfer of - isochronous media streams is one example.</t> - - <t>Note also that the move to NFV Infrastructure has resulted in many - new benchmarking initiatives across the industry. The authors are - currently doing their best to maintain alignment with many other - projects, and this Internet Draft is one part of the efforts. We - acknowledge the early work in <xref - target="I-D.huang-bmwg-virtual-network-performance"/>, and useful - discussion with the authors.</t> - </section> - - <section title="Scope"> - <t>The primary purpose and scope of the memo is to inform the industry - of work-in-progress that builds on the body of extensive BMWG literature - and experience, and describe the extensions needed for benchmarking - virtual switches. Inital feedback indicates that many of these - extensions may be applicable beyond the current scope (to hardware - switches in the NFV Infrastructure and to virtual routers, for example). - Additionally, this memo serves as a vehicle to include more detail and - commentary from BMWG and other Open Source communities, under BMWG's - chartered work to characterize the NFV Infrastructure (a virtual switch - is an important aspect of that infrastructure).</t> - </section> - - <section title="Benchmarking Considerations"> - <t>This section highlights some specific considerations (from <xref - target="I-D.ietf-bmwg-virtual-net"/>)related to Benchmarks for virtual - switches. The OPNFV project is sharing its present view on these areas, - as they develop their specifications in the Level Test Design (LTD) - document.</t> - - <section title="Comparison with Physical Network Functions"> - <t>To compare the performance of virtual designs and implementations - with their physical counterparts, identical benchmarks are needed. - BMWG has developed specifications for many network functions this memo - re-uses existing benchmarks through references, and expands them - during development of new methods. A key configuration aspect is the - number of parallel cores required to achieve comparable performance - with a given physical device, or whether some limit of scale was - reached before the cores could achieve the comparable level.</t> - - <t>It's unlikely that the virtual switch will be the only application - running on the SUT, so CPU utilization, Cache utilization, and Memory - footprint should also be recorded for the virtual implementations of - internetworking functions.</t> - </section> - - <section title="Continued Emphasis on Black-Box Benchmarks"> - <t>External observations remain essential as the basis for Benchmarks. - Internal observations with fixed specification and interpretation will - be provided in parallel to assist the development of operations - procedures when the technology is deployed.</t> - </section> - - <section title="New Configuration Parameters"> - <t>A key consideration when conducting any sort of benchmark is trying - to ensure the consistency and repeatability of test results. When - benchmarking the performance of a vSwitch there are many factors that - can affect the consistency of results, one key factor is matching the - various hardware and software details of the SUT. This section lists - some of the many new parameters which this project believes are - critical to report in order to achieve repeatability.</t> - - <t>Hardware details including:</t> - - <t><list style="symbols"> - <t>Platform details</t> - - <t>Processor details</t> - - <t>Memory information (type and size)</t> - - <t>Number of enabled cores</t> - - <t>Number of cores used for the test</t> - - <t>Number of physical NICs, as well as their details - (manufacturer, versions, type and the PCI slot they are plugged - into)</t> - - <t>NIC interrupt configuration</t> - - <t>BIOS version, release date and any configurations that were - modified</t> - - <t>CPU microcode level</t> - - <t>Memory DIMM configurations (quad rank performance may not be - the same as dual rank) in size, freq and slot locations</t> - - <t>PCI configuration parameters (payload size, early ack - option...)</t> - - <t>Power management at all levels (ACPI sleep states, processor - package, OS...)</t> - </list>Software details including:</t> - - <t><list style="symbols"> - <t>OS parameters and behavior (text vs graphical no one typing at - the console on one system)</t> - - <t>OS version (for host and VNF)</t> - - <t>Kernel version (for host and VNF)</t> - - <t>GRUB boot parameters (for host and VNF)</t> - - <t>Hypervisor details (Type and version)</t> - - <t>Selected vSwitch, version number or commit id used</t> - - <t>vSwitch launch command line if it has been parameterised</t> - - <t>Memory allocation to the vSwitch</t> - - <t>which NUMA node it is using, and how many memory channels</t> - - <t>DPDK or any other SW dependency version number or commit id - used</t> - - <t>Memory allocation to a VM - if it's from Hugpages/elsewhere</t> - - <t>VM storage type: snapshot/independent persistent/independent - non-persistent</t> - - <t>Number of VMs</t> - - <t>Number of Virtual NICs (vNICs), versions, type and driver</t> - - <t>Number of virtual CPUs and their core affinity on the host</t> - - <t>Number vNIC interrupt configuration</t> - - <t>Thread affinitization for the applications (including the - vSwitch itself) on the host</t> - - <t>Details of Resource isolation, such as CPUs designated for - Host/Kernel (isolcpu) and CPUs designated for specific processes - (taskset). - Test duration. - Number of flows.</t> - </list></t> - - <t>Test Traffic Information:<list style="symbols"> - <t>Traffic type - UDP, TCP, IMIX / Other</t> - - <t>Packet Sizes</t> - - <t>Deployment Scenario</t> - </list></t> - - <t/> - </section> - - <section title="Flow classification"> - <t>Virtual switches group packets into flows by processing and - matching particular packet or frame header information, or by matching - packets based on the input ports. Thus a flow can be thought of a - sequence of packets that have the same set of header field values - (5-tuple) or have arrived on the same port. Performance results can - vary based on the parameters the vSwitch uses to match for a flow. The - recommended flow classification parameters for any vSwitch performance - tests are: the input port, the source IP address, the destination IP - address and the Ethernet protocol type field. It is essential to - increase the flow timeout time on a vSwitch before conducting any - performance tests that do not measure the flow setup time. Normally - the first packet of a particular stream will install the flow in the - virtual switch which adds an additional latency, subsequent packets of - the same flow are not subject to this latency if the flow is already - installed on the vSwitch.</t> - </section> - - <section title="Benchmarks using Baselines with Resource Isolation"> - <t>This outline describes measurement of baseline with isolated - resources at a high level, which is the intended approach at this - time.</t> - - <t><list style="numbers"> - <t>Baselines: <list style="symbols"> - <t>Optional: Benchmark platform forwarding capability without - a vswitch or VNF for at least 72 hours (serves as a means of - platform validation and a means to obtain the base performance - for the platform in terms of its maximum forwarding rate and - latency). <figure> - <preamble>Benchmark platform forwarding - capability</preamble> - - <artwork align="right"><![CDATA[ __ - +--------------------------------------------------+ | - | +------------------------------------------+ | | - | | | | | - | | Simple Forwarding App | | Host - | | | | | - | +------------------------------------------+ | | - | | NIC | | | - +---+------------------------------------------+---+ __| - ^ : - | | - : v - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - - <postamble/> - </figure></t> - - <t>Benchmark VNF forwarding capability with direct - connectivity (vSwitch bypass, e.g., SR/IOV) for at least 72 - hours (serves as a means of VNF validation and a means to - obtain the base performance for the VNF in terms of its - maximum forwarding rate and latency). The metrics gathered - from this test will serve as a key comparison point for - vSwitch bypass technologies performance and vSwitch - performance. <figure align="right"> - <preamble>Benchmark VNF forwarding capability</preamble> - - <artwork><![CDATA[ __ - +--------------------------------------------------+ | - | +------------------------------------------+ | | - | | | | | - | | VNF | | | - | | | | | - | +------------------------------------------+ | | - | | Passthrough/SR-IOV | | Host - | +------------------------------------------+ | | - | | NIC | | | - +---+------------------------------------------+---+ __| - ^ : - | | - : v - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - - <postamble/> - </figure></t> - - <t>Benchmarking with isolated resources alone, with other - resources (both HW&SW) disabled Example, vSw and VM are - SUT</t> - - <t>Benchmarking with isolated resources alone, leaving some - resources unused</t> - - <t>Benchmark with isolated resources and all resources - occupied</t> - </list></t> - - <t>Next Steps<list style="symbols"> - <t>Limited sharing</t> - - <t>Production scenarios</t> - - <t>Stressful scenarios</t> - </list></t> - </list></t> - </section> - </section> - - <section title="VSWITCHPERF Specification Summary"> - <t>The overall specification in preparation is referred to as a Level - Test Design (LTD) document, which will contain a suite of performance - tests. The base performance tests in the LTD are based on the - pre-existing specifications developed by BMWG to test the performance of - physical switches. These specifications include:</t> - - <t><list style="symbols"> - <t><xref target="RFC2544"/> Benchmarking Methodology for Network - Interconnect Devices</t> - - <t><xref target="RFC2889"/> Benchmarking Methodology for LAN - Switching</t> - - <t><xref target="RFC6201"/> Device Reset Characterization</t> - - <t><xref target="RFC5481"/> Packet Delay Variation Applicability - Statement</t> - </list></t> - - <t>Some of the above/newer RFCs are being applied in benchmarking for - the first time, and represent a development challenge for test equipment - developers. Fortunately, many members of the testing system community - have engaged on the VSPERF project, including an open source test - system.</t> - - <t>In addition to this, the LTD also re-uses the terminology defined - by:</t> - - <t><list style="symbols"> - <t><xref target="RFC2285"/> Benchmarking Terminology for LAN - Switching Devices</t> - - <t><xref target="RFC5481"/> Packet Delay Variation Applicability - Statement</t> - </list></t> - - <t/> - - <t>Specifications to be included in future updates of the LTD - include:<list style="symbols"> - <t><xref target="RFC3918"/> Methodology for IP Multicast - Benchmarking</t> - - <t><xref target="RFC4737"/> Packet Reordering Metrics</t> - </list></t> - - <t>As one might expect, the most fundamental internetworking - characteristics of Throughput and Latency remain important when the - switch is virtualized, and these benchmarks figure prominently in the - specification.</t> - - <t>When considering characteristics important to "telco" network - functions, we must begin to consider additional performance metrics. In - this case, the project specifications have referenced metrics from the - IETF IP Performance Metrics (IPPM) literature. This means that the <xref - target="RFC2544"/> test of Latency is replaced by measurement of a - metric derived from IPPM's <xref target="RFC2679"/>, where a set of - statistical summaries will be provided (mean, max, min, etc.). Further - metrics planned to be benchmarked include packet delay variation as - defined by <xref target="RFC5481"/> , reordering, burst behaviour, DUT - availability, DUT capacity and packet loss in long term testing at - Throughput level, where some low-level of background loss may be present - and characterized.</t> - - <t>Tests have been (or will be) designed to collect the metrics - below:</t> - - <t><list style="symbols"> - <t>Throughput Tests to measure the maximum forwarding rate (in - frames per second or fps) and bit rate (in Mbps) for a constant load - (as defined by <xref target="RFC1242"/>) without traffic loss.</t> - - <t>Packet and Frame Delay Distribution Tests to measure average, min - and max packet and frame delay for constant loads.</t> - - <t>Packet Delay Tests to understand latency distribution for - different packet sizes and over an extended test run to uncover - outliers.</t> - - <t>Scalability Tests to understand how the virtual switch performs - as the number of flows, active ports, complexity of the forwarding - logic’s configuration… it has to deal with - increases.</t> - - <t>Stream Performance Tests (TCP, UDP) to measure bulk data transfer - performance, i.e. how fast systems can send and receive data through - the switch.</t> - - <t>Control Path and Datapath Coupling Tests, to understand how - closely coupled the datapath and the control path are as well as the - effect of this coupling on the performance of the DUT (example: - delay of the initial packet of a flow).</t> - - <t>CPU and Memory Consumption Tests to understand the virtual - switch’s footprint on the system, usually conducted as - auxiliary measurements with benchmarks above. They include: CPU - utilization, Cache utilization and Memory footprint.</t> - - <t>The so-called "Soak" tests, where the selected test is conducted - over a long period of time (with an ideal duration of 24 hours, and - at least 6 hours). The purpose of soak tests is to capture transient - changes in performance which may occur due to infrequent processes - or the low probability coincidence of two or more processes. The - performance must be evaluated periodically during continuous - testing, and this results in use of <xref target="RFC2889"/> Frame - Rate metrics instead of <xref target="RFC2544"/> Throughput (which - requires stopping traffic to allow time for all traffic to exit - internal queues).</t> - </list></t> - - <t>Future/planned test specs include:<list style="symbols"> - <t>Request/Response Performance Tests (TCP, UDP) which measure the - transaction rate through the switch.</t> - - <t>Noisy Neighbour Tests, to understand the effects of resource - sharing on the performance of a virtual switch.</t> - - <t>Tests derived from examination of ETSI NFV Draft GS IFA003 - requirements <xref target="IFA003"/> on characterization of - acceleration technologies applied to vswitches.</t> - </list>The flexibility of deployment of a virtual switch within a - network means that the BMWG IETF existing literature needs to be used to - characterize the performance of a switch in various deployment - scenarios. The deployment scenarios under consideration include:</t> - - <t><figure> - <preamble>Physical port to virtual switch to physical - port</preamble> - - <artwork><![CDATA[ __ - +--------------------------------------------------+ | - | +--------------------+ | | - | | | | | - | | v | | Host - | +--------------+ +--------------+ | | - | | phy port | vSwitch | phy port | | | - +---+--------------+------------+--------------+---+ __| - ^ : - | | - : v - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - </figure></t> - - <t><figure> - <preamble>Physical port to virtual switch to VNF to virtual switch - to physical port</preamble> - - <artwork><![CDATA[ __ - +---------------------------------------------------+ | - | | | - | +-------------------------------------------+ | | - | | Application | | | - | +-------------------------------------------+ | | - | ^ : | | - | | | | | Guest - | : v | | - | +---------------+ +---------------+ | | - | | logical port 0| | logical port 1| | | - +---+---------------+-----------+---------------+---+ __| - ^ : - | | - : v __ - +---+---------------+----------+---------------+---+ | - | | logical port 0| | logical port 1| | | - | +---------------+ +---------------+ | | - | ^ : | | - | | | | | Host - | : v | | - | +--------------+ +--------------+ | | - | | phy port | vSwitch | phy port | | | - +---+--------------+------------+--------------+---+ __| - ^ : - | | - : v - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - </figure><figure> - <preamble>Physical port to virtual switch to VNF to virtual switch - to VNF to virtual switch to physical port</preamble> - - <artwork><![CDATA[ __ - +----------------------+ +----------------------+ | - | Guest 1 | | Guest 2 | | - | +---------------+ | | +---------------+ | | - | | Application | | | | Application | | | - | +---------------+ | | +---------------+ | | - | ^ | | | ^ | | | - | | v | | | v | | Guests - | +---------------+ | | +---------------+ | | - | | logical ports | | | | logical ports | | | - | | 0 1 | | | | 0 1 | | | - +---+---------------+--+ +---+---------------+--+__| - ^ : ^ : - | | | | - : v : v _ - +---+---------------+---------+---------------+--+ | - | | 0 1 | | 3 4 | | | - | | logical ports | | logical ports | | | - | +---------------+ +---------------+ | | - | ^ | ^ | | | Host - | | |-----------------| v | | - | +--------------+ +--------------+ | | - | | phy ports | vSwitch | phy ports | | | - +---+--------------+----------+--------------+---+_| - ^ : - | | - : v - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - </figure><figure> - <preamble>Physical port to virtual switch to VNF</preamble> - - <artwork><![CDATA[ __ - +---------------------------------------------------+ | - | | | - | +-------------------------------------------+ | | - | | Application | | | - | +-------------------------------------------+ | | - | ^ | | - | | | | Guest - | : | | - | +---------------+ | | - | | logical port 0| | | - +---+---------------+-------------------------------+ __| - ^ - | - : __ - +---+---------------+------------------------------+ | - | | logical port 0| | | - | +---------------+ | | - | ^ | | - | | | | Host - | : | | - | +--------------+ | | - | | phy port | vSwitch | | - +---+--------------+------------ -------------- ---+ __| - ^ - | - : - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - </figure><figure> - <preamble>VNF to virtual switch to physical port</preamble> - - <artwork><![CDATA[ __ - +---------------------------------------------------+ | - | | | - | +-------------------------------------------+ | | - | | Application | | | - | +-------------------------------------------+ | | - | : | | - | | | | Guest - | v | | - | +---------------+ | | - | | logical port | | | - +-------------------------------+---------------+---+ __| - : - | - v __ - +------------------------------+---------------+---+ | - | | logical port | | | - | +---------------+ | | - | : | | - | | | | Host - | v | | - | +--------------+ | | - | vSwitch | phy port | | | - +-------------------------------+--------------+---+ __| - : - | - v - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - </figure><figure> - <preamble>VNF to virtual switch to VNF</preamble> - - <artwork><![CDATA[ __ - +----------------------+ +----------------------+ | - | Guest 1 | | Guest 2 | | - | +---------------+ | | +---------------+ | | - | | Application | | | | Application | | | - | +---------------+ | | +---------------+ | | - | | | | ^ | | - | v | | | | | Guests - | +---------------+ | | +---------------+ | | - | | logical ports | | | | logical ports | | | - | | 0 | | | | 0 | | | - +---+---------------+--+ +---+---------------+--+__| - : ^ - | | - v : _ - +---+---------------+---------+---------------+--+ | - | | 1 | | 1 | | | - | | logical ports | | logical ports | | | - | +---------------+ +---------------+ | | - | | ^ | | Host - | L-----------------+ | | - | | | - | vSwitch | | - +------------------------------------------------+_|]]></artwork> - </figure></t> - - <t>A set of Deployment Scenario figures is available on the VSPERF Test - Methodology Wiki page <xref target="TestTopo"/>.</t> - </section> - - <section title="3x3 Matrix Coverage"> - <t>This section organizes the many existing test specifications into the - "3x3" matrix (introduced in <xref target="I-D.ietf-bmwg-virtual-net"/>). - Because the LTD specification ID names are quite long, this section is - organized into lists for each occupied cell of the matrix (not all are - occupied, also the matrix has grown to 3x4 to accommodate scale metrics - when displaying the coverage of many metrics/benchmarks). The current - version of the LTD specification is available <xref target="LTD"/>.</t> - - <t>The tests listed below assess the activation of paths in the data - plane, rather than the control plane.</t> - - <t>A complete list of tests with short summaries is available on the - VSPERF "LTD Test Spec Overview" Wiki page <xref target="LTDoverV"/>.</t> - - <section title="Speed of Activation"> - <t><list style="symbols"> - <t>Activation.RFC2889.AddressLearningRate</t> - - <t>PacketLatency.InitialPacketProcessingLatency</t> - </list></t> - </section> - - <section title="Accuracy of Activation section"> - <t><list style="symbols"> - <t>CPDP.Coupling.Flow.Addition</t> - </list></t> - </section> - - <section title="Reliability of Activation"> - <t><list style="symbols"> - <t>Throughput.RFC2544.SystemRecoveryTime</t> - - <t>Throughput.RFC2544.ResetTime</t> - </list></t> - </section> - - <section title="Scale of Activation"> - <t><list style="symbols"> - <t>Activation.RFC2889.AddressCachingCapacity</t> - </list></t> - </section> - - <section title="Speed of Operation"> - <t><list style="symbols"> - <t>Throughput.RFC2544.PacketLossRate</t> - - <t>CPU.RFC2544.0PacketLoss</t> - - <t>Throughput.RFC2544.PacketLossRateFrameModification</t> - - <t>Throughput.RFC2544.BackToBackFrames</t> - - <t>Throughput.RFC2889.MaxForwardingRate</t> - - <t>Throughput.RFC2889.ForwardPressure</t> - - <t>Throughput.RFC2889.BroadcastFrameForwarding</t> - </list></t> - </section> - - <section title="Accuracy of Operation"> - <t><list style="symbols"> - <t>Throughput.RFC2889.ErrorFramesFiltering</t> - - <t>Throughput.RFC2544.Profile</t> - </list></t> - </section> - - <section title="Reliability of Operation"> - <t><list style="symbols"> - <t>Throughput.RFC2889.Soak</t> - - <t>Throughput.RFC2889.SoakFrameModification</t> - - <t>PacketDelayVariation.RFC3393.Soak</t> - </list></t> - </section> - - <section title="Scalability of Operation"> - <t><list style="symbols"> - <t>Scalability.RFC2544.0PacketLoss</t> - - <t>MemoryBandwidth.RFC2544.0PacketLoss.Scalability</t> - </list></t> - </section> - - <section title="Summary"> - <t><figure> - <artwork><![CDATA[|------------------------------------------------------------------------| -| | | | | | -| | SPEED | ACCURACY | RELIABILITY | SCALE | -| | | | | | -|------------------------------------------------------------------------| -| | | | | | -| Activation | X | X | X | X | -| | | | | | -|------------------------------------------------------------------------| -| | | | | | -| Operation | X | X | X | X | -| | | | | | -|------------------------------------------------------------------------| -| | | | | | -| De-activation | | | | | -| | | | | | -|------------------------------------------------------------------------|]]></artwork> - </figure></t> - </section> - </section> - - <section title="Security Considerations"> - <t>Benchmarking activities as described in this memo are limited to - technology characterization of a Device Under Test/System Under Test - (DUT/SUT) using controlled stimuli in a laboratory environment, with - dedicated address space and the constraints specified in the sections - above.</t> - - <t>The benchmarking network topology will be an independent test setup - and MUST NOT be connected to devices that may forward the test traffic - into a production network, or misroute traffic to the test management - network.</t> - - <t>Further, benchmarking is performed on a "black-box" basis, relying - solely on measurements observable external to the DUT/SUT.</t> - - <t>Special capabilities SHOULD NOT exist in the DUT/SUT specifically for - benchmarking purposes. Any implications for network security arising - from the DUT/SUT SHOULD be identical in the lab and in production - networks.</t> - </section> - - <section anchor="IANA" title="IANA Considerations"> - <t>No IANA Action is requested at this time.</t> - </section> - - <section title="Acknowledgements"> - <t>The authors appreciate and acknowledge comments from Scott Bradner, - Marius Georgescu, Ramki Krishnan, Doug Montgomery, Martin Klozik, - Christian Trautman, and others for their reviews.</t> - </section> - </middle> - - <back> - <references title="Normative References"> - <?rfc ?> - - <?rfc include="reference.RFC.2119"?> - - <?rfc ?> - - <?rfc include="reference.RFC.2330"?> - - <?rfc include='reference.RFC.2544'?> - - <?rfc include="reference.RFC.2679"?> - - <?rfc include='reference.RFC.2680'?> - - <?rfc include='reference.RFC.3393'?> - - <?rfc include='reference.RFC.3432'?> - - <?rfc include='reference.RFC.2681'?> - - <?rfc include='reference.RFC.5905'?> - - <?rfc include='reference.RFC.4689'?> - - <?rfc include='reference.RFC.4737'?> - - <?rfc include='reference.RFC.5357'?> - - <?rfc include='reference.RFC.2889'?> - - <?rfc include='reference.RFC.3918'?> - - <?rfc include='reference.RFC.6201'?> - - <?rfc include='reference.RFC.2285'?> - - <reference anchor="NFV.PER001"> - <front> - <title>Network Function Virtualization: Performance and Portability - Best Practices</title> - - <author fullname="ETSI NFV" initials="" surname=""> - <organization/> - </author> - - <date month="June" year="2014"/> - </front> - - <seriesInfo name="Group Specification" - value="ETSI GS NFV-PER 001 V1.1.1 (2014-06)"/> - - <format type="PDF"/> - </reference> - </references> - - <references title="Informative References"> - <?rfc include='reference.RFC.1242'?> - - <?rfc include='reference.RFC.5481'?> - - <?rfc include='reference.RFC.6049'?> - - <?rfc include='reference.RFC.6248'?> - - <?rfc include='reference.RFC.6390'?> - - <?rfc include='reference.I-D.ietf-bmwg-virtual-net'?> - - <?rfc include='reference.I-D.huang-bmwg-virtual-network-performance'?> - - <reference anchor="TestTopo"> - <front> - <title>Test Topologies - https://wiki.opnfv.org/vsperf/test_methodology</title> - - <author> - <organization/> - </author> - - <date/> - </front> - </reference> - - <reference anchor="LTDoverV"> - <front> - <title>LTD Test Spec Overview - https://wiki.opnfv.org/wiki/vswitchperf_test_spec_review</title> - - <author> - <organization/> - </author> - - <date/> - </front> - </reference> - - <reference anchor="LTD"> - <front> - <title>LTD Test Specification - http://artifacts.opnfv.org/vswitchperf/docs/requirements/index.html</title> - - <author> - <organization/> - </author> - - <date/> - </front> - </reference> - - <reference anchor="BrahRel"> - <front> - <title>Brahmaputra, Second OPNFV Release - https://www.opnfv.org/brahmaputra</title> - - <author> - <organization/> - </author> - - <date/> - </front> - </reference> - - <reference anchor="IFA003"> - <front> - <title>https://docbox.etsi.org/ISG/NFV/Open/Drafts/IFA003_Acceleration_-_vSwitch_Spec/</title> - - <author> - <organization/> - </author> - - <date/> - </front> - </reference> - </references> - </back> -</rfc> diff --git a/docs/testing/developer/devguide/requirements/ietf_draft/draft-ietf-bmwg-vswitch-opnfv-01.xml b/docs/testing/developer/devguide/requirements/ietf_draft/draft-ietf-bmwg-vswitch-opnfv-01.xml deleted file mode 100644 index c8a3d99b..00000000 --- a/docs/testing/developer/devguide/requirements/ietf_draft/draft-ietf-bmwg-vswitch-opnfv-01.xml +++ /dev/null @@ -1,1027 +0,0 @@ -<?xml version="1.0" encoding="US-ASCII"?> -<!DOCTYPE rfc SYSTEM "rfc2629.dtd"> -<?rfc toc="yes"?> -<?rfc tocompact="yes"?> -<?rfc tocdepth="3"?> -<?rfc tocindent="yes"?> -<?rfc symrefs="yes"?> -<?rfc sortrefs="yes"?> -<?rfc comments="yes"?> -<?rfc inline="yes"?> -<?rfc compact="yes"?> -<?rfc subcompact="no"?> -<rfc category="info" docName="draft-ietf-bmwg-vswitch-opnfv-01" - ipr="trust200902"> - <front> - <title abbrev="Benchmarking vSwitches">Benchmarking Virtual Switches in - OPNFV</title> - - <author fullname="Maryam Tahhan" initials="M." surname="Tahhan"> - <organization>Intel</organization> - - <address> - <postal> - <street/> - - <city/> - - <region/> - - <code/> - - <country/> - </postal> - - <phone/> - - <facsimile/> - - <email>maryam.tahhan@intel.com</email> - - <uri/> - </address> - </author> - - <author fullname="Billy O'Mahony" initials="B." surname="O'Mahony"> - <organization>Intel</organization> - - <address> - <postal> - <street/> - - <city/> - - <region/> - - <code/> - - <country/> - </postal> - - <phone/> - - <facsimile/> - - <email>billy.o.mahony@intel.com</email> - - <uri/> - </address> - </author> - - <author fullname="Al Morton" initials="A." surname="Morton"> - <organization>AT&T Labs</organization> - - <address> - <postal> - <street>200 Laurel Avenue South</street> - - <city>Middletown,</city> - - <region>NJ</region> - - <code>07748</code> - - <country>USA</country> - </postal> - - <phone>+1 732 420 1571</phone> - - <facsimile>+1 732 368 1192</facsimile> - - <email>acmorton@att.com</email> - - <uri>http://home.comcast.net/~acmacm/</uri> - </address> - </author> - - <date day="10" month="October" year="2016"/> - - <abstract> - <t>This memo describes the progress of the Open Platform for NFV (OPNFV) - project on virtual switch performance "VSWITCHPERF". This project - intends to build on the current and completed work of the Benchmarking - Methodology Working Group in IETF, by referencing existing literature. - The Benchmarking Methodology Working Group has traditionally conducted - laboratory characterization of dedicated physical implementations of - internetworking functions. Therefore, this memo begins to describe the - additional considerations when virtual switches are implemented in - general-purpose hardware. The expanded tests and benchmarks are also - influenced by the OPNFV mission to support virtualization of the "telco" - infrastructure.</t> - </abstract> - - <note title="Requirements Language"> - <t>The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", - "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this - document are to be interpreted as described in <xref - target="RFC2119">RFC 2119</xref>.</t> - - <t/> - </note> - </front> - - <middle> - <section title="Introduction"> - <t>Benchmarking Methodology Working Group (BMWG) has traditionally - conducted laboratory characterization of dedicated physical - implementations of internetworking functions. The Black-box Benchmarks - of Throughput, Latency, Forwarding Rates and others have served our - industry for many years. Now, Network Function Virtualization (NFV) has - the goal to transform how internetwork functions are implemented, and - therefore has garnered much attention.</t> - - <t>This memo summarizes the progress of the Open Platform for NFV - (OPNFV) project on virtual switch performance characterization, - "VSWITCHPERF", through the Brahmaputra (second) release <xref - target="BrahRel"/>. This project intends to build on the current and - completed work of the Benchmarking Methodology Working Group in IETF, by - referencing existing literature. For example, currently the most often - referenced RFC is <xref target="RFC2544"/> (which depends on <xref - target="RFC1242"/>) and foundation of the benchmarking work in OPNFV is - common and strong.</t> - - <t>See - https://wiki.opnfv.org/characterize_vswitch_performance_for_telco_nfv_use_cases - for more background, and the OPNFV website for general information: - https://www.opnfv.org/</t> - - <t>The authors note that OPNFV distinguishes itself from other open - source compute and networking projects through its emphasis on existing - "telco" services as opposed to cloud-computing. There are many ways in - which telco requirements have different emphasis on performance - dimensions when compared to cloud computing: support for and transfer of - isochronous media streams is one example.</t> - - <t>Note also that the move to NFV Infrastructure has resulted in many - new benchmarking initiatives across the industry. The authors are - currently doing their best to maintain alignment with many other - projects, and this Internet Draft is one part of the efforts. We - acknowledge the early work in <xref - target="I-D.huang-bmwg-virtual-network-performance"/>, and useful - discussion with the authors.</t> - </section> - - <section title="Scope"> - <t>The primary purpose and scope of the memo is to inform the industry - of work-in-progress that builds on the body of extensive BMWG literature - and experience, and describe the extensions needed for benchmarking - virtual switches. Inital feedback indicates that many of these - extensions may be applicable beyond the current scope (to hardware - switches in the NFV Infrastructure and to virtual routers, for example). - Additionally, this memo serves as a vehicle to include more detail and - commentary from BMWG and other Open Source communities, under BMWG's - chartered work to characterize the NFV Infrastructure (a virtual switch - is an important aspect of that infrastructure).</t> - - <t>The benchmarking covered in this memo should be applicable to many - types of vswitches, and remain vswitch-agnostic to great degree. There - has been no attempt to track and test all features of any specific - vswitch implementation.</t> - </section> - - <section title="Benchmarking Considerations"> - <t>This section highlights some specific considerations (from <xref - target="I-D.ietf-bmwg-virtual-net"/>)related to Benchmarks for virtual - switches. The OPNFV project is sharing its present view on these areas, - as they develop their specifications in the Level Test Design (LTD) - document.</t> - - <section title="Comparison with Physical Network Functions"> - <t>To compare the performance of virtual designs and implementations - with their physical counterparts, identical benchmarks are needed. - BMWG has developed specifications for many network functions this memo - re-uses existing benchmarks through references, and expands them - during development of new methods. A key configuration aspect is the - number of parallel cores required to achieve comparable performance - with a given physical device, or whether some limit of scale was - reached before the cores could achieve the comparable level.</t> - - <t>It's unlikely that the virtual switch will be the only application - running on the SUT, so CPU utilization, Cache utilization, and Memory - footprint should also be recorded for the virtual implementations of - internetworking functions.</t> - </section> - - <section title="Continued Emphasis on Black-Box Benchmarks"> - <t>External observations remain essential as the basis for Benchmarks. - Internal observations with fixed specification and interpretation will - be provided in parallel to assist the development of operations - procedures when the technology is deployed.</t> - </section> - - <section title="New Configuration Parameters"> - <t>A key consideration when conducting any sort of benchmark is trying - to ensure the consistency and repeatability of test results. When - benchmarking the performance of a vSwitch there are many factors that - can affect the consistency of results, one key factor is matching the - various hardware and software details of the SUT. This section lists - some of the many new parameters which this project believes are - critical to report in order to achieve repeatability.</t> - - <t>Hardware details including:</t> - - <t><list style="symbols"> - <t>Platform details</t> - - <t>Processor details</t> - - <t>Memory information (type and size)</t> - - <t>Number of enabled cores</t> - - <t>Number of cores used for the test</t> - - <t>Number of physical NICs, as well as their details - (manufacturer, versions, type and the PCI slot they are plugged - into)</t> - - <t>NIC interrupt configuration</t> - - <t>BIOS version, release date and any configurations that were - modified</t> - - <t>CPU microcode level</t> - - <t>Memory DIMM configurations (quad rank performance may not be - the same as dual rank) in size, freq and slot locations</t> - - <t>PCI configuration parameters (payload size, early ack - option...)</t> - - <t>Power management at all levels (ACPI sleep states, processor - package, OS...)</t> - </list>Software details including:</t> - - <t><list style="symbols"> - <t>OS parameters and behavior (text vs graphical no one typing at - the console on one system)</t> - - <t>OS version (for host and VNF)</t> - - <t>Kernel version (for host and VNF)</t> - - <t>GRUB boot parameters (for host and VNF)</t> - - <t>Hypervisor details (Type and version)</t> - - <t>Selected vSwitch, version number or commit id used</t> - - <t>vSwitch launch command line if it has been parameterised</t> - - <t>Memory allocation to the vSwitch</t> - - <t>which NUMA node it is using, and how many memory channels</t> - - <t>DPDK or any other SW dependency version number or commit id - used</t> - - <t>Memory allocation to a VM - if it's from Hugpages/elsewhere</t> - - <t>VM storage type: snapshot/independent persistent/independent - non-persistent</t> - - <t>Number of VMs</t> - - <t>Number of Virtual NICs (vNICs), versions, type and driver</t> - - <t>Number of virtual CPUs and their core affinity on the host</t> - - <t>Number vNIC interrupt configuration</t> - - <t>Thread affinitization for the applications (including the - vSwitch itself) on the host</t> - - <t>Details of Resource isolation, such as CPUs designated for - Host/Kernel (isolcpu) and CPUs designated for specific processes - (taskset). - Test duration. - Number of flows.</t> - </list></t> - - <t>Test Traffic Information:<list style="symbols"> - <t>Traffic type - UDP, TCP, IMIX / Other</t> - - <t>Packet Sizes</t> - - <t>Deployment Scenario</t> - </list></t> - - <t/> - </section> - - <section title="Flow classification"> - <t>Virtual switches group packets into flows by processing and - matching particular packet or frame header information, or by matching - packets based on the input ports. Thus a flow can be thought of a - sequence of packets that have the same set of header field values - (5-tuple) or have arrived on the same port. Performance results can - vary based on the parameters the vSwitch uses to match for a flow. The - recommended flow classification parameters for any vSwitch performance - tests are: the input port, the source IP address, the destination IP - address and the Ethernet protocol type field. It is essential to - increase the flow timeout time on a vSwitch before conducting any - performance tests that do not measure the flow setup time. Normally - the first packet of a particular stream will install the flow in the - virtual switch which adds an additional latency, subsequent packets of - the same flow are not subject to this latency if the flow is already - installed on the vSwitch.</t> - </section> - - <section title="Benchmarks using Baselines with Resource Isolation"> - <t>This outline describes measurement of baseline with isolated - resources at a high level, which is the intended approach at this - time.</t> - - <t><list style="numbers"> - <t>Baselines: <list style="symbols"> - <t>Optional: Benchmark platform forwarding capability without - a vswitch or VNF for at least 72 hours (serves as a means of - platform validation and a means to obtain the base performance - for the platform in terms of its maximum forwarding rate and - latency). <figure> - <preamble>Benchmark platform forwarding - capability</preamble> - - <artwork align="right"><![CDATA[ __ - +--------------------------------------------------+ | - | +------------------------------------------+ | | - | | | | | - | | Simple Forwarding App | | Host - | | | | | - | +------------------------------------------+ | | - | | NIC | | | - +---+------------------------------------------+---+ __| - ^ : - | | - : v - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - - <postamble/> - </figure></t> - - <t>Benchmark VNF forwarding capability with direct - connectivity (vSwitch bypass, e.g., SR/IOV) for at least 72 - hours (serves as a means of VNF validation and a means to - obtain the base performance for the VNF in terms of its - maximum forwarding rate and latency). The metrics gathered - from this test will serve as a key comparison point for - vSwitch bypass technologies performance and vSwitch - performance. <figure align="right"> - <preamble>Benchmark VNF forwarding capability</preamble> - - <artwork><![CDATA[ __ - +--------------------------------------------------+ | - | +------------------------------------------+ | | - | | | | | - | | VNF | | | - | | | | | - | +------------------------------------------+ | | - | | Passthrough/SR-IOV | | Host - | +------------------------------------------+ | | - | | NIC | | | - +---+------------------------------------------+---+ __| - ^ : - | | - : v - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - - <postamble/> - </figure></t> - - <t>Benchmarking with isolated resources alone, with other - resources (both HW&SW) disabled Example, vSw and VM are - SUT</t> - - <t>Benchmarking with isolated resources alone, leaving some - resources unused</t> - - <t>Benchmark with isolated resources and all resources - occupied</t> - </list></t> - - <t>Next Steps<list style="symbols"> - <t>Limited sharing</t> - - <t>Production scenarios</t> - - <t>Stressful scenarios</t> - </list></t> - </list></t> - </section> - </section> - - <section title="VSWITCHPERF Specification Summary"> - <t>The overall specification in preparation is referred to as a Level - Test Design (LTD) document, which will contain a suite of performance - tests. The base performance tests in the LTD are based on the - pre-existing specifications developed by BMWG to test the performance of - physical switches. These specifications include:</t> - - <t><list style="symbols"> - <t><xref target="RFC2544"/> Benchmarking Methodology for Network - Interconnect Devices</t> - - <t><xref target="RFC2889"/> Benchmarking Methodology for LAN - Switching</t> - - <t><xref target="RFC6201"/> Device Reset Characterization</t> - - <t><xref target="RFC5481"/> Packet Delay Variation Applicability - Statement</t> - </list></t> - - <t>Some of the above/newer RFCs are being applied in benchmarking for - the first time, and represent a development challenge for test equipment - developers. Fortunately, many members of the testing system community - have engaged on the VSPERF project, including an open source test - system.</t> - - <t>In addition to this, the LTD also re-uses the terminology defined - by:</t> - - <t><list style="symbols"> - <t><xref target="RFC2285"/> Benchmarking Terminology for LAN - Switching Devices</t> - - <t><xref target="RFC5481"/> Packet Delay Variation Applicability - Statement</t> - </list></t> - - <t/> - - <t>Specifications to be included in future updates of the LTD - include:<list style="symbols"> - <t><xref target="RFC3918"/> Methodology for IP Multicast - Benchmarking</t> - - <t><xref target="RFC4737"/> Packet Reordering Metrics</t> - </list></t> - - <t>As one might expect, the most fundamental internetworking - characteristics of Throughput and Latency remain important when the - switch is virtualized, and these benchmarks figure prominently in the - specification.</t> - - <t>When considering characteristics important to "telco" network - functions, we must begin to consider additional performance metrics. In - this case, the project specifications have referenced metrics from the - IETF IP Performance Metrics (IPPM) literature. This means that the <xref - target="RFC2544"/> test of Latency is replaced by measurement of a - metric derived from IPPM's <xref target="RFC2679"/>, where a set of - statistical summaries will be provided (mean, max, min, etc.). Further - metrics planned to be benchmarked include packet delay variation as - defined by <xref target="RFC5481"/> , reordering, burst behaviour, DUT - availability, DUT capacity and packet loss in long term testing at - Throughput level, where some low-level of background loss may be present - and characterized.</t> - - <t>Tests have been (or will be) designed to collect the metrics - below:</t> - - <t><list style="symbols"> - <t>Throughput Tests to measure the maximum forwarding rate (in - frames per second or fps) and bit rate (in Mbps) for a constant load - (as defined by <xref target="RFC1242"/>) without traffic loss.</t> - - <t>Packet and Frame Delay Distribution Tests to measure average, min - and max packet and frame delay for constant loads.</t> - - <t>Packet Delay Tests to understand latency distribution for - different packet sizes and over an extended test run to uncover - outliers.</t> - - <t>Scalability Tests to understand how the virtual switch performs - as the number of flows, active ports, complexity of the forwarding - logic’s configuration… it has to deal with - increases.</t> - - <t>Stream Performance Tests (TCP, UDP) to measure bulk data transfer - performance, i.e. how fast systems can send and receive data through - the switch.</t> - - <t>Control Path and Datapath Coupling Tests, to understand how - closely coupled the datapath and the control path are as well as the - effect of this coupling on the performance of the DUT (example: - delay of the initial packet of a flow).</t> - - <t>CPU and Memory Consumption Tests to understand the virtual - switch’s footprint on the system, usually conducted as - auxiliary measurements with benchmarks above. They include: CPU - utilization, Cache utilization and Memory footprint.</t> - - <t>The so-called "Soak" tests, where the selected test is conducted - over a long period of time (with an ideal duration of 24 hours, but - only long enough to determine that stability issues exist when - found; there is no requirement to continue a test when a DUT - exhibits instability over time). The key performance characteristics - and benchmarks for a DUT are determined (using short duration tests) - prior to conducting soak tests. The purpose of soak tests is to - capture transient changes in performance which may occur due to - infrequent processes, memory leaks, or the low probability - coincidence of two or more processes. The stability of the DUT is - the paramount consideration, so performance must be evaluated - periodically during continuous testing, and this results in use of - <xref target="RFC2889"/> Frame Rate metrics instead of <xref - target="RFC2544"/> Throughput (which requires stopping traffic to - allow time for all traffic to exit internal queues), for - example.</t> - </list></t> - - <t>Future/planned test specs include:<list style="symbols"> - <t>Request/Response Performance Tests (TCP, UDP) which measure the - transaction rate through the switch.</t> - - <t>Noisy Neighbour Tests, to understand the effects of resource - sharing on the performance of a virtual switch.</t> - - <t>Tests derived from examination of ETSI NFV Draft GS IFA003 - requirements <xref target="IFA003"/> on characterization of - acceleration technologies applied to vswitches.</t> - </list>The flexibility of deployment of a virtual switch within a - network means that the BMWG IETF existing literature needs to be used to - characterize the performance of a switch in various deployment - scenarios. The deployment scenarios under consideration include:</t> - - <t><figure> - <preamble>Physical port to virtual switch to physical - port</preamble> - - <artwork><![CDATA[ __ - +--------------------------------------------------+ | - | +--------------------+ | | - | | | | | - | | v | | Host - | +--------------+ +--------------+ | | - | | phy port | vSwitch | phy port | | | - +---+--------------+------------+--------------+---+ __| - ^ : - | | - : v - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - </figure></t> - - <t><figure> - <preamble>Physical port to virtual switch to VNF to virtual switch - to physical port</preamble> - - <artwork><![CDATA[ __ - +---------------------------------------------------+ | - | | | - | +-------------------------------------------+ | | - | | Application | | | - | +-------------------------------------------+ | | - | ^ : | | - | | | | | Guest - | : v | | - | +---------------+ +---------------+ | | - | | logical port 0| | logical port 1| | | - +---+---------------+-----------+---------------+---+ __| - ^ : - | | - : v __ - +---+---------------+----------+---------------+---+ | - | | logical port 0| | logical port 1| | | - | +---------------+ +---------------+ | | - | ^ : | | - | | | | | Host - | : v | | - | +--------------+ +--------------+ | | - | | phy port | vSwitch | phy port | | | - +---+--------------+------------+--------------+---+ __| - ^ : - | | - : v - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - </figure><figure> - <preamble>Physical port to virtual switch to VNF to virtual switch - to VNF to virtual switch to physical port</preamble> - - <artwork><![CDATA[ __ - +----------------------+ +----------------------+ | - | Guest 1 | | Guest 2 | | - | +---------------+ | | +---------------+ | | - | | Application | | | | Application | | | - | +---------------+ | | +---------------+ | | - | ^ | | | ^ | | | - | | v | | | v | | Guests - | +---------------+ | | +---------------+ | | - | | logical ports | | | | logical ports | | | - | | 0 1 | | | | 0 1 | | | - +---+---------------+--+ +---+---------------+--+__| - ^ : ^ : - | | | | - : v : v _ - +---+---------------+---------+---------------+--+ | - | | 0 1 | | 3 4 | | | - | | logical ports | | logical ports | | | - | +---------------+ +---------------+ | | - | ^ | ^ | | | Host - | | |-----------------| v | | - | +--------------+ +--------------+ | | - | | phy ports | vSwitch | phy ports | | | - +---+--------------+----------+--------------+---+_| - ^ : - | | - : v - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - </figure><figure> - <preamble>Physical port to virtual switch to VNF</preamble> - - <artwork><![CDATA[ __ - +---------------------------------------------------+ | - | | | - | +-------------------------------------------+ | | - | | Application | | | - | +-------------------------------------------+ | | - | ^ | | - | | | | Guest - | : | | - | +---------------+ | | - | | logical port 0| | | - +---+---------------+-------------------------------+ __| - ^ - | - : __ - +---+---------------+------------------------------+ | - | | logical port 0| | | - | +---------------+ | | - | ^ | | - | | | | Host - | : | | - | +--------------+ | | - | | phy port | vSwitch | | - +---+--------------+------------ -------------- ---+ __| - ^ - | - : - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - </figure><figure> - <preamble>VNF to virtual switch to physical port</preamble> - - <artwork><![CDATA[ __ - +---------------------------------------------------+ | - | | | - | +-------------------------------------------+ | | - | | Application | | | - | +-------------------------------------------+ | | - | : | | - | | | | Guest - | v | | - | +---------------+ | | - | | logical port | | | - +-------------------------------+---------------+---+ __| - : - | - v __ - +------------------------------+---------------+---+ | - | | logical port | | | - | +---------------+ | | - | : | | - | | | | Host - | v | | - | +--------------+ | | - | vSwitch | phy port | | | - +-------------------------------+--------------+---+ __| - : - | - v - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - </figure><figure> - <preamble>VNF to virtual switch to VNF</preamble> - - <artwork><![CDATA[ __ - +----------------------+ +----------------------+ | - | Guest 1 | | Guest 2 | | - | +---------------+ | | +---------------+ | | - | | Application | | | | Application | | | - | +---------------+ | | +---------------+ | | - | | | | ^ | | - | v | | | | | Guests - | +---------------+ | | +---------------+ | | - | | logical ports | | | | logical ports | | | - | | 0 | | | | 0 | | | - +---+---------------+--+ +---+---------------+--+__| - : ^ - | | - v : _ - +---+---------------+---------+---------------+--+ | - | | 1 | | 1 | | | - | | logical ports | | logical ports | | | - | +---------------+ +---------------+ | | - | | ^ | | Host - | L-----------------+ | | - | | | - | vSwitch | | - +------------------------------------------------+_|]]></artwork> - </figure></t> - - <t>A set of Deployment Scenario figures is available on the VSPERF Test - Methodology Wiki page <xref target="TestTopo"/>.</t> - </section> - - <section title="3x3 Matrix Coverage"> - <t>This section organizes the many existing test specifications into the - "3x3" matrix (introduced in <xref target="I-D.ietf-bmwg-virtual-net"/>). - Because the LTD specification ID names are quite long, this section is - organized into lists for each occupied cell of the matrix (not all are - occupied, also the matrix has grown to 3x4 to accommodate scale metrics - when displaying the coverage of many metrics/benchmarks). The current - version of the LTD specification is available <xref target="LTD"/>.</t> - - <t>The tests listed below assess the activation of paths in the data - plane, rather than the control plane.</t> - - <t>A complete list of tests with short summaries is available on the - VSPERF "LTD Test Spec Overview" Wiki page <xref target="LTDoverV"/>.</t> - - <section title="Speed of Activation"> - <t><list style="symbols"> - <t>Activation.RFC2889.AddressLearningRate</t> - - <t>PacketLatency.InitialPacketProcessingLatency</t> - </list></t> - </section> - - <section title="Accuracy of Activation section"> - <t><list style="symbols"> - <t>CPDP.Coupling.Flow.Addition</t> - </list></t> - </section> - - <section title="Reliability of Activation"> - <t><list style="symbols"> - <t>Throughput.RFC2544.SystemRecoveryTime</t> - - <t>Throughput.RFC2544.ResetTime</t> - </list></t> - </section> - - <section title="Scale of Activation"> - <t><list style="symbols"> - <t>Activation.RFC2889.AddressCachingCapacity</t> - </list></t> - </section> - - <section title="Speed of Operation"> - <t><list style="symbols"> - <t>Throughput.RFC2544.PacketLossRate</t> - - <t>CPU.RFC2544.0PacketLoss</t> - - <t>Throughput.RFC2544.PacketLossRateFrameModification</t> - - <t>Throughput.RFC2544.BackToBackFrames</t> - - <t>Throughput.RFC2889.MaxForwardingRate</t> - - <t>Throughput.RFC2889.ForwardPressure</t> - - <t>Throughput.RFC2889.BroadcastFrameForwarding</t> - </list></t> - </section> - - <section title="Accuracy of Operation"> - <t><list style="symbols"> - <t>Throughput.RFC2889.ErrorFramesFiltering</t> - - <t>Throughput.RFC2544.Profile</t> - </list></t> - </section> - - <section title="Reliability of Operation"> - <t><list style="symbols"> - <t>Throughput.RFC2889.Soak</t> - - <t>Throughput.RFC2889.SoakFrameModification</t> - - <t>PacketDelayVariation.RFC3393.Soak</t> - </list></t> - </section> - - <section title="Scalability of Operation"> - <t><list style="symbols"> - <t>Scalability.RFC2544.0PacketLoss</t> - - <t>MemoryBandwidth.RFC2544.0PacketLoss.Scalability</t> - </list></t> - </section> - - <section title="Summary"> - <t><figure> - <artwork><![CDATA[|------------------------------------------------------------------------| -| | | | | | -| | SPEED | ACCURACY | RELIABILITY | SCALE | -| | | | | | -|------------------------------------------------------------------------| -| | | | | | -| Activation | X | X | X | X | -| | | | | | -|------------------------------------------------------------------------| -| | | | | | -| Operation | X | X | X | X | -| | | | | | -|------------------------------------------------------------------------| -| | | | | | -| De-activation | | | | | -| | | | | | -|------------------------------------------------------------------------|]]></artwork> - </figure></t> - </section> - </section> - - <section title="Security Considerations"> - <t>Benchmarking activities as described in this memo are limited to - technology characterization of a Device Under Test/System Under Test - (DUT/SUT) using controlled stimuli in a laboratory environment, with - dedicated address space and the constraints specified in the sections - above.</t> - - <t>The benchmarking network topology will be an independent test setup - and MUST NOT be connected to devices that may forward the test traffic - into a production network, or misroute traffic to the test management - network.</t> - - <t>Further, benchmarking is performed on a "black-box" basis, relying - solely on measurements observable external to the DUT/SUT.</t> - - <t>Special capabilities SHOULD NOT exist in the DUT/SUT specifically for - benchmarking purposes. Any implications for network security arising - from the DUT/SUT SHOULD be identical in the lab and in production - networks.</t> - </section> - - <section anchor="IANA" title="IANA Considerations"> - <t>No IANA Action is requested at this time.</t> - </section> - - <section title="Acknowledgements"> - <t>The authors appreciate and acknowledge comments from Scott Bradner, - Marius Georgescu, Ramki Krishnan, Doug Montgomery, Martin Klozik, - Christian Trautman, and others for their reviews.</t> - </section> - </middle> - - <back> - <references title="Normative References"> - <?rfc ?> - - <?rfc include="reference.RFC.2119"?> - - <?rfc ?> - - <?rfc include="reference.RFC.2330"?> - - <?rfc include='reference.RFC.2544'?> - - <?rfc include="reference.RFC.2679"?> - - <?rfc include='reference.RFC.2680'?> - - <?rfc include='reference.RFC.3393'?> - - <?rfc include='reference.RFC.3432'?> - - <?rfc include='reference.RFC.2681'?> - - <?rfc include='reference.RFC.5905'?> - - <?rfc include='reference.RFC.4689'?> - - <?rfc include='reference.RFC.4737'?> - - <?rfc include='reference.RFC.5357'?> - - <?rfc include='reference.RFC.2889'?> - - <?rfc include='reference.RFC.3918'?> - - <?rfc include='reference.RFC.6201'?> - - <?rfc include='reference.RFC.2285'?> - - <reference anchor="NFV.PER001"> - <front> - <title>Network Function Virtualization: Performance and Portability - Best Practices</title> - - <author fullname="ETSI NFV" initials="" surname=""> - <organization/> - </author> - - <date month="June" year="2014"/> - </front> - - <seriesInfo name="Group Specification" - value="ETSI GS NFV-PER 001 V1.1.1 (2014-06)"/> - - <format type="PDF"/> - </reference> - </references> - - <references title="Informative References"> - <?rfc include='reference.RFC.1242'?> - - <?rfc include='reference.RFC.5481'?> - - <?rfc include='reference.RFC.6049'?> - - <?rfc include='reference.RFC.6248'?> - - <?rfc include='reference.RFC.6390'?> - - <?rfc include='reference.I-D.ietf-bmwg-virtual-net'?> - - <?rfc include='reference.I-D.huang-bmwg-virtual-network-performance'?> - - <reference anchor="TestTopo"> - <front> - <title>Test Topologies - https://wiki.opnfv.org/vsperf/test_methodology</title> - - <author> - <organization/> - </author> - - <date/> - </front> - </reference> - - <reference anchor="LTDoverV"> - <front> - <title>LTD Test Spec Overview - https://wiki.opnfv.org/wiki/vswitchperf_test_spec_review</title> - - <author> - <organization/> - </author> - - <date/> - </front> - </reference> - - <reference anchor="LTD"> - <front> - <title>LTD Test Specification - http://artifacts.opnfv.org/vswitchperf/brahmaputra/docs/requirements/index.html</title> - - <author> - <organization/> - </author> - - <date/> - </front> - </reference> - - <reference anchor="BrahRel"> - <front> - <title>Brahmaputra, Second OPNFV Release - https://www.opnfv.org/brahmaputra</title> - - <author> - <organization/> - </author> - - <date/> - </front> - </reference> - - <reference anchor="IFA003"> - <front> - <title>https://docbox.etsi.org/ISG/NFV/Open/Drafts/IFA003_Acceleration_-_vSwitch_Spec/</title> - - <author> - <organization/> - </author> - - <date/> - </front> - </reference> - </references> - </back> -</rfc> diff --git a/docs/testing/developer/devguide/requirements/ietf_draft/draft-vsperf-bmwg-vswitch-opnfv-00.xml b/docs/testing/developer/devguide/requirements/ietf_draft/draft-vsperf-bmwg-vswitch-opnfv-00.xml deleted file mode 100644 index b5f7f833..00000000 --- a/docs/testing/developer/devguide/requirements/ietf_draft/draft-vsperf-bmwg-vswitch-opnfv-00.xml +++ /dev/null @@ -1,964 +0,0 @@ -<?xml version="1.0" encoding="US-ASCII"?> -<!DOCTYPE rfc SYSTEM "rfc2629.dtd"> -<?rfc toc="yes"?> -<?rfc tocompact="yes"?> -<?rfc tocdepth="3"?> -<?rfc tocindent="yes"?> -<?rfc symrefs="yes"?> -<?rfc sortrefs="yes"?> -<?rfc comments="yes"?> -<?rfc inline="yes"?> -<?rfc compact="yes"?> -<?rfc subcompact="no"?> -<rfc category="info" docName="draft-vsperf-bmwg-vswitch-opnfv-01" - ipr="trust200902"> - <front> - <title abbrev="Benchmarking vSwitches">Benchmarking Virtual Switches in - OPNFV</title> - - <author fullname="Maryam Tahhan" initials="M." surname="Tahhan"> - <organization>Intel</organization> - - <address> - <postal> - <street/> - - <city/> - - <region/> - - <code/> - - <country/> - </postal> - - <phone/> - - <facsimile/> - - <email>maryam.tahhan@intel.com</email> - - <uri/> - </address> - </author> - - <author fullname="Billy O'Mahony" initials="B." surname="O'Mahony"> - <organization>Intel</organization> - - <address> - <postal> - <street/> - - <city/> - - <region/> - - <code/> - - <country/> - </postal> - - <phone/> - - <facsimile/> - - <email>billy.o.mahony@intel.com</email> - - <uri/> - </address> - </author> - - <author fullname="Al Morton" initials="A." surname="Morton"> - <organization>AT&T Labs</organization> - - <address> - <postal> - <street>200 Laurel Avenue South</street> - - <city>Middletown,</city> - - <region>NJ</region> - - <code>07748</code> - - <country>USA</country> - </postal> - - <phone>+1 732 420 1571</phone> - - <facsimile>+1 732 368 1192</facsimile> - - <email>acmorton@att.com</email> - - <uri>http://home.comcast.net/~acmacm/</uri> - </address> - </author> - - <date day="14" month="October" year="2015"/> - - <abstract> - <t>This memo describes the progress of the Open Platform for NFV (OPNFV) - project on virtual switch performance "VSWITCHPERF". This project - intends to build on the current and completed work of the Benchmarking - Methodology Working Group in IETF, by referencing existing literature. - The Benchmarking Methodology Working Group has traditionally conducted - laboratory characterization of dedicated physical implementations of - internetworking functions. Therefore, this memo begins to describe the - additional considerations when virtual switches are implemented in - general-purpose hardware. The expanded tests and benchmarks are also - influenced by the OPNFV mission to support virtualization of the "telco" - infrastructure.</t> - </abstract> - - <note title="Requirements Language"> - <t>The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", - "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this - document are to be interpreted as described in <xref - target="RFC2119">RFC 2119</xref>.</t> - - <t/> - </note> - </front> - - <middle> - <section title="Introduction"> - <t>Benchmarking Methodology Working Group (BMWG) has traditionally - conducted laboratory characterization of dedicated physical - implementations of internetworking functions. The Black-box Benchmarks - of Throughput, Latency, Forwarding Rates and others have served our - industry for many years. Now, Network Function Virtualization (NFV) has - the goal to transform how internetwork functions are implemented, and - therefore has garnered much attention.</t> - - <t>This memo describes the progress of the Open Platform for NFV (OPNFV) - project on virtual switch performance characterization, "VSWITCHPERF". - This project intends to build on the current and completed work of the - Benchmarking Methodology Working Group in IETF, by referencing existing - literature. For example, currently the most often referenced RFC is - <xref target="RFC2544"/> (which depends on <xref target="RFC1242"/>) and - foundation of the benchmarking work in OPNFV is common and strong.</t> - - <t>See - https://wiki.opnfv.org/characterize_vswitch_performance_for_telco_nfv_use_cases - for more background, and the OPNFV website for general information: - https://www.opnfv.org/</t> - - <t>The authors note that OPNFV distinguishes itself from other open - source compute and networking projects through its emphasis on existing - "telco" services as opposed to cloud-computing. There are many ways in - which telco requirements have different emphasis on performance - dimensions when compared to cloud computing: support for and transfer of - isochronous media streams is one example.</t> - - <t>Note also that the move to NFV Infrastructure has resulted in many - new benchmarking initiatives across the industry, and the authors are - currently doing their best to maintain alignment with many other - projects, and this Internet Draft is evidence of the efforts.</t> - </section> - - <section title="Scope"> - <t>The primary purpose and scope of the memo is to inform BMWG of - work-in-progress that builds on the body of extensive literature and - experience. Additionally, once the initial information conveyed here is - received, this memo may be expanded to include more detail and - commentary from both BMWG and OPNFV communities, under BMWG's chartered - work to characterize the NFV Infrastructure (a virtual switch is an - important aspect of that infrastructure).</t> - </section> - - <section title="Benchmarking Considerations"> - <t>This section highlights some specific considerations (from <xref - target="I-D.ietf-bmwg-virtual-net"/>)related to Benchmarks for virtual - switches. The OPNFV project is sharing its present view on these areas, - as they develop their specifications in the Level Test Design (LTD) - document.</t> - - <section title="Comparison with Physical Network Functions"> - <t>To compare the performance of virtual designs and implementations - with their physical counterparts, identical benchmarks are needed. - BMWG has developed specifications for many network functions this memo - re-uses existing benchmarks through references, and expands them - during development of new methods. A key configuration aspect is the - number of parallel cores required to achieve comparable performance - with a given physical device, or whether some limit of scale was - reached before the cores could achieve the comparable level.</t> - - <t>It's unlikely that the virtual switch will be the only application - running on the SUT, so CPU utilization, Cache utilization, and Memory - footprint should also be recorded for the virtual implementations of - internetworking functions.</t> - </section> - - <section title="Continued Emphasis on Black-Box Benchmarks"> - <t>External observations remain essential as the basis for Benchmarks. - Internal observations with fixed specification and interpretation will - be provided in parallel to assist the development of operations - procedures when the technology is deployed.</t> - </section> - - <section title="New Configuration Parameters"> - <t>A key consideration when conducting any sort of benchmark is trying - to ensure the consistency and repeatability of test results. When - benchmarking the performance of a vSwitch there are many factors that - can affect the consistency of results, one key factor is matching the - various hardware and software details of the SUT. This section lists - some of the many new parameters which this project believes are - critical to report in order to achieve repeatability.</t> - - <t>Hardware details including:</t> - - <t><list style="symbols"> - <t>Platform details</t> - - <t>Processor details</t> - - <t>Memory information (type and size)</t> - - <t>Number of enabled cores</t> - - <t>Number of cores used for the test</t> - - <t>Number of physical NICs, as well as their details - (manufacturer, versions, type and the PCI slot they are plugged - into)</t> - - <t>NIC interrupt configuration</t> - - <t>BIOS version, release date and any configurations that were - modified</t> - - <t>CPU microcode level</t> - - <t>Memory DIMM configurations (quad rank performance may not be - the same as dual rank) in size, freq and slot locations</t> - - <t>PCI configuration parameters (payload size, early ack - option...)</t> - - <t>Power management at all levels (ACPI sleep states, processor - package, OS...)</t> - </list>Software details including:</t> - - <t><list style="symbols"> - <t>OS parameters and behavior (text vs graphical no one typing at - the console on one system)</t> - - <t>OS version (for host and VNF)</t> - - <t>Kernel version (for host and VNF)</t> - - <t>GRUB boot parameters (for host and VNF)</t> - - <t>Hypervisor details (Type and version)</t> - - <t>Selected vSwitch, version number or commit id used</t> - - <t>vSwitch launch command line if it has been parameterised</t> - - <t>Memory allocation to the vSwitch</t> - - <t>which NUMA node it is using, and how many memory channels</t> - - <t>DPDK or any other SW dependency version number or commit id - used</t> - - <t>Memory allocation to a VM - if it's from Hugpages/elsewhere</t> - - <t>VM storage type: snapshot/independent persistent/independent - non-persistent</t> - - <t>Number of VMs</t> - - <t>Number of Virtual NICs (vNICs), versions, type and driver</t> - - <t>Number of virtual CPUs and their core affinity on the host</t> - - <t>Number vNIC interrupt configuration</t> - - <t>Thread affinitization for the applications (including the - vSwitch itself) on the host</t> - - <t>Details of Resource isolation, such as CPUs designated for - Host/Kernel (isolcpu) and CPUs designated for specific processes - (taskset). - Test duration. - Number of flows.</t> - </list></t> - - <t>Test Traffic Information:<list style="symbols"> - <t>Traffic type - UDP, TCP, IMIX / Other</t> - - <t>Packet Sizes</t> - - <t>Deployment Scenario</t> - </list></t> - - <t/> - </section> - - <section title="Flow classification"> - <t>Virtual switches group packets into flows by processing and - matching particular packet or frame header information, or by matching - packets based on the input ports. Thus a flow can be thought of a - sequence of packets that have the same set of header field values or - have arrived on the same port. Performance results can vary based on - the parameters the vSwitch uses to match for a flow. The recommended - flow classification parameters for any vSwitch performance tests are: - the input port, the source IP address, the destination IP address and - the Ethernet protocol type field. It is essential to increase the flow - timeout time on a vSwitch before conducting any performance tests that - do not measure the flow setup time. Normally the first packet of a - particular stream will install the flow in the virtual switch which - adds an additional latency, subsequent packets of the same flow are - not subject to this latency if the flow is already installed on the - vSwitch.</t> - </section> - - <section title="Benchmarks using Baselines with Resource Isolation"> - <t>This outline describes measurement of baseline with isolated - resources at a high level, which is the intended approach at this - time.</t> - - <t><list style="numbers"> - <t>Baselines: <list style="symbols"> - <t>Optional: Benchmark platform forwarding capability without - a vswitch or VNF for at least 72 hours (serves as a means of - platform validation and a means to obtain the base performance - for the platform in terms of its maximum forwarding rate and - latency). <figure> - <preamble>Benchmark platform forwarding - capability</preamble> - - <artwork align="right"><![CDATA[ __ - +--------------------------------------------------+ | - | +------------------------------------------+ | | - | | | | | - | | Simple Forwarding App | | Host - | | | | | - | +------------------------------------------+ | | - | | NIC | | | - +---+------------------------------------------+---+ __| - ^ : - | | - : v - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - - <postamble/> - </figure></t> - - <t>Benchmark VNF forwarding capability with direct - connectivity (vSwitch bypass, e.g., SR/IOV) for at least 72 - hours (serves as a means of VNF validation and a means to - obtain the base performance for the VNF in terms of its - maximum forwarding rate and latency). The metrics gathered - from this test will serve as a key comparison point for - vSwitch bypass technologies performance and vSwitch - performance. <figure align="right"> - <preamble>Benchmark VNF forwarding capability</preamble> - - <artwork><![CDATA[ __ - +--------------------------------------------------+ | - | +------------------------------------------+ | | - | | | | | - | | VNF | | | - | | | | | - | +------------------------------------------+ | | - | | Passthrough/SR-IOV | | Host - | +------------------------------------------+ | | - | | NIC | | | - +---+------------------------------------------+---+ __| - ^ : - | | - : v - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - - <postamble/> - </figure></t> - - <t>Benchmarking with isolated resources alone, with other - resources (both HW&SW) disabled Example, vSw and VM are - SUT</t> - - <t>Benchmarking with isolated resources alone, leaving some - resources unused</t> - - <t>Benchmark with isolated resources and all resources - occupied</t> - </list></t> - - <t>Next Steps<list style="symbols"> - <t>Limited sharing</t> - - <t>Production scenarios</t> - - <t>Stressful scenarios</t> - </list></t> - </list></t> - </section> - </section> - - <section title="VSWITCHPERF Specification Summary"> - <t>The overall specification in preparation is referred to as a Level - Test Design (LTD) document, which will contain a suite of performance - tests. The base performance tests in the LTD are based on the - pre-existing specifications developed by BMWG to test the performance of - physical switches. These specifications include:</t> - - <t><list style="symbols"> - <t><xref target="RFC2544"/> Benchmarking Methodology for Network - Interconnect Devices</t> - - <t><xref target="RFC2889"/> Benchmarking Methodology for LAN - Switching</t> - - <t><xref target="RFC6201"/> Device Reset Characterization</t> - - <t><xref target="RFC5481"/> Packet Delay Variation Applicability - Statement</t> - </list></t> - - <t>Some of the above/newer RFCs are being applied in benchmarking for - the first time, and represent a development challenge for test equipment - developers. Fortunately, many members of the testing system community - have engaged on the VSPERF project, including an open source test - system.</t> - - <t>In addition to this, the LTD also re-uses the terminology defined - by:</t> - - <t><list style="symbols"> - <t><xref target="RFC2285"/> Benchmarking Terminology for LAN - Switching Devices</t> - - <t><xref target="RFC5481"/> Packet Delay Variation Applicability - Statement</t> - </list></t> - - <t/> - - <t>Specifications to be included in future updates of the LTD - include:<list style="symbols"> - <t><xref target="RFC3918"/> Methodology for IP Multicast - Benchmarking</t> - - <t><xref target="RFC4737"/> Packet Reordering Metrics</t> - </list></t> - - <t>As one might expect, the most fundamental internetworking - characteristics of Throughput and Latency remain important when the - switch is virtualized, and these benchmarks figure prominently in the - specification.</t> - - <t>When considering characteristics important to "telco" network - functions, we must begin to consider additional performance metrics. In - this case, the project specifications have referenced metrics from the - IETF IP Performance Metrics (IPPM) literature. This means that the <xref - target="RFC2544"/> test of Latency is replaced by measurement of a - metric derived from IPPM's <xref target="RFC2679"/>, where a set of - statistical summaries will be provided (mean, max, min, etc.). Further - metrics planned to be benchmarked include packet delay variation as - defined by <xref target="RFC5481"/> , reordering, burst behaviour, DUT - availability, DUT capacity and packet loss in long term testing at - Throughput level, where some low-level of background loss may be present - and characterized.</t> - - <t>Tests have been (or will be) designed to collect the metrics - below:</t> - - <t><list style="symbols"> - <t>Throughput Tests to measure the maximum forwarding rate (in - frames per second or fps) and bit rate (in Mbps) for a constant load - (as defined by RFC1242) without traffic loss.</t> - - <t>Packet and Frame Delay Distribution Tests to measure average, min - and max packet and frame delay for constant loads.</t> - - <t>Packet Delay Tests to understand latency distribution for - different packet sizes and over an extended test run to uncover - outliers.</t> - - <t>Scalability Tests to understand how the virtual switch performs - as the number of flows, active ports, complexity of the forwarding - logic’s configuration… it has to deal with - increases.</t> - - <t>Stream Performance Tests (TCP, UDP) to measure bulk data transfer - performance, i.e. how fast systems can send and receive data through - the switch.</t> - - <t>Control Path and Datapath Coupling Tests, to understand how - closely coupled the datapath and the control path are as well as the - effect of this coupling on the performance of the DUT (example: - delay of the initial packet of a flow).</t> - - <t>CPU and Memory Consumption Tests to understand the virtual - switch’s footprint on the system, usually conducted as - auxiliary measurements with benchmarks above. They include: CPU - utilization, Cache utilization and Memory footprint.</t> - </list></t> - - <t>Future/planned test specs include:<list style="symbols"> - <t>Request/Response Performance Tests (TCP, UDP) which measure the - transaction rate through the switch.</t> - - <t>Noisy Neighbour Tests, to understand the effects of resource - sharing on the performance of a virtual switch.</t> - - <t>Tests derived from examination of ETSI NFV Draft GS IFA003 - requirements <xref target="IFA003"/> on characterization of - acceleration technologies applied to vswitches.</t> - </list>The flexibility of deployment of a virtual switch within a - network means that the BMWG IETF existing literature needs to be used to - characterize the performance of a switch in various deployment - scenarios. The deployment scenarios under consideration include:</t> - - <t><figure> - <preamble>Physical port to virtual switch to physical - port</preamble> - - <artwork><![CDATA[ __ - +--------------------------------------------------+ | - | +--------------------+ | | - | | | | | - | | v | | Host - | +--------------+ +--------------+ | | - | | phy port | vSwitch | phy port | | | - +---+--------------+------------+--------------+---+ __| - ^ : - | | - : v - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - </figure></t> - - <t><figure> - <preamble>Physical port to virtual switch to VNF to virtual switch - to physical port</preamble> - - <artwork><![CDATA[ __ - +---------------------------------------------------+ | - | | | - | +-------------------------------------------+ | | - | | Application | | | - | +-------------------------------------------+ | | - | ^ : | | - | | | | | Guest - | : v | | - | +---------------+ +---------------+ | | - | | logical port 0| | logical port 1| | | - +---+---------------+-----------+---------------+---+ __| - ^ : - | | - : v __ - +---+---------------+----------+---------------+---+ | - | | logical port 0| | logical port 1| | | - | +---------------+ +---------------+ | | - | ^ : | | - | | | | | Host - | : v | | - | +--------------+ +--------------+ | | - | | phy port | vSwitch | phy port | | | - +---+--------------+------------+--------------+---+ __| - ^ : - | | - : v - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - </figure><figure> - <preamble>Physical port to virtual switch to VNF to virtual switch - to VNF to virtual switch to physical port</preamble> - - <artwork><![CDATA[ __ - +----------------------+ +----------------------+ | - | Guest 1 | | Guest 2 | | - | +---------------+ | | +---------------+ | | - | | Application | | | | Application | | | - | +---------------+ | | +---------------+ | | - | ^ | | | ^ | | | - | | v | | | v | | Guests - | +---------------+ | | +---------------+ | | - | | logical ports | | | | logical ports | | | - | | 0 1 | | | | 0 1 | | | - +---+---------------+--+ +---+---------------+--+__| - ^ : ^ : - | | | | - : v : v _ - +---+---------------+---------+---------------+--+ | - | | 0 1 | | 3 4 | | | - | | logical ports | | logical ports | | | - | +---------------+ +---------------+ | | - | ^ | ^ | | | Host - | | |-----------------| v | | - | +--------------+ +--------------+ | | - | | phy ports | vSwitch | phy ports | | | - +---+--------------+----------+--------------+---+_| - ^ : - | | - : v - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - </figure><figure> - <preamble>Physical port to virtual switch to VNF</preamble> - - <artwork><![CDATA[ __ - +---------------------------------------------------+ | - | | | - | +-------------------------------------------+ | | - | | Application | | | - | +-------------------------------------------+ | | - | ^ | | - | | | | Guest - | : | | - | +---------------+ | | - | | logical port 0| | | - +---+---------------+-------------------------------+ __| - ^ - | - : __ - +---+---------------+------------------------------+ | - | | logical port 0| | | - | +---------------+ | | - | ^ | | - | | | | Host - | : | | - | +--------------+ | | - | | phy port | vSwitch | | - +---+--------------+------------ -------------- ---+ __| - ^ - | - : - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - </figure><figure> - <preamble>VNF to virtual switch to physical port</preamble> - - <artwork><![CDATA[ __ - +---------------------------------------------------+ | - | | | - | +-------------------------------------------+ | | - | | Application | | | - | +-------------------------------------------+ | | - | : | | - | | | | Guest - | v | | - | +---------------+ | | - | | logical port | | | - +-------------------------------+---------------+---+ __| - : - | - v __ - +------------------------------+---------------+---+ | - | | logical port | | | - | +---------------+ | | - | : | | - | | | | Host - | v | | - | +--------------+ | | - | vSwitch | phy port | | | - +-------------------------------+--------------+---+ __| - : - | - v - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - </figure><figure> - <preamble>VNF to virtual switch to VNF</preamble> - - <artwork><![CDATA[ __ - +----------------------+ +----------------------+ | - | Guest 1 | | Guest 2 | | - | +---------------+ | | +---------------+ | | - | | Application | | | | Application | | | - | +---------------+ | | +---------------+ | | - | | | | ^ | | - | v | | | | | Guests - | +---------------+ | | +---------------+ | | - | | logical ports | | | | logical ports | | | - | | 0 | | | | 0 | | | - +---+---------------+--+ +---+---------------+--+__| - : ^ - | | - v : _ - +---+---------------+---------+---------------+--+ | - | | 1 | | 1 | | | - | | logical ports | | logical ports | | | - | +---------------+ +---------------+ | | - | | ^ | | Host - | L-----------------+ | | - | | | - | vSwitch | | - +------------------------------------------------+_|]]></artwork> - </figure></t> - - <t>A set of Deployment Scenario figures is available on the VSPERF Test - Methodology Wiki page <xref target="TestTopo"/>. </t> - </section> - - <section title="3x3 Matrix Coverage"> - <t>This section organizes the many existing test specifications into the - "3x3" matrix (introduced in <xref target="I-D.ietf-bmwg-virtual-net"/>). - Because the LTD specification ID names are quite long, this section is - organized into lists for each occupied cell of the matrix (not all are - occupied, also the matrix has grown to 3x4 to accommodate scale metrics - when displaying the coverage of many metrics/benchmarks).</t> - - <t>The tests listed below assess the activation of paths in the data - plane, rather than the control plane.</t> - - <t>A complete list of tests with short summaries is available on the - VSPERF "LTD Test Spec Overview" Wiki page <xref target="LTDoverV"/>.</t> - - <section title="Speed of Activation"> - <t><list style="symbols"> - <t>Activation.RFC2889.AddressLearningRate</t> - - <t>PacketLatency.InitialPacketProcessingLatency</t> - </list></t> - </section> - - <section title="Accuracy of Activation section"> - <t><list style="symbols"> - <t>CPDP.Coupling.Flow.Addition</t> - </list></t> - </section> - - <section title="Reliability of Activation"> - <t><list style="symbols"> - <t>Throughput.RFC2544.SystemRecoveryTime</t> - - <t>Throughput.RFC2544.ResetTime</t> - </list></t> - </section> - - <section title="Scale of Activation"> - <t><list style="symbols"> - <t>Activation.RFC2889.AddressCachingCapacity</t> - </list></t> - </section> - - <section title="Speed of Operation"> - <t><list style="symbols"> - <t>Throughput.RFC2544.PacketLossRate</t> - - <t>CPU.RFC2544.0PacketLoss</t> - - <t>Throughput.RFC2544.PacketLossRateFrameModification</t> - - <t>Throughput.RFC2544.BackToBackFrames</t> - - <t>Throughput.RFC2889.MaxForwardingRate</t> - - <t>Throughput.RFC2889.ForwardPressure</t> - - <t>Throughput.RFC2889.BroadcastFrameForwarding</t> - </list></t> - </section> - - <section title="Accuracy of Operation"> - <t><list style="symbols"> - <t>Throughput.RFC2889.ErrorFramesFiltering</t> - - <t>Throughput.RFC2544.Profile</t> - </list></t> - </section> - - <section title="Reliability of Operation"> - <t><list style="symbols"> - <t>Throughput.RFC2889.Soak</t> - - <t>Throughput.RFC2889.SoakFrameModification</t> - - <t>PacketDelayVariation.RFC3393.Soak</t> - </list></t> - </section> - - <section title="Scalability of Operation"> - <t><list style="symbols"> - <t>Scalability.RFC2544.0PacketLoss</t> - - <t>MemoryBandwidth.RFC2544.0PacketLoss.Scalability</t> - </list></t> - </section> - - <section title="Summary"> - <t><figure> - <artwork><![CDATA[|------------------------------------------------------------------------| -| | | | | | -| | SPEED | ACCURACY | RELIABILITY | SCALE | -| | | | | | -|------------------------------------------------------------------------| -| | | | | | -| Activation | X | X | X | X | -| | | | | | -|------------------------------------------------------------------------| -| | | | | | -| Operation | X | X | X | X | -| | | | | | -|------------------------------------------------------------------------| -| | | | | | -| De-activation | | | | | -| | | | | | -|------------------------------------------------------------------------|]]></artwork> - </figure></t> - </section> - </section> - - <section title="Security Considerations"> - <t>Benchmarking activities as described in this memo are limited to - technology characterization of a Device Under Test/System Under Test - (DUT/SUT) using controlled stimuli in a laboratory environment, with - dedicated address space and the constraints specified in the sections - above.</t> - - <t>The benchmarking network topology will be an independent test setup - and MUST NOT be connected to devices that may forward the test traffic - into a production network, or misroute traffic to the test management - network.</t> - - <t>Further, benchmarking is performed on a "black-box" basis, relying - solely on measurements observable external to the DUT/SUT.</t> - - <t>Special capabilities SHOULD NOT exist in the DUT/SUT specifically for - benchmarking purposes. Any implications for network security arising - from the DUT/SUT SHOULD be identical in the lab and in production - networks.</t> - </section> - - <section anchor="IANA" title="IANA Considerations"> - <t>No IANA Action is requested at this time.</t> - </section> - - <section title="Acknowledgements"> - <t>The authors acknowledge</t> - </section> - </middle> - - <back> - <references title="Normative References"> - <?rfc ?> - - <?rfc include="reference.RFC.2119"?> - - <?rfc include="reference.RFC.2330"?> - - <?rfc include='reference.RFC.2544'?> - - <?rfc include="reference.RFC.2679"?> - - <?rfc include='reference.RFC.2680'?> - - <?rfc include='reference.RFC.3393'?> - - <?rfc include='reference.RFC.3432'?> - - <?rfc include='reference.RFC.2681'?> - - <?rfc include='reference.RFC.5905'?> - - <?rfc include='reference.RFC.4689'?> - - <?rfc include='reference.RFC.4737'?> - - <?rfc include='reference.RFC.5357'?> - - <?rfc include='reference.RFC.2889'?> - - <?rfc include='reference.RFC.3918'?> - - <?rfc include='reference.RFC.6201'?> - - <?rfc include='reference.RFC.2285'?> - - <reference anchor="NFV.PER001"> - <front> - <title>Network Function Virtualization: Performance and Portability - Best Practices</title> - - <author fullname="ETSI NFV" initials="" surname=""> - <organization/> - </author> - - <date month="June" year="2014"/> - </front> - - <seriesInfo name="Group Specification" - value="ETSI GS NFV-PER 001 V1.1.1 (2014-06)"/> - - <format type="PDF"/> - </reference> - </references> - - <references title="Informative References"> - <?rfc include='reference.RFC.1242'?> - - <?rfc include='reference.RFC.5481'?> - - <?rfc include='reference.RFC.6049'?> - - <?rfc include='reference.RFC.6248'?> - - <?rfc include='reference.RFC.6390'?> - - <?rfc include='reference.I-D.ietf-bmwg-virtual-net'?> - - <reference anchor="TestTopo"> - <front> - <title>Test Topologies - https://wiki.opnfv.org/vsperf/test_methodology</title> - - <author> - <organization/> - </author> - - <date/> - </front> - </reference> - - <reference anchor="LTDoverV"> - <front> - <title>LTD Test Spec Overview - https://wiki.opnfv.org/wiki/vswitchperf_test_spec_review </title> - - <author> - <organization/> - </author> - - <date/> - </front> - </reference> - - <reference anchor="IFA003"> - <front> - <title>https://docbox.etsi.org/ISG/NFV/Open/Drafts/IFA003_Acceleration_-_vSwitch_Spec/</title> - - <author> - <organization/> - </author> - - <date/> - </front> - </reference> - </references> - </back> -</rfc> diff --git a/docs/testing/developer/devguide/requirements/ietf_draft/draft-vsperf-bmwg-vswitch-opnfv-01.xml b/docs/testing/developer/devguide/requirements/ietf_draft/draft-vsperf-bmwg-vswitch-opnfv-01.xml deleted file mode 100644 index a9405a77..00000000 --- a/docs/testing/developer/devguide/requirements/ietf_draft/draft-vsperf-bmwg-vswitch-opnfv-01.xml +++ /dev/null @@ -1,1016 +0,0 @@ -<?xml version="1.0" encoding="US-ASCII"?> -<!DOCTYPE rfc SYSTEM "rfc2629.dtd"> -<?rfc toc="yes"?> -<?rfc tocompact="yes"?> -<?rfc tocdepth="3"?> -<?rfc tocindent="yes"?> -<?rfc symrefs="yes"?> -<?rfc sortrefs="yes"?> -<?rfc comments="yes"?> -<?rfc inline="yes"?> -<?rfc compact="yes"?> -<?rfc subcompact="no"?> -<rfc category="info" docName="draft-vsperf-bmwg-vswitch-opnfv-02" - ipr="trust200902"> - <front> - <title abbrev="Benchmarking vSwitches">Benchmarking Virtual Switches in - OPNFV</title> - - <author fullname="Maryam Tahhan" initials="M." surname="Tahhan"> - <organization>Intel</organization> - - <address> - <postal> - <street/> - - <city/> - - <region/> - - <code/> - - <country/> - </postal> - - <phone/> - - <facsimile/> - - <email>maryam.tahhan@intel.com</email> - - <uri/> - </address> - </author> - - <author fullname="Billy O'Mahony" initials="B." surname="O'Mahony"> - <organization>Intel</organization> - - <address> - <postal> - <street/> - - <city/> - - <region/> - - <code/> - - <country/> - </postal> - - <phone/> - - <facsimile/> - - <email>billy.o.mahony@intel.com</email> - - <uri/> - </address> - </author> - - <author fullname="Al Morton" initials="A." surname="Morton"> - <organization>AT&T Labs</organization> - - <address> - <postal> - <street>200 Laurel Avenue South</street> - - <city>Middletown,</city> - - <region>NJ</region> - - <code>07748</code> - - <country>USA</country> - </postal> - - <phone>+1 732 420 1571</phone> - - <facsimile>+1 732 368 1192</facsimile> - - <email>acmorton@att.com</email> - - <uri>http://home.comcast.net/~acmacm/</uri> - </address> - </author> - - <date day="20" month="March" year="2016"/> - - <abstract> - <t>This memo describes the progress of the Open Platform for NFV (OPNFV) - project on virtual switch performance "VSWITCHPERF". This project - intends to build on the current and completed work of the Benchmarking - Methodology Working Group in IETF, by referencing existing literature. - The Benchmarking Methodology Working Group has traditionally conducted - laboratory characterization of dedicated physical implementations of - internetworking functions. Therefore, this memo begins to describe the - additional considerations when virtual switches are implemented in - general-purpose hardware. The expanded tests and benchmarks are also - influenced by the OPNFV mission to support virtualization of the "telco" - infrastructure.</t> - </abstract> - - <note title="Requirements Language"> - <t>The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", - "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this - document are to be interpreted as described in <xref - target="RFC2119">RFC 2119</xref>.</t> - - <t/> - </note> - </front> - - <middle> - <section title="Introduction"> - <t>Benchmarking Methodology Working Group (BMWG) has traditionally - conducted laboratory characterization of dedicated physical - implementations of internetworking functions. The Black-box Benchmarks - of Throughput, Latency, Forwarding Rates and others have served our - industry for many years. Now, Network Function Virtualization (NFV) has - the goal to transform how internetwork functions are implemented, and - therefore has garnered much attention.</t> - - <t>This memo summarizes the progress of the Open Platform for NFV - (OPNFV) project on virtual switch performance characterization, - "VSWITCHPERF", through the Brahmaputra (second) release <xref - target="BrahRel"/>. This project intends to build on the current and - completed work of the Benchmarking Methodology Working Group in IETF, by - referencing existing literature. For example, currently the most often - referenced RFC is <xref target="RFC2544"/> (which depends on <xref - target="RFC1242"/>) and foundation of the benchmarking work in OPNFV is - common and strong.</t> - - <t>See - https://wiki.opnfv.org/characterize_vswitch_performance_for_telco_nfv_use_cases - for more background, and the OPNFV website for general information: - https://www.opnfv.org/</t> - - <t>The authors note that OPNFV distinguishes itself from other open - source compute and networking projects through its emphasis on existing - "telco" services as opposed to cloud-computing. There are many ways in - which telco requirements have different emphasis on performance - dimensions when compared to cloud computing: support for and transfer of - isochronous media streams is one example.</t> - - <t>Note also that the move to NFV Infrastructure has resulted in many - new benchmarking initiatives across the industry. The authors are - currently doing their best to maintain alignment with many other - projects, and this Internet Draft is one part of the efforts. We - acknowledge the early work in <xref - target="I-D.huang-bmwg-virtual-network-performance"/>, and useful - discussion with the authors.</t> - </section> - - <section title="Scope"> - <t>The primary purpose and scope of the memo is to inform the industry - of work-in-progress that builds on the body of extensive BMWG literature - and experience, and describe the extensions needed for benchmarking - virtual switches. Inital feedback indicates that many of these - extensions may be applicable beyond the current scope (to hardware - switches in the NFV Infrastructure and to virtual routers, for example). - Additionally, this memo serves as a vehicle to include more detail and - commentary from BMWG and other Open Source communities, under BMWG's - chartered work to characterize the NFV Infrastructure (a virtual switch - is an important aspect of that infrastructure).</t> - </section> - - <section title="Benchmarking Considerations"> - <t>This section highlights some specific considerations (from <xref - target="I-D.ietf-bmwg-virtual-net"/>)related to Benchmarks for virtual - switches. The OPNFV project is sharing its present view on these areas, - as they develop their specifications in the Level Test Design (LTD) - document.</t> - - <section title="Comparison with Physical Network Functions"> - <t>To compare the performance of virtual designs and implementations - with their physical counterparts, identical benchmarks are needed. - BMWG has developed specifications for many network functions this memo - re-uses existing benchmarks through references, and expands them - during development of new methods. A key configuration aspect is the - number of parallel cores required to achieve comparable performance - with a given physical device, or whether some limit of scale was - reached before the cores could achieve the comparable level.</t> - - <t>It's unlikely that the virtual switch will be the only application - running on the SUT, so CPU utilization, Cache utilization, and Memory - footprint should also be recorded for the virtual implementations of - internetworking functions.</t> - </section> - - <section title="Continued Emphasis on Black-Box Benchmarks"> - <t>External observations remain essential as the basis for Benchmarks. - Internal observations with fixed specification and interpretation will - be provided in parallel to assist the development of operations - procedures when the technology is deployed.</t> - </section> - - <section title="New Configuration Parameters"> - <t>A key consideration when conducting any sort of benchmark is trying - to ensure the consistency and repeatability of test results. When - benchmarking the performance of a vSwitch there are many factors that - can affect the consistency of results, one key factor is matching the - various hardware and software details of the SUT. This section lists - some of the many new parameters which this project believes are - critical to report in order to achieve repeatability.</t> - - <t>Hardware details including:</t> - - <t><list style="symbols"> - <t>Platform details</t> - - <t>Processor details</t> - - <t>Memory information (type and size)</t> - - <t>Number of enabled cores</t> - - <t>Number of cores used for the test</t> - - <t>Number of physical NICs, as well as their details - (manufacturer, versions, type and the PCI slot they are plugged - into)</t> - - <t>NIC interrupt configuration</t> - - <t>BIOS version, release date and any configurations that were - modified</t> - - <t>CPU microcode level</t> - - <t>Memory DIMM configurations (quad rank performance may not be - the same as dual rank) in size, freq and slot locations</t> - - <t>PCI configuration parameters (payload size, early ack - option...)</t> - - <t>Power management at all levels (ACPI sleep states, processor - package, OS...)</t> - </list>Software details including:</t> - - <t><list style="symbols"> - <t>OS parameters and behavior (text vs graphical no one typing at - the console on one system)</t> - - <t>OS version (for host and VNF)</t> - - <t>Kernel version (for host and VNF)</t> - - <t>GRUB boot parameters (for host and VNF)</t> - - <t>Hypervisor details (Type and version)</t> - - <t>Selected vSwitch, version number or commit id used</t> - - <t>vSwitch launch command line if it has been parameterised</t> - - <t>Memory allocation to the vSwitch</t> - - <t>which NUMA node it is using, and how many memory channels</t> - - <t>DPDK or any other SW dependency version number or commit id - used</t> - - <t>Memory allocation to a VM - if it's from Hugpages/elsewhere</t> - - <t>VM storage type: snapshot/independent persistent/independent - non-persistent</t> - - <t>Number of VMs</t> - - <t>Number of Virtual NICs (vNICs), versions, type and driver</t> - - <t>Number of virtual CPUs and their core affinity on the host</t> - - <t>Number vNIC interrupt configuration</t> - - <t>Thread affinitization for the applications (including the - vSwitch itself) on the host</t> - - <t>Details of Resource isolation, such as CPUs designated for - Host/Kernel (isolcpu) and CPUs designated for specific processes - (taskset). - Test duration. - Number of flows.</t> - </list></t> - - <t>Test Traffic Information:<list style="symbols"> - <t>Traffic type - UDP, TCP, IMIX / Other</t> - - <t>Packet Sizes</t> - - <t>Deployment Scenario</t> - </list></t> - - <t/> - </section> - - <section title="Flow classification"> - <t>Virtual switches group packets into flows by processing and - matching particular packet or frame header information, or by matching - packets based on the input ports. Thus a flow can be thought of a - sequence of packets that have the same set of header field values or - have arrived on the same port. Performance results can vary based on - the parameters the vSwitch uses to match for a flow. The recommended - flow classification parameters for any vSwitch performance tests are: - the input port, the source IP address, the destination IP address and - the Ethernet protocol type field. It is essential to increase the flow - timeout time on a vSwitch before conducting any performance tests that - do not measure the flow setup time. Normally the first packet of a - particular stream will install the flow in the virtual switch which - adds an additional latency, subsequent packets of the same flow are - not subject to this latency if the flow is already installed on the - vSwitch.</t> - </section> - - <section title="Benchmarks using Baselines with Resource Isolation"> - <t>This outline describes measurement of baseline with isolated - resources at a high level, which is the intended approach at this - time.</t> - - <t><list style="numbers"> - <t>Baselines: <list style="symbols"> - <t>Optional: Benchmark platform forwarding capability without - a vswitch or VNF for at least 72 hours (serves as a means of - platform validation and a means to obtain the base performance - for the platform in terms of its maximum forwarding rate and - latency). <figure> - <preamble>Benchmark platform forwarding - capability</preamble> - - <artwork align="right"><![CDATA[ __ - +--------------------------------------------------+ | - | +------------------------------------------+ | | - | | | | | - | | Simple Forwarding App | | Host - | | | | | - | +------------------------------------------+ | | - | | NIC | | | - +---+------------------------------------------+---+ __| - ^ : - | | - : v - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - - <postamble/> - </figure></t> - - <t>Benchmark VNF forwarding capability with direct - connectivity (vSwitch bypass, e.g., SR/IOV) for at least 72 - hours (serves as a means of VNF validation and a means to - obtain the base performance for the VNF in terms of its - maximum forwarding rate and latency). The metrics gathered - from this test will serve as a key comparison point for - vSwitch bypass technologies performance and vSwitch - performance. <figure align="right"> - <preamble>Benchmark VNF forwarding capability</preamble> - - <artwork><![CDATA[ __ - +--------------------------------------------------+ | - | +------------------------------------------+ | | - | | | | | - | | VNF | | | - | | | | | - | +------------------------------------------+ | | - | | Passthrough/SR-IOV | | Host - | +------------------------------------------+ | | - | | NIC | | | - +---+------------------------------------------+---+ __| - ^ : - | | - : v - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - - <postamble/> - </figure></t> - - <t>Benchmarking with isolated resources alone, with other - resources (both HW&SW) disabled Example, vSw and VM are - SUT</t> - - <t>Benchmarking with isolated resources alone, leaving some - resources unused</t> - - <t>Benchmark with isolated resources and all resources - occupied</t> - </list></t> - - <t>Next Steps<list style="symbols"> - <t>Limited sharing</t> - - <t>Production scenarios</t> - - <t>Stressful scenarios</t> - </list></t> - </list></t> - </section> - </section> - - <section title="VSWITCHPERF Specification Summary"> - <t>The overall specification in preparation is referred to as a Level - Test Design (LTD) document, which will contain a suite of performance - tests. The base performance tests in the LTD are based on the - pre-existing specifications developed by BMWG to test the performance of - physical switches. These specifications include:</t> - - <t><list style="symbols"> - <t><xref target="RFC2544"/> Benchmarking Methodology for Network - Interconnect Devices</t> - - <t><xref target="RFC2889"/> Benchmarking Methodology for LAN - Switching</t> - - <t><xref target="RFC6201"/> Device Reset Characterization</t> - - <t><xref target="RFC5481"/> Packet Delay Variation Applicability - Statement</t> - </list></t> - - <t>Some of the above/newer RFCs are being applied in benchmarking for - the first time, and represent a development challenge for test equipment - developers. Fortunately, many members of the testing system community - have engaged on the VSPERF project, including an open source test - system.</t> - - <t>In addition to this, the LTD also re-uses the terminology defined - by:</t> - - <t><list style="symbols"> - <t><xref target="RFC2285"/> Benchmarking Terminology for LAN - Switching Devices</t> - - <t><xref target="RFC5481"/> Packet Delay Variation Applicability - Statement</t> - </list></t> - - <t/> - - <t>Specifications to be included in future updates of the LTD - include:<list style="symbols"> - <t><xref target="RFC3918"/> Methodology for IP Multicast - Benchmarking</t> - - <t><xref target="RFC4737"/> Packet Reordering Metrics</t> - </list></t> - - <t>As one might expect, the most fundamental internetworking - characteristics of Throughput and Latency remain important when the - switch is virtualized, and these benchmarks figure prominently in the - specification.</t> - - <t>When considering characteristics important to "telco" network - functions, we must begin to consider additional performance metrics. In - this case, the project specifications have referenced metrics from the - IETF IP Performance Metrics (IPPM) literature. This means that the <xref - target="RFC2544"/> test of Latency is replaced by measurement of a - metric derived from IPPM's <xref target="RFC2679"/>, where a set of - statistical summaries will be provided (mean, max, min, etc.). Further - metrics planned to be benchmarked include packet delay variation as - defined by <xref target="RFC5481"/> , reordering, burst behaviour, DUT - availability, DUT capacity and packet loss in long term testing at - Throughput level, where some low-level of background loss may be present - and characterized.</t> - - <t>Tests have been (or will be) designed to collect the metrics - below:</t> - - <t><list style="symbols"> - <t>Throughput Tests to measure the maximum forwarding rate (in - frames per second or fps) and bit rate (in Mbps) for a constant load - (as defined by <xref target="RFC1242"/>) without traffic loss.</t> - - <t>Packet and Frame Delay Distribution Tests to measure average, min - and max packet and frame delay for constant loads.</t> - - <t>Packet Delay Tests to understand latency distribution for - different packet sizes and over an extended test run to uncover - outliers.</t> - - <t>Scalability Tests to understand how the virtual switch performs - as the number of flows, active ports, complexity of the forwarding - logic’s configuration… it has to deal with - increases.</t> - - <t>Stream Performance Tests (TCP, UDP) to measure bulk data transfer - performance, i.e. how fast systems can send and receive data through - the switch.</t> - - <t>Control Path and Datapath Coupling Tests, to understand how - closely coupled the datapath and the control path are as well as the - effect of this coupling on the performance of the DUT (example: - delay of the initial packet of a flow).</t> - - <t>CPU and Memory Consumption Tests to understand the virtual - switch’s footprint on the system, usually conducted as - auxiliary measurements with benchmarks above. They include: CPU - utilization, Cache utilization and Memory footprint.</t> - - <t>The so-called "Soak" tests, where the selected test is conducted - over a long period of time (with an ideal duration of 24 hours, and - at least 6 hours). The purpose of soak tests is to capture transient - changes in performance which may occur due to infrequent processes - or the low probability coincidence of two or more processes. The - performance must be evaluated periodically during continuous - testing, and this results in use of <xref target="RFC2889"/> Frame - Rate metrics instead of <xref target="RFC2544"/> Throughput (which - requires stopping traffic to allow time for all traffic to exit - internal queues).</t> - </list></t> - - <t>Future/planned test specs include:<list style="symbols"> - <t>Request/Response Performance Tests (TCP, UDP) which measure the - transaction rate through the switch.</t> - - <t>Noisy Neighbour Tests, to understand the effects of resource - sharing on the performance of a virtual switch.</t> - - <t>Tests derived from examination of ETSI NFV Draft GS IFA003 - requirements <xref target="IFA003"/> on characterization of - acceleration technologies applied to vswitches.</t> - </list>The flexibility of deployment of a virtual switch within a - network means that the BMWG IETF existing literature needs to be used to - characterize the performance of a switch in various deployment - scenarios. The deployment scenarios under consideration include:</t> - - <t><figure> - <preamble>Physical port to virtual switch to physical - port</preamble> - - <artwork><![CDATA[ __ - +--------------------------------------------------+ | - | +--------------------+ | | - | | | | | - | | v | | Host - | +--------------+ +--------------+ | | - | | phy port | vSwitch | phy port | | | - +---+--------------+------------+--------------+---+ __| - ^ : - | | - : v - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - </figure></t> - - <t><figure> - <preamble>Physical port to virtual switch to VNF to virtual switch - to physical port</preamble> - - <artwork><![CDATA[ __ - +---------------------------------------------------+ | - | | | - | +-------------------------------------------+ | | - | | Application | | | - | +-------------------------------------------+ | | - | ^ : | | - | | | | | Guest - | : v | | - | +---------------+ +---------------+ | | - | | logical port 0| | logical port 1| | | - +---+---------------+-----------+---------------+---+ __| - ^ : - | | - : v __ - +---+---------------+----------+---------------+---+ | - | | logical port 0| | logical port 1| | | - | +---------------+ +---------------+ | | - | ^ : | | - | | | | | Host - | : v | | - | +--------------+ +--------------+ | | - | | phy port | vSwitch | phy port | | | - +---+--------------+------------+--------------+---+ __| - ^ : - | | - : v - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - </figure><figure> - <preamble>Physical port to virtual switch to VNF to virtual switch - to VNF to virtual switch to physical port</preamble> - - <artwork><![CDATA[ __ - +----------------------+ +----------------------+ | - | Guest 1 | | Guest 2 | | - | +---------------+ | | +---------------+ | | - | | Application | | | | Application | | | - | +---------------+ | | +---------------+ | | - | ^ | | | ^ | | | - | | v | | | v | | Guests - | +---------------+ | | +---------------+ | | - | | logical ports | | | | logical ports | | | - | | 0 1 | | | | 0 1 | | | - +---+---------------+--+ +---+---------------+--+__| - ^ : ^ : - | | | | - : v : v _ - +---+---------------+---------+---------------+--+ | - | | 0 1 | | 3 4 | | | - | | logical ports | | logical ports | | | - | +---------------+ +---------------+ | | - | ^ | ^ | | | Host - | | |-----------------| v | | - | +--------------+ +--------------+ | | - | | phy ports | vSwitch | phy ports | | | - +---+--------------+----------+--------------+---+_| - ^ : - | | - : v - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - </figure><figure> - <preamble>Physical port to virtual switch to VNF</preamble> - - <artwork><![CDATA[ __ - +---------------------------------------------------+ | - | | | - | +-------------------------------------------+ | | - | | Application | | | - | +-------------------------------------------+ | | - | ^ | | - | | | | Guest - | : | | - | +---------------+ | | - | | logical port 0| | | - +---+---------------+-------------------------------+ __| - ^ - | - : __ - +---+---------------+------------------------------+ | - | | logical port 0| | | - | +---------------+ | | - | ^ | | - | | | | Host - | : | | - | +--------------+ | | - | | phy port | vSwitch | | - +---+--------------+------------ -------------- ---+ __| - ^ - | - : - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - </figure><figure> - <preamble>VNF to virtual switch to physical port</preamble> - - <artwork><![CDATA[ __ - +---------------------------------------------------+ | - | | | - | +-------------------------------------------+ | | - | | Application | | | - | +-------------------------------------------+ | | - | : | | - | | | | Guest - | v | | - | +---------------+ | | - | | logical port | | | - +-------------------------------+---------------+---+ __| - : - | - v __ - +------------------------------+---------------+---+ | - | | logical port | | | - | +---------------+ | | - | : | | - | | | | Host - | v | | - | +--------------+ | | - | vSwitch | phy port | | | - +-------------------------------+--------------+---+ __| - : - | - v - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - </figure><figure> - <preamble>VNF to virtual switch to VNF</preamble> - - <artwork><![CDATA[ __ - +----------------------+ +----------------------+ | - | Guest 1 | | Guest 2 | | - | +---------------+ | | +---------------+ | | - | | Application | | | | Application | | | - | +---------------+ | | +---------------+ | | - | | | | ^ | | - | v | | | | | Guests - | +---------------+ | | +---------------+ | | - | | logical ports | | | | logical ports | | | - | | 0 | | | | 0 | | | - +---+---------------+--+ +---+---------------+--+__| - : ^ - | | - v : _ - +---+---------------+---------+---------------+--+ | - | | 1 | | 1 | | | - | | logical ports | | logical ports | | | - | +---------------+ +---------------+ | | - | | ^ | | Host - | L-----------------+ | | - | | | - | vSwitch | | - +------------------------------------------------+_|]]></artwork> - </figure></t> - - <t>A set of Deployment Scenario figures is available on the VSPERF Test - Methodology Wiki page <xref target="TestTopo"/>.</t> - </section> - - <section title="3x3 Matrix Coverage"> - <t>This section organizes the many existing test specifications into the - "3x3" matrix (introduced in <xref target="I-D.ietf-bmwg-virtual-net"/>). - Because the LTD specification ID names are quite long, this section is - organized into lists for each occupied cell of the matrix (not all are - occupied, also the matrix has grown to 3x4 to accommodate scale metrics - when displaying the coverage of many metrics/benchmarks). The current - version of the LTD specification is available <xref target="LTD"/>.</t> - - <t>The tests listed below assess the activation of paths in the data - plane, rather than the control plane.</t> - - <t>A complete list of tests with short summaries is available on the - VSPERF "LTD Test Spec Overview" Wiki page <xref target="LTDoverV"/>.</t> - - <section title="Speed of Activation"> - <t><list style="symbols"> - <t>Activation.RFC2889.AddressLearningRate</t> - - <t>PacketLatency.InitialPacketProcessingLatency</t> - </list></t> - </section> - - <section title="Accuracy of Activation section"> - <t><list style="symbols"> - <t>CPDP.Coupling.Flow.Addition</t> - </list></t> - </section> - - <section title="Reliability of Activation"> - <t><list style="symbols"> - <t>Throughput.RFC2544.SystemRecoveryTime</t> - - <t>Throughput.RFC2544.ResetTime</t> - </list></t> - </section> - - <section title="Scale of Activation"> - <t><list style="symbols"> - <t>Activation.RFC2889.AddressCachingCapacity</t> - </list></t> - </section> - - <section title="Speed of Operation"> - <t><list style="symbols"> - <t>Throughput.RFC2544.PacketLossRate</t> - - <t>CPU.RFC2544.0PacketLoss</t> - - <t>Throughput.RFC2544.PacketLossRateFrameModification</t> - - <t>Throughput.RFC2544.BackToBackFrames</t> - - <t>Throughput.RFC2889.MaxForwardingRate</t> - - <t>Throughput.RFC2889.ForwardPressure</t> - - <t>Throughput.RFC2889.BroadcastFrameForwarding</t> - </list></t> - </section> - - <section title="Accuracy of Operation"> - <t><list style="symbols"> - <t>Throughput.RFC2889.ErrorFramesFiltering</t> - - <t>Throughput.RFC2544.Profile</t> - </list></t> - </section> - - <section title="Reliability of Operation"> - <t><list style="symbols"> - <t>Throughput.RFC2889.Soak</t> - - <t>Throughput.RFC2889.SoakFrameModification</t> - - <t>PacketDelayVariation.RFC3393.Soak</t> - </list></t> - </section> - - <section title="Scalability of Operation"> - <t><list style="symbols"> - <t>Scalability.RFC2544.0PacketLoss</t> - - <t>MemoryBandwidth.RFC2544.0PacketLoss.Scalability</t> - </list></t> - </section> - - <section title="Summary"> - <t><figure> - <artwork><![CDATA[|------------------------------------------------------------------------| -| | | | | | -| | SPEED | ACCURACY | RELIABILITY | SCALE | -| | | | | | -|------------------------------------------------------------------------| -| | | | | | -| Activation | X | X | X | X | -| | | | | | -|------------------------------------------------------------------------| -| | | | | | -| Operation | X | X | X | X | -| | | | | | -|------------------------------------------------------------------------| -| | | | | | -| De-activation | | | | | -| | | | | | -|------------------------------------------------------------------------|]]></artwork> - </figure></t> - </section> - </section> - - <section title="Security Considerations"> - <t>Benchmarking activities as described in this memo are limited to - technology characterization of a Device Under Test/System Under Test - (DUT/SUT) using controlled stimuli in a laboratory environment, with - dedicated address space and the constraints specified in the sections - above.</t> - - <t>The benchmarking network topology will be an independent test setup - and MUST NOT be connected to devices that may forward the test traffic - into a production network, or misroute traffic to the test management - network.</t> - - <t>Further, benchmarking is performed on a "black-box" basis, relying - solely on measurements observable external to the DUT/SUT.</t> - - <t>Special capabilities SHOULD NOT exist in the DUT/SUT specifically for - benchmarking purposes. Any implications for network security arising - from the DUT/SUT SHOULD be identical in the lab and in production - networks.</t> - </section> - - <section anchor="IANA" title="IANA Considerations"> - <t>No IANA Action is requested at this time.</t> - </section> - - <section title="Acknowledgements"> - <t>The authors appreciate and acknowledge comments from Scott Bradner, - Marius Georgescu, Ramki Krishnan, and Doug Montgomery, and others for - their reviews.</t> - </section> - </middle> - - <back> - <references title="Normative References"> - <?rfc ?> - - <?rfc include="reference.RFC.2119"?> - - <?rfc ?> - - <?rfc include="reference.RFC.2330"?> - - <?rfc include='reference.RFC.2544'?> - - <?rfc include="reference.RFC.2679"?> - - <?rfc include='reference.RFC.2680'?> - - <?rfc include='reference.RFC.3393'?> - - <?rfc include='reference.RFC.3432'?> - - <?rfc include='reference.RFC.2681'?> - - <?rfc include='reference.RFC.5905'?> - - <?rfc include='reference.RFC.4689'?> - - <?rfc include='reference.RFC.4737'?> - - <?rfc include='reference.RFC.5357'?> - - <?rfc include='reference.RFC.2889'?> - - <?rfc include='reference.RFC.3918'?> - - <?rfc include='reference.RFC.6201'?> - - <?rfc include='reference.RFC.2285'?> - - <reference anchor="NFV.PER001"> - <front> - <title>Network Function Virtualization: Performance and Portability - Best Practices</title> - - <author fullname="ETSI NFV" initials="" surname=""> - <organization/> - </author> - - <date month="June" year="2014"/> - </front> - - <seriesInfo name="Group Specification" - value="ETSI GS NFV-PER 001 V1.1.1 (2014-06)"/> - - <format type="PDF"/> - </reference> - </references> - - <references title="Informative References"> - <?rfc include='reference.RFC.1242'?> - - <?rfc include='reference.RFC.5481'?> - - <?rfc include='reference.RFC.6049'?> - - <?rfc include='reference.RFC.6248'?> - - <?rfc include='reference.RFC.6390'?> - - <?rfc include='reference.I-D.ietf-bmwg-virtual-net'?> - - <?rfc include='reference.I-D.huang-bmwg-virtual-network-performance'?> - - <reference anchor="TestTopo"> - <front> - <title>Test Topologies - https://wiki.opnfv.org/vsperf/test_methodology</title> - - <author> - <organization/> - </author> - - <date/> - </front> - </reference> - - <reference anchor="LTDoverV"> - <front> - <title>LTD Test Spec Overview - https://wiki.opnfv.org/wiki/vswitchperf_test_spec_review</title> - - <author> - <organization/> - </author> - - <date/> - </front> - </reference> - - <reference anchor="LTD"> - <front> - <title>LTD Test Specification - http://artifacts.opnfv.org/vswitchperf/docs/requirements/index.html</title> - - <author> - <organization/> - </author> - - <date/> - </front> - </reference> - - <reference anchor="BrahRel"> - <front> - <title>Brahmaputra, Second OPNFV Release - https://www.opnfv.org/brahmaputra</title> - - <author> - <organization/> - </author> - - <date/> - </front> - </reference> - - <reference anchor="IFA003"> - <front> - <title>https://docbox.etsi.org/ISG/NFV/Open/Drafts/IFA003_Acceleration_-_vSwitch_Spec/</title> - - <author> - <organization/> - </author> - - <date/> - </front> - </reference> - </references> - </back> -</rfc> diff --git a/docs/testing/developer/devguide/requirements/ietf_draft/draft-vsperf-bmwg-vswitch-opnfv-02.xml b/docs/testing/developer/devguide/requirements/ietf_draft/draft-vsperf-bmwg-vswitch-opnfv-02.xml deleted file mode 100644 index 9157763e..00000000 --- a/docs/testing/developer/devguide/requirements/ietf_draft/draft-vsperf-bmwg-vswitch-opnfv-02.xml +++ /dev/null @@ -1,1016 +0,0 @@ -<?xml version="1.0" encoding="US-ASCII"?> -<!DOCTYPE rfc SYSTEM "rfc2629.dtd"> -<?rfc toc="yes"?> -<?rfc tocompact="yes"?> -<?rfc tocdepth="3"?> -<?rfc tocindent="yes"?> -<?rfc symrefs="yes"?> -<?rfc sortrefs="yes"?> -<?rfc comments="yes"?> -<?rfc inline="yes"?> -<?rfc compact="yes"?> -<?rfc subcompact="no"?> -<rfc category="info" docName="draft-vsperf-bmwg-vswitch-opnfv-02" - ipr="trust200902"> - <front> - <title abbrev="Benchmarking vSwitches">Benchmarking Virtual Switches in - OPNFV</title> - - <author fullname="Maryam Tahhan" initials="M." surname="Tahhan"> - <organization>Intel</organization> - - <address> - <postal> - <street/> - - <city/> - - <region/> - - <code/> - - <country/> - </postal> - - <phone/> - - <facsimile/> - - <email>maryam.tahhan@intel.com</email> - - <uri/> - </address> - </author> - - <author fullname="Billy O'Mahony" initials="B." surname="O'Mahony"> - <organization>Intel</organization> - - <address> - <postal> - <street/> - - <city/> - - <region/> - - <code/> - - <country/> - </postal> - - <phone/> - - <facsimile/> - - <email>billy.o.mahony@intel.com</email> - - <uri/> - </address> - </author> - - <author fullname="Al Morton" initials="A." surname="Morton"> - <organization>AT&T Labs</organization> - - <address> - <postal> - <street>200 Laurel Avenue South</street> - - <city>Middletown,</city> - - <region>NJ</region> - - <code>07748</code> - - <country>USA</country> - </postal> - - <phone>+1 732 420 1571</phone> - - <facsimile>+1 732 368 1192</facsimile> - - <email>acmorton@att.com</email> - - <uri>http://home.comcast.net/~acmacm/</uri> - </address> - </author> - - <date day="21" month="March" year="2016"/> - - <abstract> - <t>This memo describes the progress of the Open Platform for NFV (OPNFV) - project on virtual switch performance "VSWITCHPERF". This project - intends to build on the current and completed work of the Benchmarking - Methodology Working Group in IETF, by referencing existing literature. - The Benchmarking Methodology Working Group has traditionally conducted - laboratory characterization of dedicated physical implementations of - internetworking functions. Therefore, this memo begins to describe the - additional considerations when virtual switches are implemented in - general-purpose hardware. The expanded tests and benchmarks are also - influenced by the OPNFV mission to support virtualization of the "telco" - infrastructure.</t> - </abstract> - - <note title="Requirements Language"> - <t>The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", - "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this - document are to be interpreted as described in <xref - target="RFC2119">RFC 2119</xref>.</t> - - <t/> - </note> - </front> - - <middle> - <section title="Introduction"> - <t>Benchmarking Methodology Working Group (BMWG) has traditionally - conducted laboratory characterization of dedicated physical - implementations of internetworking functions. The Black-box Benchmarks - of Throughput, Latency, Forwarding Rates and others have served our - industry for many years. Now, Network Function Virtualization (NFV) has - the goal to transform how internetwork functions are implemented, and - therefore has garnered much attention.</t> - - <t>This memo summarizes the progress of the Open Platform for NFV - (OPNFV) project on virtual switch performance characterization, - "VSWITCHPERF", through the Brahmaputra (second) release <xref - target="BrahRel"/>. This project intends to build on the current and - completed work of the Benchmarking Methodology Working Group in IETF, by - referencing existing literature. For example, currently the most often - referenced RFC is <xref target="RFC2544"/> (which depends on <xref - target="RFC1242"/>) and foundation of the benchmarking work in OPNFV is - common and strong.</t> - - <t>See - https://wiki.opnfv.org/characterize_vswitch_performance_for_telco_nfv_use_cases - for more background, and the OPNFV website for general information: - https://www.opnfv.org/</t> - - <t>The authors note that OPNFV distinguishes itself from other open - source compute and networking projects through its emphasis on existing - "telco" services as opposed to cloud-computing. There are many ways in - which telco requirements have different emphasis on performance - dimensions when compared to cloud computing: support for and transfer of - isochronous media streams is one example.</t> - - <t>Note also that the move to NFV Infrastructure has resulted in many - new benchmarking initiatives across the industry. The authors are - currently doing their best to maintain alignment with many other - projects, and this Internet Draft is one part of the efforts. We - acknowledge the early work in <xref - target="I-D.huang-bmwg-virtual-network-performance"/>, and useful - discussion with the authors.</t> - </section> - - <section title="Scope"> - <t>The primary purpose and scope of the memo is to inform the industry - of work-in-progress that builds on the body of extensive BMWG literature - and experience, and describe the extensions needed for benchmarking - virtual switches. Inital feedback indicates that many of these - extensions may be applicable beyond the current scope (to hardware - switches in the NFV Infrastructure and to virtual routers, for example). - Additionally, this memo serves as a vehicle to include more detail and - commentary from BMWG and other Open Source communities, under BMWG's - chartered work to characterize the NFV Infrastructure (a virtual switch - is an important aspect of that infrastructure).</t> - </section> - - <section title="Benchmarking Considerations"> - <t>This section highlights some specific considerations (from <xref - target="I-D.ietf-bmwg-virtual-net"/>)related to Benchmarks for virtual - switches. The OPNFV project is sharing its present view on these areas, - as they develop their specifications in the Level Test Design (LTD) - document.</t> - - <section title="Comparison with Physical Network Functions"> - <t>To compare the performance of virtual designs and implementations - with their physical counterparts, identical benchmarks are needed. - BMWG has developed specifications for many network functions this memo - re-uses existing benchmarks through references, and expands them - during development of new methods. A key configuration aspect is the - number of parallel cores required to achieve comparable performance - with a given physical device, or whether some limit of scale was - reached before the cores could achieve the comparable level.</t> - - <t>It's unlikely that the virtual switch will be the only application - running on the SUT, so CPU utilization, Cache utilization, and Memory - footprint should also be recorded for the virtual implementations of - internetworking functions.</t> - </section> - - <section title="Continued Emphasis on Black-Box Benchmarks"> - <t>External observations remain essential as the basis for Benchmarks. - Internal observations with fixed specification and interpretation will - be provided in parallel to assist the development of operations - procedures when the technology is deployed.</t> - </section> - - <section title="New Configuration Parameters"> - <t>A key consideration when conducting any sort of benchmark is trying - to ensure the consistency and repeatability of test results. When - benchmarking the performance of a vSwitch there are many factors that - can affect the consistency of results, one key factor is matching the - various hardware and software details of the SUT. This section lists - some of the many new parameters which this project believes are - critical to report in order to achieve repeatability.</t> - - <t>Hardware details including:</t> - - <t><list style="symbols"> - <t>Platform details</t> - - <t>Processor details</t> - - <t>Memory information (type and size)</t> - - <t>Number of enabled cores</t> - - <t>Number of cores used for the test</t> - - <t>Number of physical NICs, as well as their details - (manufacturer, versions, type and the PCI slot they are plugged - into)</t> - - <t>NIC interrupt configuration</t> - - <t>BIOS version, release date and any configurations that were - modified</t> - - <t>CPU microcode level</t> - - <t>Memory DIMM configurations (quad rank performance may not be - the same as dual rank) in size, freq and slot locations</t> - - <t>PCI configuration parameters (payload size, early ack - option...)</t> - - <t>Power management at all levels (ACPI sleep states, processor - package, OS...)</t> - </list>Software details including:</t> - - <t><list style="symbols"> - <t>OS parameters and behavior (text vs graphical no one typing at - the console on one system)</t> - - <t>OS version (for host and VNF)</t> - - <t>Kernel version (for host and VNF)</t> - - <t>GRUB boot parameters (for host and VNF)</t> - - <t>Hypervisor details (Type and version)</t> - - <t>Selected vSwitch, version number or commit id used</t> - - <t>vSwitch launch command line if it has been parameterised</t> - - <t>Memory allocation to the vSwitch</t> - - <t>which NUMA node it is using, and how many memory channels</t> - - <t>DPDK or any other SW dependency version number or commit id - used</t> - - <t>Memory allocation to a VM - if it's from Hugpages/elsewhere</t> - - <t>VM storage type: snapshot/independent persistent/independent - non-persistent</t> - - <t>Number of VMs</t> - - <t>Number of Virtual NICs (vNICs), versions, type and driver</t> - - <t>Number of virtual CPUs and their core affinity on the host</t> - - <t>Number vNIC interrupt configuration</t> - - <t>Thread affinitization for the applications (including the - vSwitch itself) on the host</t> - - <t>Details of Resource isolation, such as CPUs designated for - Host/Kernel (isolcpu) and CPUs designated for specific processes - (taskset). - Test duration. - Number of flows.</t> - </list></t> - - <t>Test Traffic Information:<list style="symbols"> - <t>Traffic type - UDP, TCP, IMIX / Other</t> - - <t>Packet Sizes</t> - - <t>Deployment Scenario</t> - </list></t> - - <t/> - </section> - - <section title="Flow classification"> - <t>Virtual switches group packets into flows by processing and - matching particular packet or frame header information, or by matching - packets based on the input ports. Thus a flow can be thought of a - sequence of packets that have the same set of header field values or - have arrived on the same port. Performance results can vary based on - the parameters the vSwitch uses to match for a flow. The recommended - flow classification parameters for any vSwitch performance tests are: - the input port, the source IP address, the destination IP address and - the Ethernet protocol type field. It is essential to increase the flow - timeout time on a vSwitch before conducting any performance tests that - do not measure the flow setup time. Normally the first packet of a - particular stream will install the flow in the virtual switch which - adds an additional latency, subsequent packets of the same flow are - not subject to this latency if the flow is already installed on the - vSwitch.</t> - </section> - - <section title="Benchmarks using Baselines with Resource Isolation"> - <t>This outline describes measurement of baseline with isolated - resources at a high level, which is the intended approach at this - time.</t> - - <t><list style="numbers"> - <t>Baselines: <list style="symbols"> - <t>Optional: Benchmark platform forwarding capability without - a vswitch or VNF for at least 72 hours (serves as a means of - platform validation and a means to obtain the base performance - for the platform in terms of its maximum forwarding rate and - latency). <figure> - <preamble>Benchmark platform forwarding - capability</preamble> - - <artwork align="right"><![CDATA[ __ - +--------------------------------------------------+ | - | +------------------------------------------+ | | - | | | | | - | | Simple Forwarding App | | Host - | | | | | - | +------------------------------------------+ | | - | | NIC | | | - +---+------------------------------------------+---+ __| - ^ : - | | - : v - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - - <postamble/> - </figure></t> - - <t>Benchmark VNF forwarding capability with direct - connectivity (vSwitch bypass, e.g., SR/IOV) for at least 72 - hours (serves as a means of VNF validation and a means to - obtain the base performance for the VNF in terms of its - maximum forwarding rate and latency). The metrics gathered - from this test will serve as a key comparison point for - vSwitch bypass technologies performance and vSwitch - performance. <figure align="right"> - <preamble>Benchmark VNF forwarding capability</preamble> - - <artwork><![CDATA[ __ - +--------------------------------------------------+ | - | +------------------------------------------+ | | - | | | | | - | | VNF | | | - | | | | | - | +------------------------------------------+ | | - | | Passthrough/SR-IOV | | Host - | +------------------------------------------+ | | - | | NIC | | | - +---+------------------------------------------+---+ __| - ^ : - | | - : v - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - - <postamble/> - </figure></t> - - <t>Benchmarking with isolated resources alone, with other - resources (both HW&SW) disabled Example, vSw and VM are - SUT</t> - - <t>Benchmarking with isolated resources alone, leaving some - resources unused</t> - - <t>Benchmark with isolated resources and all resources - occupied</t> - </list></t> - - <t>Next Steps<list style="symbols"> - <t>Limited sharing</t> - - <t>Production scenarios</t> - - <t>Stressful scenarios</t> - </list></t> - </list></t> - </section> - </section> - - <section title="VSWITCHPERF Specification Summary"> - <t>The overall specification in preparation is referred to as a Level - Test Design (LTD) document, which will contain a suite of performance - tests. The base performance tests in the LTD are based on the - pre-existing specifications developed by BMWG to test the performance of - physical switches. These specifications include:</t> - - <t><list style="symbols"> - <t><xref target="RFC2544"/> Benchmarking Methodology for Network - Interconnect Devices</t> - - <t><xref target="RFC2889"/> Benchmarking Methodology for LAN - Switching</t> - - <t><xref target="RFC6201"/> Device Reset Characterization</t> - - <t><xref target="RFC5481"/> Packet Delay Variation Applicability - Statement</t> - </list></t> - - <t>Some of the above/newer RFCs are being applied in benchmarking for - the first time, and represent a development challenge for test equipment - developers. Fortunately, many members of the testing system community - have engaged on the VSPERF project, including an open source test - system.</t> - - <t>In addition to this, the LTD also re-uses the terminology defined - by:</t> - - <t><list style="symbols"> - <t><xref target="RFC2285"/> Benchmarking Terminology for LAN - Switching Devices</t> - - <t><xref target="RFC5481"/> Packet Delay Variation Applicability - Statement</t> - </list></t> - - <t/> - - <t>Specifications to be included in future updates of the LTD - include:<list style="symbols"> - <t><xref target="RFC3918"/> Methodology for IP Multicast - Benchmarking</t> - - <t><xref target="RFC4737"/> Packet Reordering Metrics</t> - </list></t> - - <t>As one might expect, the most fundamental internetworking - characteristics of Throughput and Latency remain important when the - switch is virtualized, and these benchmarks figure prominently in the - specification.</t> - - <t>When considering characteristics important to "telco" network - functions, we must begin to consider additional performance metrics. In - this case, the project specifications have referenced metrics from the - IETF IP Performance Metrics (IPPM) literature. This means that the <xref - target="RFC2544"/> test of Latency is replaced by measurement of a - metric derived from IPPM's <xref target="RFC2679"/>, where a set of - statistical summaries will be provided (mean, max, min, etc.). Further - metrics planned to be benchmarked include packet delay variation as - defined by <xref target="RFC5481"/> , reordering, burst behaviour, DUT - availability, DUT capacity and packet loss in long term testing at - Throughput level, where some low-level of background loss may be present - and characterized.</t> - - <t>Tests have been (or will be) designed to collect the metrics - below:</t> - - <t><list style="symbols"> - <t>Throughput Tests to measure the maximum forwarding rate (in - frames per second or fps) and bit rate (in Mbps) for a constant load - (as defined by <xref target="RFC1242"/>) without traffic loss.</t> - - <t>Packet and Frame Delay Distribution Tests to measure average, min - and max packet and frame delay for constant loads.</t> - - <t>Packet Delay Tests to understand latency distribution for - different packet sizes and over an extended test run to uncover - outliers.</t> - - <t>Scalability Tests to understand how the virtual switch performs - as the number of flows, active ports, complexity of the forwarding - logic’s configuration… it has to deal with - increases.</t> - - <t>Stream Performance Tests (TCP, UDP) to measure bulk data transfer - performance, i.e. how fast systems can send and receive data through - the switch.</t> - - <t>Control Path and Datapath Coupling Tests, to understand how - closely coupled the datapath and the control path are as well as the - effect of this coupling on the performance of the DUT (example: - delay of the initial packet of a flow).</t> - - <t>CPU and Memory Consumption Tests to understand the virtual - switch’s footprint on the system, usually conducted as - auxiliary measurements with benchmarks above. They include: CPU - utilization, Cache utilization and Memory footprint.</t> - - <t>The so-called "Soak" tests, where the selected test is conducted - over a long period of time (with an ideal duration of 24 hours, and - at least 6 hours). The purpose of soak tests is to capture transient - changes in performance which may occur due to infrequent processes - or the low probability coincidence of two or more processes. The - performance must be evaluated periodically during continuous - testing, and this results in use of <xref target="RFC2889"/> Frame - Rate metrics instead of <xref target="RFC2544"/> Throughput (which - requires stopping traffic to allow time for all traffic to exit - internal queues).</t> - </list></t> - - <t>Future/planned test specs include:<list style="symbols"> - <t>Request/Response Performance Tests (TCP, UDP) which measure the - transaction rate through the switch.</t> - - <t>Noisy Neighbour Tests, to understand the effects of resource - sharing on the performance of a virtual switch.</t> - - <t>Tests derived from examination of ETSI NFV Draft GS IFA003 - requirements <xref target="IFA003"/> on characterization of - acceleration technologies applied to vswitches.</t> - </list>The flexibility of deployment of a virtual switch within a - network means that the BMWG IETF existing literature needs to be used to - characterize the performance of a switch in various deployment - scenarios. The deployment scenarios under consideration include:</t> - - <t><figure> - <preamble>Physical port to virtual switch to physical - port</preamble> - - <artwork><![CDATA[ __ - +--------------------------------------------------+ | - | +--------------------+ | | - | | | | | - | | v | | Host - | +--------------+ +--------------+ | | - | | phy port | vSwitch | phy port | | | - +---+--------------+------------+--------------+---+ __| - ^ : - | | - : v - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - </figure></t> - - <t><figure> - <preamble>Physical port to virtual switch to VNF to virtual switch - to physical port</preamble> - - <artwork><![CDATA[ __ - +---------------------------------------------------+ | - | | | - | +-------------------------------------------+ | | - | | Application | | | - | +-------------------------------------------+ | | - | ^ : | | - | | | | | Guest - | : v | | - | +---------------+ +---------------+ | | - | | logical port 0| | logical port 1| | | - +---+---------------+-----------+---------------+---+ __| - ^ : - | | - : v __ - +---+---------------+----------+---------------+---+ | - | | logical port 0| | logical port 1| | | - | +---------------+ +---------------+ | | - | ^ : | | - | | | | | Host - | : v | | - | +--------------+ +--------------+ | | - | | phy port | vSwitch | phy port | | | - +---+--------------+------------+--------------+---+ __| - ^ : - | | - : v - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - </figure><figure> - <preamble>Physical port to virtual switch to VNF to virtual switch - to VNF to virtual switch to physical port</preamble> - - <artwork><![CDATA[ __ - +----------------------+ +----------------------+ | - | Guest 1 | | Guest 2 | | - | +---------------+ | | +---------------+ | | - | | Application | | | | Application | | | - | +---------------+ | | +---------------+ | | - | ^ | | | ^ | | | - | | v | | | v | | Guests - | +---------------+ | | +---------------+ | | - | | logical ports | | | | logical ports | | | - | | 0 1 | | | | 0 1 | | | - +---+---------------+--+ +---+---------------+--+__| - ^ : ^ : - | | | | - : v : v _ - +---+---------------+---------+---------------+--+ | - | | 0 1 | | 3 4 | | | - | | logical ports | | logical ports | | | - | +---------------+ +---------------+ | | - | ^ | ^ | | | Host - | | |-----------------| v | | - | +--------------+ +--------------+ | | - | | phy ports | vSwitch | phy ports | | | - +---+--------------+----------+--------------+---+_| - ^ : - | | - : v - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - </figure><figure> - <preamble>Physical port to virtual switch to VNF</preamble> - - <artwork><![CDATA[ __ - +---------------------------------------------------+ | - | | | - | +-------------------------------------------+ | | - | | Application | | | - | +-------------------------------------------+ | | - | ^ | | - | | | | Guest - | : | | - | +---------------+ | | - | | logical port 0| | | - +---+---------------+-------------------------------+ __| - ^ - | - : __ - +---+---------------+------------------------------+ | - | | logical port 0| | | - | +---------------+ | | - | ^ | | - | | | | Host - | : | | - | +--------------+ | | - | | phy port | vSwitch | | - +---+--------------+------------ -------------- ---+ __| - ^ - | - : - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - </figure><figure> - <preamble>VNF to virtual switch to physical port</preamble> - - <artwork><![CDATA[ __ - +---------------------------------------------------+ | - | | | - | +-------------------------------------------+ | | - | | Application | | | - | +-------------------------------------------+ | | - | : | | - | | | | Guest - | v | | - | +---------------+ | | - | | logical port | | | - +-------------------------------+---------------+---+ __| - : - | - v __ - +------------------------------+---------------+---+ | - | | logical port | | | - | +---------------+ | | - | : | | - | | | | Host - | v | | - | +--------------+ | | - | vSwitch | phy port | | | - +-------------------------------+--------------+---+ __| - : - | - v - +--------------------------------------------------+ - | | - | traffic generator | - | | - +--------------------------------------------------+]]></artwork> - </figure><figure> - <preamble>VNF to virtual switch to VNF</preamble> - - <artwork><![CDATA[ __ - +----------------------+ +----------------------+ | - | Guest 1 | | Guest 2 | | - | +---------------+ | | +---------------+ | | - | | Application | | | | Application | | | - | +---------------+ | | +---------------+ | | - | | | | ^ | | - | v | | | | | Guests - | +---------------+ | | +---------------+ | | - | | logical ports | | | | logical ports | | | - | | 0 | | | | 0 | | | - +---+---------------+--+ +---+---------------+--+__| - : ^ - | | - v : _ - +---+---------------+---------+---------------+--+ | - | | 1 | | 1 | | | - | | logical ports | | logical ports | | | - | +---------------+ +---------------+ | | - | | ^ | | Host - | L-----------------+ | | - | | | - | vSwitch | | - +------------------------------------------------+_|]]></artwork> - </figure></t> - - <t>A set of Deployment Scenario figures is available on the VSPERF Test - Methodology Wiki page <xref target="TestTopo"/>.</t> - </section> - - <section title="3x3 Matrix Coverage"> - <t>This section organizes the many existing test specifications into the - "3x3" matrix (introduced in <xref target="I-D.ietf-bmwg-virtual-net"/>). - Because the LTD specification ID names are quite long, this section is - organized into lists for each occupied cell of the matrix (not all are - occupied, also the matrix has grown to 3x4 to accommodate scale metrics - when displaying the coverage of many metrics/benchmarks). The current - version of the LTD specification is available <xref target="LTD"/>.</t> - - <t>The tests listed below assess the activation of paths in the data - plane, rather than the control plane.</t> - - <t>A complete list of tests with short summaries is available on the - VSPERF "LTD Test Spec Overview" Wiki page <xref target="LTDoverV"/>.</t> - - <section title="Speed of Activation"> - <t><list style="symbols"> - <t>Activation.RFC2889.AddressLearningRate</t> - - <t>PacketLatency.InitialPacketProcessingLatency</t> - </list></t> - </section> - - <section title="Accuracy of Activation section"> - <t><list style="symbols"> - <t>CPDP.Coupling.Flow.Addition</t> - </list></t> - </section> - - <section title="Reliability of Activation"> - <t><list style="symbols"> - <t>Throughput.RFC2544.SystemRecoveryTime</t> - - <t>Throughput.RFC2544.ResetTime</t> - </list></t> - </section> - - <section title="Scale of Activation"> - <t><list style="symbols"> - <t>Activation.RFC2889.AddressCachingCapacity</t> - </list></t> - </section> - - <section title="Speed of Operation"> - <t><list style="symbols"> - <t>Throughput.RFC2544.PacketLossRate</t> - - <t>CPU.RFC2544.0PacketLoss</t> - - <t>Throughput.RFC2544.PacketLossRateFrameModification</t> - - <t>Throughput.RFC2544.BackToBackFrames</t> - - <t>Throughput.RFC2889.MaxForwardingRate</t> - - <t>Throughput.RFC2889.ForwardPressure</t> - - <t>Throughput.RFC2889.BroadcastFrameForwarding</t> - </list></t> - </section> - - <section title="Accuracy of Operation"> - <t><list style="symbols"> - <t>Throughput.RFC2889.ErrorFramesFiltering</t> - - <t>Throughput.RFC2544.Profile</t> - </list></t> - </section> - - <section title="Reliability of Operation"> - <t><list style="symbols"> - <t>Throughput.RFC2889.Soak</t> - - <t>Throughput.RFC2889.SoakFrameModification</t> - - <t>PacketDelayVariation.RFC3393.Soak</t> - </list></t> - </section> - - <section title="Scalability of Operation"> - <t><list style="symbols"> - <t>Scalability.RFC2544.0PacketLoss</t> - - <t>MemoryBandwidth.RFC2544.0PacketLoss.Scalability</t> - </list></t> - </section> - - <section title="Summary"> - <t><figure> - <artwork><![CDATA[|------------------------------------------------------------------------| -| | | | | | -| | SPEED | ACCURACY | RELIABILITY | SCALE | -| | | | | | -|------------------------------------------------------------------------| -| | | | | | -| Activation | X | X | X | X | -| | | | | | -|------------------------------------------------------------------------| -| | | | | | -| Operation | X | X | X | X | -| | | | | | -|------------------------------------------------------------------------| -| | | | | | -| De-activation | | | | | -| | | | | | -|------------------------------------------------------------------------|]]></artwork> - </figure></t> - </section> - </section> - - <section title="Security Considerations"> - <t>Benchmarking activities as described in this memo are limited to - technology characterization of a Device Under Test/System Under Test - (DUT/SUT) using controlled stimuli in a laboratory environment, with - dedicated address space and the constraints specified in the sections - above.</t> - - <t>The benchmarking network topology will be an independent test setup - and MUST NOT be connected to devices that may forward the test traffic - into a production network, or misroute traffic to the test management - network.</t> - - <t>Further, benchmarking is performed on a "black-box" basis, relying - solely on measurements observable external to the DUT/SUT.</t> - - <t>Special capabilities SHOULD NOT exist in the DUT/SUT specifically for - benchmarking purposes. Any implications for network security arising - from the DUT/SUT SHOULD be identical in the lab and in production - networks.</t> - </section> - - <section anchor="IANA" title="IANA Considerations"> - <t>No IANA Action is requested at this time.</t> - </section> - - <section title="Acknowledgements"> - <t>The authors appreciate and acknowledge comments from Scott Bradner, - Marius Georgescu, Ramki Krishnan, Doug Montgomery, Martin Klozik, - Christian Trautman, and others for their reviews.</t> - </section> - </middle> - - <back> - <references title="Normative References"> - <?rfc ?> - - <?rfc include="reference.RFC.2119"?> - - <?rfc ?> - - <?rfc include="reference.RFC.2330"?> - - <?rfc include='reference.RFC.2544'?> - - <?rfc include="reference.RFC.2679"?> - - <?rfc include='reference.RFC.2680'?> - - <?rfc include='reference.RFC.3393'?> - - <?rfc include='reference.RFC.3432'?> - - <?rfc include='reference.RFC.2681'?> - - <?rfc include='reference.RFC.5905'?> - - <?rfc include='reference.RFC.4689'?> - - <?rfc include='reference.RFC.4737'?> - - <?rfc include='reference.RFC.5357'?> - - <?rfc include='reference.RFC.2889'?> - - <?rfc include='reference.RFC.3918'?> - - <?rfc include='reference.RFC.6201'?> - - <?rfc include='reference.RFC.2285'?> - - <reference anchor="NFV.PER001"> - <front> - <title>Network Function Virtualization: Performance and Portability - Best Practices</title> - - <author fullname="ETSI NFV" initials="" surname=""> - <organization/> - </author> - - <date month="June" year="2014"/> - </front> - - <seriesInfo name="Group Specification" - value="ETSI GS NFV-PER 001 V1.1.1 (2014-06)"/> - - <format type="PDF"/> - </reference> - </references> - - <references title="Informative References"> - <?rfc include='reference.RFC.1242'?> - - <?rfc include='reference.RFC.5481'?> - - <?rfc include='reference.RFC.6049'?> - - <?rfc include='reference.RFC.6248'?> - - <?rfc include='reference.RFC.6390'?> - - <?rfc include='reference.I-D.ietf-bmwg-virtual-net'?> - - <?rfc include='reference.I-D.huang-bmwg-virtual-network-performance'?> - - <reference anchor="TestTopo"> - <front> - <title>Test Topologies - https://wiki.opnfv.org/vsperf/test_methodology</title> - - <author> - <organization/> - </author> - - <date/> - </front> - </reference> - - <reference anchor="LTDoverV"> - <front> - <title>LTD Test Spec Overview - https://wiki.opnfv.org/wiki/vswitchperf_test_spec_review</title> - - <author> - <organization/> - </author> - - <date/> - </front> - </reference> - - <reference anchor="LTD"> - <front> - <title>LTD Test Specification - http://artifacts.opnfv.org/vswitchperf/docs/requirements/index.html</title> - - <author> - <organization/> - </author> - - <date/> - </front> - </reference> - - <reference anchor="BrahRel"> - <front> - <title>Brahmaputra, Second OPNFV Release - https://www.opnfv.org/brahmaputra</title> - - <author> - <organization/> - </author> - - <date/> - </front> - </reference> - - <reference anchor="IFA003"> - <front> - <title>https://docbox.etsi.org/ISG/NFV/Open/Drafts/IFA003_Acceleration_-_vSwitch_Spec/</title> - - <author> - <organization/> - </author> - - <date/> - </front> - </reference> - </references> - </back> -</rfc> diff --git a/docs/testing/developer/devguide/requirements/ietf_draft/rfc8204-vsperf-bmwg-vswitch-opnfv.rst b/docs/testing/developer/devguide/requirements/ietf_draft/rfc8204-vsperf-bmwg-vswitch-opnfv.rst new file mode 100644 index 00000000..ee7f98b5 --- /dev/null +++ b/docs/testing/developer/devguide/requirements/ietf_draft/rfc8204-vsperf-bmwg-vswitch-opnfv.rst @@ -0,0 +1,19 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) OPNFV, Intel Corporation, AT&T, Red Hat, Spirent, Ixia and others. + +*************************************** +IETF benchmarking specification RFC8204 +*************************************** + +The directory /ietf_draft was used to store draft versions of the VSPERF test specification proposed +as an Internet Draft and subsequently approved for publication as RFC8204. The draft versions have +been removed. "Benchmarking Virtual Switches in the Open Platform for NFV (OPNFV)" is an +informational RFC published by the IETF available here https://tools.ietf.org/html/rfc8204. + +For more information about VSPERF refer to: + +* Wiki: https://wiki.opnfv.org/characterize_vswitch_performance_for_telco_nfv_use_cases +* Repository: https://git.opnfv.org/vswitchperf +* Artifacts: https://artifacts.opnfv.org/vswitchperf.html +* Continuous Integration: https://build.opnfv.org/ci/view/vswitchperf/ -- cgit