From 5478e77dab55d30ebd3e49b8b8f6002a03b4a065 Mon Sep 17 00:00:00 2001 From: Jing Zhang Date: Tue, 11 Apr 2017 10:52:30 -0400 Subject: Add multiqueue support Problem: Neither OVS nor SRIOV multi-queue is not supported. Guest VM lacks tuning to reach high throughput. Solution: (1) Build SRIOV multi-queue capable guest image by recompiling the igxbevf driver (make CFLAGS_EXTRA=-DIXGBE_ENABLE_VF_MQ install). (2) Change pktgen to send on multiple queues. (3) Add tuning in guest VM (Disable irqbalance and setup vNIC interrupt affinity to vCPUs) Update 1: Takes care comments plus adds unit test cases Update 2: Jenkins reports code coverage 81%, local reports 96%, add more unit test cases Update 3: Manually rebased to adapt to SSH.from_node() Update 4: Takes care comments for vnic_type, vnic_name Change-Id: Ieb15381c653b13697487d095efa4be6c3c49fa42 JIRA: YARDSTICK-619 Signed-off-by: Jing Zhang --- yardstick/benchmark/scenarios/networking/pktgen.py | 242 ++++++++++++++++++++- .../scenarios/networking/pktgen_benchmark.bash | 139 ++++++++---- 2 files changed, 334 insertions(+), 47 deletions(-) (limited to 'yardstick') diff --git a/yardstick/benchmark/scenarios/networking/pktgen.py b/yardstick/benchmark/scenarios/networking/pktgen.py index e6aa7e5fb..8ca1ca60e 100644 --- a/yardstick/benchmark/scenarios/networking/pktgen.py +++ b/yardstick/benchmark/scenarios/networking/pktgen.py @@ -9,6 +9,7 @@ from __future__ import absolute_import from __future__ import print_function +import os import logging import pkg_resources @@ -19,6 +20,9 @@ from yardstick.benchmark.scenarios import base LOG = logging.getLogger(__name__) +VNIC_TYPE_LIST = ["ovs", "sriov"] +SRIOV_DRIVER_LIST = ["ixgbevf", "i40evf"] + class Pktgen(base.Scenario): """Execute pktgen between two hosts @@ -44,7 +48,11 @@ class Pktgen(base.Scenario): def __init__(self, scenario_cfg, context_cfg): self.scenario_cfg = scenario_cfg self.context_cfg = context_cfg + self.vnic_name = "eth0" + self.vnic_type = "ovs" + self.queue_number = 1 self.setup_done = False + self.multiqueue_setup_done = False def setup(self): """scenario setup""" @@ -67,6 +75,212 @@ class Pktgen(base.Scenario): self.setup_done = True + def multiqueue_setup(self): + # one time setup stuff + cmd = "sudo sysctl -w net.core.netdev_budget=3000" + self.server.send_command(cmd) + self.client.send_command(cmd) + + cmd = "sudo sysctl -w net.core.netdev_max_backlog=100000" + self.server.send_command(cmd) + self.client.send_command(cmd) + + """multiqueue setup""" + if not self._is_irqbalance_disabled(): + self._disable_irqbalance() + + vnic_driver_name = self._get_vnic_driver_name() + if vnic_driver_name in SRIOV_DRIVER_LIST: + self.vnic_type = "sriov" + + # one time setup stuff + cmd = "sudo ethtool -G %s rx 4096 tx 4096" % self.vnic_name + self.server.send_command(cmd) + self.client.send_command(cmd) + + self.queue_number = self._get_sriov_queue_number() + self._setup_irqmapping_sriov(self.queue_number) + else: + self.vnic_type = "ovs" + self.queue_number = self._enable_ovs_multiqueue() + self._setup_irqmapping_ovs(self.queue_number) + + self.multiqueue_setup_done = True + + def _get_vnic_driver_name(self): + cmd = "readlink /sys/class/net/%s/device/driver" % self.vnic_name + LOG.debug("Executing command: %s", cmd) + status, stdout, stderr = self.server.execute(cmd) + if status: + raise RuntimeError(stderr) + return os.path.basename(stdout.strip()) + + def _is_irqbalance_disabled(self): + """Did we disable irqbalance already in the guest?""" + is_disabled = False + cmd = "grep ENABLED /etc/default/irqbalance" + status, stdout, stderr = self.server.execute(cmd) + if status: + raise RuntimeError(stderr) + if "0" in stdout: + is_disabled = True + + return is_disabled + + def _disable_irqbalance(self): + cmd = "sudo sed -i -e 's/ENABLED=\"1\"/ENABLED=\"0\"/g' " \ + "/etc/default/irqbalance" + status, stdout, stderr = self.server.execute(cmd) + status, stdout, stderr = self.client.execute(cmd) + if status: + raise RuntimeError(stderr) + + cmd = "sudo service irqbalance stop" + status, stdout, stderr = self.server.execute(cmd) + status, stdout, stderr = self.client.execute(cmd) + if status: + raise RuntimeError(stderr) + + cmd = "sudo service irqbalance disable" + status, stdout, stderr = self.server.execute(cmd) + status, stdout, stderr = self.client.execute(cmd) + if status: + raise RuntimeError(stderr) + + def _setup_irqmapping_ovs(self, queue_number): + cmd = "grep 'virtio0-input.0' /proc/interrupts |" \ + "awk '{match($0,/ +[0-9]+/)} " \ + "{print substr($1,RSTART,RLENGTH-1)}'" + status, stdout, stderr = self.server.execute(cmd) + if status: + raise RuntimeError(stderr) + + cmd = "echo 1 | sudo tee /proc/irq/%s/smp_affinity" % (int(stdout)) + status, stdout, stderr = self.server.execute(cmd) + status, stdout, stderr = self.client.execute(cmd) + if status: + raise RuntimeError(stderr) + + cmd = "grep 'virtio0-output.0' /proc/interrupts |" \ + "awk '{match($0,/ +[0-9]+/)} " \ + "{print substr($1,RSTART,RLENGTH-1)}'" + status, stdout, stderr = self.server.execute(cmd) + if status: + raise RuntimeError(stderr) + + cmd = "echo 1 | sudo tee /proc/irq/%s/smp_affinity" % (int(stdout)) + status, stdout, stderr = self.server.execute(cmd) + status, stdout, stderr = self.client.execute(cmd) + if status: + raise RuntimeError(stderr) + + if queue_number == 1: + return + + for i in range(1, queue_number): + cmd = "grep 'virtio0-input.%s' /proc/interrupts |" \ + "awk '{match($0,/ +[0-9]+/)} " \ + "{print substr($1,RSTART,RLENGTH-1)}'" % (i) + status, stdout, stderr = self.server.execute(cmd) + if status: + raise RuntimeError(stderr) + + cmd = "echo %s | sudo tee /proc/irq/%s/smp_affinity" \ + % (1 << i, int(stdout)) + status, stdout, stderr = self.server.execute(cmd) + status, stdout, stderr = self.client.execute(cmd) + if status: + raise RuntimeError(stderr) + + cmd = "grep 'virtio0-output.%s' /proc/interrupts |" \ + "awk '{match($0,/ +[0-9]+/)} " \ + "{print substr($1,RSTART,RLENGTH-1)}'" % (i) + status, stdout, stderr = self.server.execute(cmd) + if status: + raise RuntimeError(stderr) + + cmd = "echo %s | sudo tee /proc/irq/%s/smp_affinity" \ + % (1 << i, int(stdout)) + status, stdout, stderr = self.server.execute(cmd) + status, stdout, stderr = self.client.execute(cmd) + if status: + raise RuntimeError(stderr) + + def _setup_irqmapping_sriov(self, queue_number): + cmd = "grep '%s-TxRx-0' /proc/interrupts |" \ + "awk '{match($0,/ +[0-9]+/)} " \ + "{print substr($1,RSTART,RLENGTH-1)}'" % self.vnic_name + status, stdout, stderr = self.server.execute(cmd) + if status: + raise RuntimeError(stderr) + + cmd = "echo 1 | sudo tee /proc/irq/%s/smp_affinity" % (int(stdout)) + status, stdout, stderr = self.server.execute(cmd) + status, stdout, stderr = self.client.execute(cmd) + if status: + raise RuntimeError(stderr) + + if queue_number == 1: + return + + for i in range(1, queue_number): + cmd = "grep '%s-TxRx-%s' /proc/interrupts |" \ + "awk '{match($0,/ +[0-9]+/)} " \ + "{print substr($1,RSTART,RLENGTH-1)}'" % (self.vnic_name, i) + status, stdout, stderr = self.server.execute(cmd) + if status: + raise RuntimeError(stderr) + + cmd = "echo %s | sudo tee /proc/irq/%s/smp_affinity" \ + % (1 << i, int(stdout)) + status, stdout, stderr = self.server.execute(cmd) + status, stdout, stderr = self.client.execute(cmd) + if status: + raise RuntimeError(stderr) + + def _get_sriov_queue_number(self): + """Get queue number from server as both VMs are the same""" + cmd = "grep %s-TxRx- /proc/interrupts | wc -l" % self.vnic_name + LOG.debug("Executing command: %s", cmd) + status, stdout, stderr = self.server.execute(cmd) + if status: + raise RuntimeError(stderr) + return int(stdout) + + def _get_available_queue_number(self): + """Get queue number from client as both VMs are the same""" + cmd = "sudo ethtool -l %s | grep Combined | head -1 |" \ + "awk '{printf $2}'" % self.vnic_name + LOG.debug("Executing command: %s", cmd) + status, stdout, stderr = self.server.execute(cmd) + if status: + raise RuntimeError(stderr) + return int(stdout) + + def _get_usable_queue_number(self): + """Get queue number from client as both VMs are the same""" + cmd = "sudo ethtool -l %s | grep Combined | tail -1 |" \ + "awk '{printf $2}'" % self.vnic_name + LOG.debug("Executing command: %s", cmd) + status, stdout, stderr = self.server.execute(cmd) + if status: + raise RuntimeError(stderr) + return int(stdout) + + def _enable_ovs_multiqueue(self): + available_queue_number = self._get_available_queue_number() + usable_queue_number = self._get_usable_queue_number() + if available_queue_number > 1 and \ + available_queue_number != usable_queue_number: + cmd = "sudo ethtool -L %s combined %s" % \ + (self.vnic_name, available_queue_number) + LOG.debug("Executing command: %s", cmd) + status, stdout, stderr = self.server.execute(cmd) + status, stdout, stderr = self.client.execute(cmd) + if status: + raise RuntimeError(stderr) + return available_queue_number + def _iptables_setup(self): """Setup iptables on server to monitor for received packets""" cmd = "sudo iptables -F; " \ @@ -99,6 +313,14 @@ class Pktgen(base.Scenario): options = self.scenario_cfg['options'] packetsize = options.get("packetsize", 60) self.number_of_ports = options.get("number_of_ports", 10) + self.vnic_name = options.get("vnic_name", "eth0") + ovs_dpdk = options.get("ovs_dpdk", False) + pps = options.get("pps", 1000000) + multiqueue = options.get("multiqueue", False) + + if multiqueue and not self.multiqueue_setup_done: + self.multiqueue_setup() + # if run by a duration runner duration_time = self.scenario_cfg["runner"].get("duration", None) \ if "runner" in self.scenario_cfg else None @@ -114,8 +336,18 @@ class Pktgen(base.Scenario): self._iptables_setup() - cmd = "sudo bash pktgen.sh %s %s %s %s" \ - % (ipaddr, self.number_of_ports, packetsize, duration) + queue_number = self.queue_number + + # For native OVS, half of vCPUs are used by vhost kernel threads + # hence set the queue_number to half number of vCPUs + # e.g. set queue_number to 2 if there are 4 vCPUs + if self.vnic_type == "ovs" and not ovs_dpdk and self.queue_number > 1: + queue_number = self.queue_number / 2 + + cmd = "sudo bash pktgen.sh %s %s %s %s %s %s" \ + % (ipaddr, self.number_of_ports, packetsize, + duration, queue_number, pps) + LOG.debug("Executing command: %s", cmd) status, stdout, stderr = self.client.execute(cmd) @@ -131,12 +363,15 @@ class Pktgen(base.Scenario): sent = result['packets_sent'] received = result['packets_received'] ppm = 1000000 * (sent - received) / sent + # if ppm is 1, then 11 out of 10 million is no pass + ppm += (sent - received) % sent > 0 + LOG.debug("Lost packets %d - Lost ppm %d", (sent - received), ppm) sla_max_ppm = int(self.scenario_cfg["sla"]["max_ppm"]) assert ppm <= sla_max_ppm, "ppm %d > sla_max_ppm %d; " \ % (ppm, sla_max_ppm) -def _test(): +def _test(): # pragma: no cover """internal test function""" key_filename = pkg_resources.resource_filename('yardstick.resources', 'files/yardstick_key') @@ -165,6 +400,5 @@ def _test(): p.run(result) print(result) - if __name__ == '__main__': _test() diff --git a/yardstick/benchmark/scenarios/networking/pktgen_benchmark.bash b/yardstick/benchmark/scenarios/networking/pktgen_benchmark.bash index 4224c5abf..e338a1b09 100644 --- a/yardstick/benchmark/scenarios/networking/pktgen_benchmark.bash +++ b/yardstick/benchmark/scenarios/networking/pktgen_benchmark.bash @@ -16,6 +16,8 @@ DST_IP=$1 # destination IP address NUM_PORTS=$2 # number of source ports PKT_SIZE=$3 # packet size DURATION=$4 # test duration (seconds) +TRXQUEUE=$5 # number of RX/TX queues to use +PPS=$6 # packets per second to send # Configuration UDP_SRC_MIN=1000 # UDP source port min @@ -37,62 +39,100 @@ pgset() fi } +# remove all devices from thread +pgclean() +{ + COUNTER=0 + while [ ${COUNTER} -lt ${TRXQUEUE} ]; do + # + # Thread commands + # + + PGDEV=/proc/net/pktgen/kpktgend_${COUNTER} + + # Remove all devices from this thread + pgset "rem_device_all" + let COUNTER=COUNTER+1 + done +} + # configure pktgen (see pktgen doc for details) pgconfig() { - # - # Thread commands - # + pps=$(( PPS / TRXQUEUE )) + COUNTER=0 + while [ ${COUNTER} -lt ${TRXQUEUE} ]; do + # + # Thread commands + # - PGDEV=/proc/net/pktgen/kpktgend_0 + PGDEV=/proc/net/pktgen/kpktgend_${COUNTER} - # Remove all devices from this thread - pgset "rem_device_all" + # Add device to thread + pgset "add_device $DEV@${COUNTER}" - # Add device to thread - pgset "add_device $DEV" + # + # Device commands + # - # - # Device commands - # + PGDEV=/proc/net/pktgen/$DEV@${COUNTER} - PGDEV=/proc/net/pktgen/$DEV + # 0 means continious sends untill explicitly stopped + pgset "count 0" - # 0 means continious sends untill explicitly stopped - pgset "count 0" + # set pps count to test with an explicit number. if 0 will try with bandwidth + if [ ${pps} -gt 0 ] + then + pgset "ratep ${pps}" + fi - # use single SKB for all transmits - pgset "clone_skb 0" + pgset "clone_skb 10" - # packet size, NIC adds 4 bytes CRC - pgset "pkt_size $PKT_SIZE" + # use different queue per thread + pgset "queue_map_min ${COUNTER}" + pgset "queue_map_max ${COUNTER}" - # random address within the min-max range - pgset "flag IPDST_RND UDPSRC_RND UDPDST_RND" + # packet size, NIC adds 4 bytes CRC + pgset "pkt_size $PKT_SIZE" - # destination IP - pgset "dst_min $DST_IP" - pgset "dst_max $DST_IP" + # random address within the min-max range + pgset "flag UDPDST_RND" + pgset "flag UDPSRC_RND" + pgset "flag IPDST_RND" - # destination MAC address - pgset "dst_mac $MAC" + # destination IP + pgset "dst_min $DST_IP" + pgset "dst_max $DST_IP" + + # destination MAC address + pgset "dst_mac $MAC" + + # source UDP port range + pgset "udp_src_min $UDP_SRC_MIN" + pgset "udp_src_max $UDP_SRC_MAX" - # source UDP port range - pgset "udp_src_min $UDP_SRC_MIN" - pgset "udp_src_max $UDP_SRC_MAX" + # destination UDP port range + pgset "udp_dst_min $UDP_DST_MIN" + pgset "udp_dst_max $UDP_DST_MAX" - # destination UDP port range - pgset "udp_dst_min $UDP_DST_MIN" - pgset "udp_dst_max $UDP_DST_MAX" + let COUNTER=COUNTER+1 + + done } # run pktgen pgrun() { - # Time to run, result can be vieved in /proc/net/pktgen/$DEV + # Time to run, result can be viewed in /proc/net/pktgen/$DEV PGDEV=/proc/net/pktgen/pgctrl # Will hang, Ctrl-C or SIGINT to stop pgset "start" start + + COUNTER=0 + while [ ${COUNTER} -lt ${TRXQUEUE} ]; do + taskset -c ${COUNTER} kpktgend_${COUNTER} + let COUNTER=COUNTER+1 + done } # run pktgen for ${DURATION} seconds @@ -111,19 +151,28 @@ run_test() # write the result to stdout in json format output_json() { - sent=$(awk '/^Result:/{print $5}' <$PGDEV) - pps=$(awk 'match($0,/'\([0-9]+\)pps'/, a) {print a[1]}' <$PGDEV) - errors=$(awk '/errors:/{print $5}' <$PGDEV) + sent=0 + result_pps=0 + errors=0 + PGDEV=/proc/net/pktgen/$DEV@ + COUNTER=0 + while [ ${COUNTER} -lt ${TRXQUEUE} ]; do + sent=$(($sent + $(awk '/^Result:/{print $5}' <$PGDEV${COUNTER}))) + result_pps=$(($result_pps + $(awk 'match($0,/'\([0-9]+\)pps'/, a) {print a[1]}' <$PGDEV${COUNTER}))) + errors=$(($errors + $(awk '/errors:/{print $5}' <$PGDEV${COUNTER}))) + let COUNTER=COUNTER+1 + done flows=$(( NUM_PORTS * (NUM_PORTS + 1) )) - echo { '"packets_sent"':$sent , '"packets_per_second"':$pps, '"flows"':$flows, '"errors"':$errors } + echo '{ "packets_sent"':${sent} , '"packets_per_second"':${result_pps}, '"flows"':${flows}, '"errors"':${errors} '}' } # main entry main() { modprobe pktgen + pgclean ping -c 3 $DST_IP >/dev/null @@ -137,16 +186,20 @@ main() pgconfig # run the test - run_test >/dev/null + run_test - PGDEV=/proc/net/pktgen/$DEV + PGDEV=/proc/net/pktgen/$DEV@ # check result - result=$(cat $PGDEV | fgrep "Result: OK:") - if [ "$result" = "" ]; then - cat $PGDEV | fgrep Result: >/dev/stderr - exit 1 - fi + COUNTER=0 + while [ ${COUNTER} -lt ${TRXQUEUE} ]; do + result=$(cat $PGDEV${COUNTER} | fgrep "Result: OK:") + if [ "$result" = "" ]; then + cat $PGDEV${COUNTER} | fgrep Result: >/dev/stderr + exit 1 + fi + let COUNTER=COUNTER+1 + done # output result output_json -- cgit 1.2.3-korg