Merge "Add multiqueue support"

2 files changed, 334 insertions, 47 deletions

diff --git a/yardstick/benchmark/scenarios/networking/pktgen.py b/yardstick/benchmark/scenarios/networking/pktgen.py
index e6aa7e5fb..8ca1ca60e 100644
--- a/yardstick/benchmark/scenarios/networking/pktgen.py
+++ b/yardstick/benchmark/scenarios/networking/pktgen.py
@@ -9,6 +9,7 @@
 from __future__ import absolute_import
 from __future__ import print_function
 
+import os
 import logging
 
 import pkg_resources
@@ -19,6 +20,9 @@ from yardstick.benchmark.scenarios import base
 
 LOG = logging.getLogger(__name__)
 
+VNIC_TYPE_LIST = ["ovs", "sriov"]
+SRIOV_DRIVER_LIST = ["ixgbevf", "i40evf"]
+
 
 class Pktgen(base.Scenario):
     """Execute pktgen between two hosts
@@ -44,7 +48,11 @@ class Pktgen(base.Scenario):
     def __init__(self, scenario_cfg, context_cfg):
         self.scenario_cfg = scenario_cfg
         self.context_cfg = context_cfg
+        self.vnic_name = "eth0"
+        self.vnic_type = "ovs"
+        self.queue_number = 1
         self.setup_done = False
+        self.multiqueue_setup_done = False
 
     def setup(self):
         """scenario setup"""
@@ -67,6 +75,212 @@ class Pktgen(base.Scenario):
 
         self.setup_done = True
 
+    def multiqueue_setup(self):
+        # one time setup stuff
+        cmd = "sudo sysctl -w net.core.netdev_budget=3000"
+        self.server.send_command(cmd)
+        self.client.send_command(cmd)
+
+        cmd = "sudo sysctl -w net.core.netdev_max_backlog=100000"
+        self.server.send_command(cmd)
+        self.client.send_command(cmd)
+
+        """multiqueue setup"""
+        if not self._is_irqbalance_disabled():
+            self._disable_irqbalance()
+
+        vnic_driver_name = self._get_vnic_driver_name()
+        if vnic_driver_name in SRIOV_DRIVER_LIST:
+            self.vnic_type = "sriov"
+
+            # one time setup stuff
+            cmd = "sudo ethtool -G %s rx 4096 tx 4096" % self.vnic_name
+            self.server.send_command(cmd)
+            self.client.send_command(cmd)
+
+            self.queue_number = self._get_sriov_queue_number()
+            self._setup_irqmapping_sriov(self.queue_number)
+        else:
+            self.vnic_type = "ovs"
+            self.queue_number = self._enable_ovs_multiqueue()
+            self._setup_irqmapping_ovs(self.queue_number)
+
+        self.multiqueue_setup_done = True
+
+    def _get_vnic_driver_name(self):
+        cmd = "readlink /sys/class/net/%s/device/driver" % self.vnic_name
+        LOG.debug("Executing command: %s", cmd)
+        status, stdout, stderr = self.server.execute(cmd)
+        if status:
+            raise RuntimeError(stderr)
+        return os.path.basename(stdout.strip())
+
+    def _is_irqbalance_disabled(self):
+        """Did we disable irqbalance already in the guest?"""
+        is_disabled = False
+        cmd = "grep ENABLED /etc/default/irqbalance"
+        status, stdout, stderr = self.server.execute(cmd)
+        if status:
+            raise RuntimeError(stderr)
+        if "0" in stdout:
+            is_disabled = True
+
+        return is_disabled
+
+    def _disable_irqbalance(self):
+        cmd = "sudo sed -i -e 's/ENABLED=\"1\"/ENABLED=\"0\"/g' " \
+              "/etc/default/irqbalance"
+        status, stdout, stderr = self.server.execute(cmd)
+        status, stdout, stderr = self.client.execute(cmd)
+        if status:
+            raise RuntimeError(stderr)
+
+        cmd = "sudo service irqbalance stop"
+        status, stdout, stderr = self.server.execute(cmd)
+        status, stdout, stderr = self.client.execute(cmd)
+        if status:
+            raise RuntimeError(stderr)
+
+        cmd = "sudo service irqbalance disable"
+        status, stdout, stderr = self.server.execute(cmd)
+        status, stdout, stderr = self.client.execute(cmd)
+        if status:
+            raise RuntimeError(stderr)
+
+    def _setup_irqmapping_ovs(self, queue_number):
+        cmd = "grep 'virtio0-input.0' /proc/interrupts |" \
+              "awk '{match($0,/ +[0-9]+/)} " \
+              "{print substr($1,RSTART,RLENGTH-1)}'"
+        status, stdout, stderr = self.server.execute(cmd)
+        if status:
+            raise RuntimeError(stderr)
+
+        cmd = "echo 1 | sudo tee /proc/irq/%s/smp_affinity" % (int(stdout))
+        status, stdout, stderr = self.server.execute(cmd)
+        status, stdout, stderr = self.client.execute(cmd)
+        if status:
+            raise RuntimeError(stderr)
+
+        cmd = "grep 'virtio0-output.0' /proc/interrupts |" \
+              "awk '{match($0,/ +[0-9]+/)} " \
+              "{print substr($1,RSTART,RLENGTH-1)}'"
+        status, stdout, stderr = self.server.execute(cmd)
+        if status:
+            raise RuntimeError(stderr)
+
+        cmd = "echo 1 | sudo tee /proc/irq/%s/smp_affinity" % (int(stdout))
+        status, stdout, stderr = self.server.execute(cmd)
+        status, stdout, stderr = self.client.execute(cmd)
+        if status:
+            raise RuntimeError(stderr)
+
+        if queue_number == 1:
+            return
+
+        for i in range(1, queue_number):
+            cmd = "grep 'virtio0-input.%s' /proc/interrupts |" \
+                  "awk '{match($0,/ +[0-9]+/)} " \
+                  "{print substr($1,RSTART,RLENGTH-1)}'" % (i)
+            status, stdout, stderr = self.server.execute(cmd)
+            if status:
+                raise RuntimeError(stderr)
+
+            cmd = "echo %s | sudo tee /proc/irq/%s/smp_affinity" \
+                % (1 << i, int(stdout))
+            status, stdout, stderr = self.server.execute(cmd)
+            status, stdout, stderr = self.client.execute(cmd)
+            if status:
+                raise RuntimeError(stderr)
+
+            cmd = "grep 'virtio0-output.%s' /proc/interrupts |" \
+                  "awk '{match($0,/ +[0-9]+/)} " \
+                  "{print substr($1,RSTART,RLENGTH-1)}'" % (i)
+            status, stdout, stderr = self.server.execute(cmd)
+            if status:
+                raise RuntimeError(stderr)
+
+            cmd = "echo %s | sudo tee /proc/irq/%s/smp_affinity" \
+                % (1 << i, int(stdout))
+            status, stdout, stderr = self.server.execute(cmd)
+            status, stdout, stderr = self.client.execute(cmd)
+            if status:
+                raise RuntimeError(stderr)
+
+    def _setup_irqmapping_sriov(self, queue_number):
+        cmd = "grep '%s-TxRx-0' /proc/interrupts |" \
+              "awk '{match($0,/ +[0-9]+/)} " \
+              "{print substr($1,RSTART,RLENGTH-1)}'" % self.vnic_name
+        status, stdout, stderr = self.server.execute(cmd)
+        if status:
+            raise RuntimeError(stderr)
+
+        cmd = "echo 1 | sudo tee /proc/irq/%s/smp_affinity" % (int(stdout))
+        status, stdout, stderr = self.server.execute(cmd)
+        status, stdout, stderr = self.client.execute(cmd)
+        if status:
+            raise RuntimeError(stderr)
+
+        if queue_number == 1:
+            return
+
+        for i in range(1, queue_number):
+            cmd = "grep '%s-TxRx-%s' /proc/interrupts |" \
+                  "awk '{match($0,/ +[0-9]+/)} " \
+                  "{print substr($1,RSTART,RLENGTH-1)}'" % (self.vnic_name, i)
+            status, stdout, stderr = self.server.execute(cmd)
+            if status:
+                raise RuntimeError(stderr)
+
+            cmd = "echo %s | sudo tee /proc/irq/%s/smp_affinity" \
+                % (1 << i, int(stdout))
+            status, stdout, stderr = self.server.execute(cmd)
+            status, stdout, stderr = self.client.execute(cmd)
+            if status:
+                raise RuntimeError(stderr)
+
+    def _get_sriov_queue_number(self):
+        """Get queue number from server as both VMs are the same"""
+        cmd = "grep %s-TxRx- /proc/interrupts | wc -l" % self.vnic_name
+        LOG.debug("Executing command: %s", cmd)
+        status, stdout, stderr = self.server.execute(cmd)
+        if status:
+            raise RuntimeError(stderr)
+        return int(stdout)
+
+    def _get_available_queue_number(self):
+        """Get queue number from client as both VMs are the same"""
+        cmd = "sudo ethtool -l %s | grep Combined | head -1 |" \
+              "awk '{printf $2}'" % self.vnic_name
+        LOG.debug("Executing command: %s", cmd)
+        status, stdout, stderr = self.server.execute(cmd)
+        if status:
+            raise RuntimeError(stderr)
+        return int(stdout)
+
+    def _get_usable_queue_number(self):
+        """Get queue number from client as both VMs are the same"""
+        cmd = "sudo ethtool -l %s | grep Combined | tail -1 |" \
+              "awk '{printf $2}'" % self.vnic_name
+        LOG.debug("Executing command: %s", cmd)
+        status, stdout, stderr = self.server.execute(cmd)
+        if status:
+            raise RuntimeError(stderr)
+        return int(stdout)
+
+    def _enable_ovs_multiqueue(self):
+        available_queue_number = self._get_available_queue_number()
+        usable_queue_number = self._get_usable_queue_number()
+        if available_queue_number > 1 and \
+           available_queue_number != usable_queue_number:
+            cmd = "sudo ethtool -L %s combined %s" % \
+                (self.vnic_name, available_queue_number)
+            LOG.debug("Executing command: %s", cmd)
+            status, stdout, stderr = self.server.execute(cmd)
+            status, stdout, stderr = self.client.execute(cmd)
+            if status:
+                raise RuntimeError(stderr)
+        return available_queue_number
+
     def _iptables_setup(self):
         """Setup iptables on server to monitor for received packets"""
         cmd = "sudo iptables -F; " \
@@ -99,6 +313,14 @@ class Pktgen(base.Scenario):
         options = self.scenario_cfg['options']
         packetsize = options.get("packetsize", 60)
         self.number_of_ports = options.get("number_of_ports", 10)
+        self.vnic_name = options.get("vnic_name", "eth0")
+        ovs_dpdk = options.get("ovs_dpdk", False)
+        pps = options.get("pps", 1000000)
+        multiqueue = options.get("multiqueue", False)
+
+        if multiqueue and not self.multiqueue_setup_done:
+            self.multiqueue_setup()
+
         # if run by a duration runner
         duration_time = self.scenario_cfg["runner"].get("duration", None) \
             if "runner" in self.scenario_cfg else None
@@ -114,8 +336,18 @@ class Pktgen(base.Scenario):
 
         self._iptables_setup()
 
-        cmd = "sudo bash pktgen.sh %s %s %s %s" \
-            % (ipaddr, self.number_of_ports, packetsize, duration)
+        queue_number = self.queue_number
+
+        # For native OVS, half of vCPUs are used by vhost kernel threads
+        # hence set the queue_number to half number of vCPUs
+        # e.g. set queue_number to 2 if there are 4 vCPUs
+        if self.vnic_type == "ovs" and not ovs_dpdk and self.queue_number > 1:
+            queue_number = self.queue_number / 2
+
+        cmd = "sudo bash pktgen.sh %s %s %s %s %s %s" \
+            % (ipaddr, self.number_of_ports, packetsize,
+               duration, queue_number, pps)
+
         LOG.debug("Executing command: %s", cmd)
         status, stdout, stderr = self.client.execute(cmd)
 
@@ -131,12 +363,15 @@ class Pktgen(base.Scenario):
             sent = result['packets_sent']
             received = result['packets_received']
             ppm = 1000000 * (sent - received) / sent
+            # if ppm is 1, then 11 out of 10 million is no pass
+            ppm += (sent - received) % sent > 0
+            LOG.debug("Lost packets %d - Lost ppm %d", (sent - received), ppm)
             sla_max_ppm = int(self.scenario_cfg["sla"]["max_ppm"])
             assert ppm <= sla_max_ppm, "ppm %d > sla_max_ppm %d; " \
                 % (ppm, sla_max_ppm)
 
 
-def _test():
+def _test():  # pragma: no cover
     """internal test function"""
     key_filename = pkg_resources.resource_filename('yardstick.resources',
                                                    'files/yardstick_key')
@@ -165,6 +400,5 @@ def _test():
     p.run(result)
     print(result)
 
-
 if __name__ == '__main__':
     _test()
diff --git a/yardstick/benchmark/scenarios/networking/pktgen_benchmark.bash b/yardstick/benchmark/scenarios/networking/pktgen_benchmark.bash
index 4224c5abf..e338a1b09 100644
--- a/yardstick/benchmark/scenarios/networking/pktgen_benchmark.bash
+++ b/yardstick/benchmark/scenarios/networking/pktgen_benchmark.bash
@@ -16,6 +16,8 @@ DST_IP=$1         # destination IP address
 NUM_PORTS=$2      # number of source ports
 PKT_SIZE=$3       # packet size
 DURATION=$4       # test duration (seconds)
+TRXQUEUE=$5       # number of RX/TX queues to use
+PPS=$6            # packets per second to send
 
 # Configuration
 UDP_SRC_MIN=1000                               # UDP source port min
@@ -37,62 +39,100 @@ pgset()
     fi
 }
 
+# remove all devices from thread
+pgclean()
+{
+    COUNTER=0
+    while [ ${COUNTER} -lt ${TRXQUEUE} ]; do
+        #
+        # Thread commands
+        #
+
+        PGDEV=/proc/net/pktgen/kpktgend_${COUNTER}
+
+        # Remove all devices from this thread
+        pgset "rem_device_all"
+        let COUNTER=COUNTER+1
+    done
+}
+
 # configure pktgen (see pktgen doc for details)
 pgconfig()
 {
-    #
-    # Thread commands
-    #
+    pps=$(( PPS / TRXQUEUE ))
+    COUNTER=0
+    while [ ${COUNTER} -lt ${TRXQUEUE} ]; do
+        #
+        # Thread commands
+        #
 
-    PGDEV=/proc/net/pktgen/kpktgend_0
+        PGDEV=/proc/net/pktgen/kpktgend_${COUNTER}
 
-    # Remove all devices from this thread
-    pgset "rem_device_all"
+        # Add device to thread
+        pgset "add_device $DEV@${COUNTER}"
 
-    # Add device to thread
-    pgset "add_device $DEV"
+        #
+        # Device commands
+        #
 
-    #
-    # Device commands
-    #
+        PGDEV=/proc/net/pktgen/$DEV@${COUNTER}
 
-    PGDEV=/proc/net/pktgen/$DEV
+        # 0 means continious sends untill explicitly stopped
+        pgset "count 0"
 
-    # 0 means continious sends untill explicitly stopped
-    pgset "count 0"
+        # set pps count to test with an explicit number. if 0 will try with bandwidth
+        if [ ${pps} -gt 0 ]
+        then
+            pgset "ratep ${pps}"
+        fi
 
-    # use single SKB for all transmits
-    pgset "clone_skb 0"
+        pgset "clone_skb 10"
 
-    # packet size, NIC adds 4 bytes CRC
-    pgset "pkt_size $PKT_SIZE"
+        # use different queue per thread
+        pgset "queue_map_min ${COUNTER}"
+        pgset "queue_map_max ${COUNTER}"
 
-    # random address within the min-max range
-    pgset "flag IPDST_RND UDPSRC_RND UDPDST_RND"
+        # packet size, NIC adds 4 bytes CRC
+        pgset "pkt_size $PKT_SIZE"
 
-    # destination IP
-    pgset "dst_min $DST_IP"
-    pgset "dst_max $DST_IP"
+        # random address within the min-max range
+        pgset "flag UDPDST_RND"
+        pgset "flag UDPSRC_RND"
+        pgset "flag IPDST_RND"
 
-    # destination MAC address
-    pgset "dst_mac $MAC"
+        # destination IP
+        pgset "dst_min $DST_IP"
+        pgset "dst_max $DST_IP"
+
+        # destination MAC address
+        pgset "dst_mac $MAC"
+
+        # source UDP port range
+        pgset "udp_src_min $UDP_SRC_MIN"
+        pgset "udp_src_max $UDP_SRC_MAX"
 
-    # source UDP port range
-    pgset "udp_src_min $UDP_SRC_MIN"
-    pgset "udp_src_max $UDP_SRC_MAX"
+        # destination UDP port range
+        pgset "udp_dst_min $UDP_DST_MIN"
+        pgset "udp_dst_max $UDP_DST_MAX"
 
-    # destination UDP port range
-    pgset "udp_dst_min $UDP_DST_MIN"
-    pgset "udp_dst_max $UDP_DST_MAX"
+        let COUNTER=COUNTER+1
+
+    done
 }
 
 # run pktgen
 pgrun()
 {
-    # Time to run, result can be vieved in /proc/net/pktgen/$DEV
+    # Time to run, result can be viewed in /proc/net/pktgen/$DEV
     PGDEV=/proc/net/pktgen/pgctrl
     # Will hang, Ctrl-C or SIGINT to stop
     pgset "start" start
+
+    COUNTER=0
+    while [ ${COUNTER} -lt ${TRXQUEUE} ]; do
+        taskset -c ${COUNTER} kpktgend_${COUNTER}
+        let COUNTER=COUNTER+1
+    done
 }
 
 # run pktgen for ${DURATION} seconds
@@ -111,19 +151,28 @@ run_test()
 # write the result to stdout in json format
 output_json()
 {
-    sent=$(awk '/^Result:/{print $5}' <$PGDEV)
-    pps=$(awk 'match($0,/'\([0-9]+\)pps'/, a) {print a[1]}' <$PGDEV)
-    errors=$(awk '/errors:/{print $5}' <$PGDEV)
+    sent=0
+    result_pps=0
+    errors=0
+    PGDEV=/proc/net/pktgen/$DEV@
+    COUNTER=0
+    while [ ${COUNTER} -lt ${TRXQUEUE} ]; do
+        sent=$(($sent + $(awk '/^Result:/{print $5}' <$PGDEV${COUNTER})))
+        result_pps=$(($result_pps + $(awk 'match($0,/'\([0-9]+\)pps'/, a) {print a[1]}' <$PGDEV${COUNTER})))
+        errors=$(($errors + $(awk '/errors:/{print $5}' <$PGDEV${COUNTER})))
+        let COUNTER=COUNTER+1
+    done
 
     flows=$(( NUM_PORTS * (NUM_PORTS + 1) ))
 
-    echo { '"packets_sent"':$sent , '"packets_per_second"':$pps, '"flows"':$flows, '"errors"':$errors }
+    echo '{ "packets_sent"':${sent} , '"packets_per_second"':${result_pps}, '"flows"':${flows}, '"errors"':${errors} '}'
 }
 
 # main entry
 main()
 {
     modprobe pktgen
+    pgclean
 
     ping -c 3 $DST_IP >/dev/null
 
@@ -137,16 +186,20 @@ main()
     pgconfig
 
     # run the test
-    run_test >/dev/null
+    run_test
 
-    PGDEV=/proc/net/pktgen/$DEV
+    PGDEV=/proc/net/pktgen/$DEV@
 
     # check result
-    result=$(cat $PGDEV | fgrep "Result: OK:")
-    if [ "$result" = "" ]; then
-         cat $PGDEV | fgrep Result: >/dev/stderr
-         exit 1
-    fi
+    COUNTER=0
+    while [  ${COUNTER} -lt ${TRXQUEUE} ]; do
+        result=$(cat $PGDEV${COUNTER} | fgrep "Result: OK:")
+        if [ "$result" = "" ]; then
+            cat $PGDEV${COUNTER} | fgrep Result: >/dev/stderr
+            exit 1
+        fi
+        let COUNTER=COUNTER+1
+    done
 
     # output result
     output_json