Add multiqueue support

Problem:
Neither OVS nor SRIOV multi-queue is not supported. Guest VM lacks tuning to reach high throughput.
Solution:
(1) Build SRIOV multi-queue capable guest image by recompiling the igxbevf driver (make CFLAGS_EXTRA=-DIXGBE_ENABLE_VF_MQ install).
(2) Change pktgen to send on multiple queues.
(3) Add tuning in guest VM (Disable irqbalance and setup vNIC interrupt affinity to vCPUs)

Update 1: Takes care comments plus adds unit test cases
Update 2: Jenkins reports code coverage 81%, local reports 96%, add more unit test cases
Update 3: Manually rebased to adapt to SSH.from_node()
Update 4: Takes care comments for vnic_type, vnic_name

Change-Id: Ieb15381c653b13697487d095efa4be6c3c49fa42
JIRA: YARDSTICK-619
Signed-off-by: Jing Zhang <jing.c.zhang@nokia.com>

4 files changed, 877 insertions, 47 deletions

diff --git a/tests/opnfv/test_cases/opnfv_yardstick_tc008.yaml b/tests/opnfv/test_cases/opnfv_yardstick_tc008.yaml
index 4c7fdab90..f5ccb255a 100644
--- a/tests/opnfv/test_cases/opnfv_yardstick_tc008.yaml
+++ b/tests/opnfv/test_cases/opnfv_yardstick_tc008.yaml
@@ -29,12 +29,21 @@ scenarios:
     packetsize: {{pkt_size}}
     number_of_ports: {{num_ports}}
     duration: 20
+    # choose vnic name: default to eth0
+    # vnic_name: 'ens3'
+    # turn on multiqueue inside VM
+    # multiqueue: True
+    # choose starting pps: default 1M;
+    # works with binary search runner Dynamictp to find max throughput per sla
+    # pps: 3000000
 
   host: demeter.yardstick-TC008
   target: poseidon.yardstick-TC008
 
   runner:
     type: Iteration
+    # binary search runner
+    # type: Dynamictp
     iterations: 10
     interval: 1
 
diff --git a/tests/unit/benchmark/scenarios/networking/test_pktgen.py b/tests/unit/benchmark/scenarios/networking/test_pktgen.py
index d4eb1246f..2914c8e02 100644
--- a/tests/unit/benchmark/scenarios/networking/test_pktgen.py
+++ b/tests/unit/benchmark/scenarios/networking/test_pktgen.py
@@ -138,6 +138,7 @@ class PktgenTestCase(unittest.TestCase):
         p.run(result)
         expected_result = jsonutils.loads(sample_output)
         expected_result["packets_received"] = 149300
+        expected_result["packetsize"] = 60
         self.assertEqual(result, expected_result)
 
     def test_pktgen_successful_sla(self, mock_ssh):
@@ -164,6 +165,7 @@ class PktgenTestCase(unittest.TestCase):
         p.run(result)
         expected_result = jsonutils.loads(sample_output)
         expected_result["packets_received"] = 149300
+        expected_result["packetsize"] = 60
         self.assertEqual(result, expected_result)
 
     def test_pktgen_unsuccessful_sla(self, mock_ssh):
@@ -204,6 +206,538 @@ class PktgenTestCase(unittest.TestCase):
         mock_ssh.SSH.from_node().execute.return_value = (1, '', 'FOOBAR')
         self.assertRaises(RuntimeError, p.run, result)
 
+    def test_pktgen_get_vnic_driver_name(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, 'ixgbevf', '')
+
+        vnic_driver_name = p._get_vnic_driver_name()
+        self.assertEqual(vnic_driver_name, 'ixgbevf')
+
+    def test_pktgen_unsuccessful_get_vnic_driver_name(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (1, '', '')
+
+        self.assertRaises(RuntimeError, p._get_vnic_driver_name)
+
+    def test_pktgen_get_sriov_queue_number(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, '2', '')
+
+        p.queue_number = p._get_sriov_queue_number()
+        self.assertEqual(p.queue_number, 2)
+
+    def test_pktgen_unsuccessful_get_sriov_queue_number(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (1, '', '')
+
+        self.assertRaises(RuntimeError, p._get_sriov_queue_number)
+
+    def test_pktgen_get_available_queue_number(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, '4', '')
+
+        p._get_available_queue_number()
+
+        mock_ssh.SSH.from_node().execute.assert_called_with(
+            "sudo ethtool -l eth0 | grep Combined | head -1 |" \
+            "awk '{printf $2}'")
+
+    def test_pktgen_unsuccessful_get_available_queue_number(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (1, '', '')
+
+        self.assertRaises(RuntimeError, p._get_available_queue_number)
+
+    def test_pktgen_get_usable_queue_number(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, '1', '')
+
+        p._get_usable_queue_number()
+
+        mock_ssh.SSH.from_node().execute.assert_called_with(
+            "sudo ethtool -l eth0 | grep Combined | tail -1 |" \
+            "awk '{printf $2}'")
+
+    def test_pktgen_unsuccessful_get_usable_queue_number(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (1, '', '')
+
+        self.assertRaises(RuntimeError, p._get_usable_queue_number)
+
+    def test_pktgen_enable_ovs_multiqueue(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, '4', '')
+
+        mock_result1 = mock.Mock()
+        mock_result1.return_value = 1
+        p._get_usable_queue_number = mock_result1
+
+        mock_result2 = mock.Mock()
+        mock_result2.return_value = 4
+        p._get_available_queue_number = mock_result2
+
+        p.queue_number = p._enable_ovs_multiqueue()
+        self.assertEqual(p.queue_number, 4)
+
+    def test_pktgen_enable_ovs_multiqueue_1q(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, '1', '')
+
+        mock_result1 = mock.Mock()
+        mock_result1.return_value = 1
+        p._get_usable_queue_number = mock_result1
+
+        mock_result2 = mock.Mock()
+        mock_result2.return_value = 1
+        p._get_available_queue_number = mock_result2
+
+        p.queue_number = p._enable_ovs_multiqueue()
+        self.assertEqual(p.queue_number, 1)
+
+    def test_pktgen_unsuccessful_enable_ovs_multiqueue(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (1, '', '')
+
+        mock_result1 = mock.Mock()
+        mock_result1.return_value = 1
+        p._get_usable_queue_number = mock_result1
+
+        mock_result2 = mock.Mock()
+        mock_result2.return_value = 4
+        p._get_available_queue_number = mock_result2
+
+        self.assertRaises(RuntimeError, p._enable_ovs_multiqueue)
+
+    def test_pktgen_setup_irqmapping_ovs(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, '10', '')
+
+        p._setup_irqmapping_ovs(4)
+
+        mock_ssh.SSH.from_node().execute.assert_called_with(
+            "echo 8 | sudo tee /proc/irq/10/smp_affinity")
+
+    def test_pktgen_setup_irqmapping_ovs_1q(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, '10', '')
+
+        p._setup_irqmapping_ovs(1)
+
+        mock_ssh.SSH.from_node().execute.assert_called_with(
+            "echo 1 | sudo tee /proc/irq/10/smp_affinity")
+
+    def test_pktgen_unsuccessful_setup_irqmapping_ovs(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (1, '', '')
+
+        self.assertRaises(RuntimeError, p._setup_irqmapping_ovs, 4)
+
+    def test_pktgen_unsuccessful_setup_irqmapping_ovs_1q(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (1, '', '')
+
+        self.assertRaises(RuntimeError, p._setup_irqmapping_ovs, 1)
+
+    def test_pktgen_setup_irqmapping_sriov(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, '10', '')
+
+        p._setup_irqmapping_sriov(2)
+
+        mock_ssh.SSH.from_node().execute.assert_called_with(
+            "echo 2 | sudo tee /proc/irq/10/smp_affinity")
+
+    def test_pktgen_setup_irqmapping_sriov_1q(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, '10', '')
+
+        p._setup_irqmapping_sriov(1)
+
+        mock_ssh.SSH.from_node().execute.assert_called_with(
+            "echo 1 | sudo tee /proc/irq/10/smp_affinity")
+
+    def test_pktgen_unsuccessful_setup_irqmapping_sriov(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (1, '', '')
+
+        self.assertRaises(RuntimeError, p._setup_irqmapping_sriov, 2)
+
+    def test_pktgen_unsuccessful_setup_irqmapping_sriov_1q(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (1, '', '')
+
+        self.assertRaises(RuntimeError, p._setup_irqmapping_sriov, 1)
+
+    def test_pktgen_is_irqbalance_disabled(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, '', '')
+
+        p._is_irqbalance_disabled()
+
+        mock_ssh.SSH.from_node().execute.assert_called_with(
+            "grep ENABLED /etc/default/irqbalance")
+
+    def test_pktgen_unsuccessful_is_irqbalance_disabled(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (1, '', '')
+
+        self.assertRaises(RuntimeError, p._is_irqbalance_disabled)
+
+    def test_pktgen_disable_irqbalance(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, '', '')
+
+        p._disable_irqbalance()
+
+        mock_ssh.SSH.from_node().execute.assert_called_with(
+            "sudo service irqbalance disable")
+
+    def test_pktgen_unsuccessful_disable_irqbalance(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (1, '', '')
+
+        self.assertRaises(RuntimeError, p._disable_irqbalance)
+
+    def test_pktgen_multiqueue_setup_ovs(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60, 'multiqueue': True},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, '4', '')
+
+        mock_result1 = mock.Mock()
+        mock_result1.return_value = False
+        p._is_irqbalance_disabled = mock_result1
+
+        mock_result2 = mock.Mock()
+        mock_result2.return_value = "virtio_net" 
+        p._get_vnic_driver_name = mock_result2
+
+        mock_result3 = mock.Mock()
+        mock_result3.return_value = 1
+        p._get_usable_queue_number = mock_result3
+
+        mock_result4 = mock.Mock()
+        mock_result4.return_value = 4
+        p._get_available_queue_number = mock_result4
+
+        p.multiqueue_setup()
+
+        self.assertEqual(p.queue_number, 4)
+
+    def test_pktgen_multiqueue_setup_ovs_1q(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60, 'multiqueue': True},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, '1', '')
+
+        mock_result1 = mock.Mock()
+        mock_result1.return_value = False
+        p._is_irqbalance_disabled = mock_result1
+
+        mock_result2 = mock.Mock()
+        mock_result2.return_value = "virtio_net" 
+        p._get_vnic_driver_name = mock_result2
+
+        mock_result3 = mock.Mock()
+        mock_result3.return_value = 1
+        p._get_usable_queue_number = mock_result3
+
+        mock_result4 = mock.Mock()
+        mock_result4.return_value = 1
+        p._get_available_queue_number = mock_result4
+
+        p.multiqueue_setup()
+
+        self.assertEqual(p.queue_number, 1)
+
+    def test_pktgen_multiqueue_setup_sriov(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60, 'multiqueue': True},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, '2', '')
+
+        mock_result1 = mock.Mock()
+        mock_result1.return_value = False
+        p._is_irqbalance_disabled = mock_result1
+
+        mock_result2 = mock.Mock()
+        mock_result2.return_value = "ixgbevf" 
+        p._get_vnic_driver_name = mock_result2
+
+        p.multiqueue_setup()
+
+        self.assertEqual(p.queue_number, 2)
+
+    def test_pktgen_multiqueue_setup_sriov_1q(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60, 'multiqueue': True},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, '1', '')
+
+        mock_result1 = mock.Mock()
+        mock_result1.return_value = False
+        p._is_irqbalance_disabled = mock_result1
+
+        mock_result2 = mock.Mock()
+        mock_result2.return_value = "ixgbevf" 
+        p._get_vnic_driver_name = mock_result2
+
+        p.multiqueue_setup()
+
+        self.assertEqual(p.queue_number, 1)
+
+    def test_pktgen_run_with_setup_done(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60, 'number_of_ports': 10, 'duration': 20, 'multiqueue': True},
+            'sla': {'max_ppm': 1}
+        }
+        result = {}
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        p.setup_done = True
+        p.multiqueue_setup_done = True
+
+        mock_iptables_result = mock.Mock()
+        mock_iptables_result.return_value = 149300
+        p._iptables_get_result = mock_iptables_result
+
+        sample_output = '{"packets_per_second": 9753, "errors": 0, \
+            "packets_sent": 149300, "flows": 110}'
+        mock_ssh.SSH.from_node().execute.return_value = (0, sample_output, '')
+
+        p.run(result)
+        expected_result = jsonutils.loads(sample_output)
+        expected_result["packets_received"] = 149300
+        expected_result["packetsize"] = 60
+        self.assertEqual(result, expected_result)
+
+    def test_pktgen_run_with_ovs_multiqueque(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60, 'number_of_ports': 10, 'duration': 20, 'multiqueue': True},
+            'sla': {'max_ppm': 1}
+        }
+        result = {}
+
+        p = pktgen.Pktgen(args, self.ctx)
+
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_result = mock.Mock()
+        mock_result.return_value = "virtio_net" 
+        p._get_vnic_driver_name = mock_result
+
+        mock_result1 = mock.Mock()
+        mock_result1.return_value = 1
+        p._get_usable_queue_number = mock_result1
+
+        mock_result2 = mock.Mock()
+        mock_result2.return_value = 4
+        p._get_available_queue_number = mock_result2
+
+        mock_result3 = mock.Mock()
+        mock_result3.return_value = 4
+        p._enable_ovs_multiqueue  = mock_result3
+
+        mock_result4 = mock.Mock()
+        p._setup_irqmapping_ovs = mock_result4
+
+        mock_iptables_result = mock.Mock()
+        mock_iptables_result.return_value = 149300
+        p._iptables_get_result = mock_iptables_result
+
+        sample_output = '{"packets_per_second": 9753, "errors": 0, \
+            "packets_sent": 149300, "flows": 110}'
+        mock_ssh.SSH.from_node().execute.return_value = (0, sample_output, '')
+
+        p.run(result)
+        expected_result = jsonutils.loads(sample_output)
+        expected_result["packets_received"] = 149300
+        expected_result["packetsize"] = 60
+        self.assertEqual(result, expected_result)
+
+    def test_pktgen_run_with_sriov_multiqueque(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60, 'number_of_ports': 10, 'duration': 20, 'multiqueue': True},
+            'sla': {'max_ppm': 1}
+        }
+        result = {}
+
+        p = pktgen.Pktgen(args, self.ctx)
+
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_result1 = mock.Mock()
+        mock_result1.return_value = "ixgbevf"
+        p._get_vnic_driver_name = mock_result1
+
+        mock_result2 = mock.Mock()
+        mock_result2.return_value = 2
+        p._get_sriov_queue_number = mock_result2
+
+        mock_result3 = mock.Mock()
+        p._setup_irqmapping_sriov = mock_result3
+
+        mock_iptables_result = mock.Mock()
+        mock_iptables_result.return_value = 149300
+        p._iptables_get_result = mock_iptables_result
+
+        sample_output = '{"packets_per_second": 9753, "errors": 0, \
+            "packets_sent": 149300, "flows": 110}'
+        mock_ssh.SSH.from_node().execute.return_value = (0, sample_output, '')
+
+        p.run(result)
+        expected_result = jsonutils.loads(sample_output)
+        expected_result["packets_received"] = 149300
+        expected_result["packetsize"] = 60
+        self.assertEqual(result, expected_result)
 
 def main():
     unittest.main()
diff --git a/yardstick/benchmark/scenarios/networking/pktgen.py b/yardstick/benchmark/scenarios/networking/pktgen.py
index e6aa7e5fb..8ca1ca60e 100644
--- a/yardstick/benchmark/scenarios/networking/pktgen.py
+++ b/yardstick/benchmark/scenarios/networking/pktgen.py
@@ -9,6 +9,7 @@
 from __future__ import absolute_import
 from __future__ import print_function
 
+import os
 import logging
 
 import pkg_resources
@@ -19,6 +20,9 @@ from yardstick.benchmark.scenarios import base
 
 LOG = logging.getLogger(__name__)
 
+VNIC_TYPE_LIST = ["ovs", "sriov"]
+SRIOV_DRIVER_LIST = ["ixgbevf", "i40evf"]
+
 
 class Pktgen(base.Scenario):
     """Execute pktgen between two hosts
@@ -44,7 +48,11 @@ class Pktgen(base.Scenario):
     def __init__(self, scenario_cfg, context_cfg):
         self.scenario_cfg = scenario_cfg
         self.context_cfg = context_cfg
+        self.vnic_name = "eth0"
+        self.vnic_type = "ovs"
+        self.queue_number = 1
         self.setup_done = False
+        self.multiqueue_setup_done = False
 
     def setup(self):
         """scenario setup"""
@@ -67,6 +75,212 @@ class Pktgen(base.Scenario):
 
         self.setup_done = True
 
+    def multiqueue_setup(self):
+        # one time setup stuff
+        cmd = "sudo sysctl -w net.core.netdev_budget=3000"
+        self.server.send_command(cmd)
+        self.client.send_command(cmd)
+
+        cmd = "sudo sysctl -w net.core.netdev_max_backlog=100000"
+        self.server.send_command(cmd)
+        self.client.send_command(cmd)
+
+        """multiqueue setup"""
+        if not self._is_irqbalance_disabled():
+            self._disable_irqbalance()
+
+        vnic_driver_name = self._get_vnic_driver_name()
+        if vnic_driver_name in SRIOV_DRIVER_LIST:
+            self.vnic_type = "sriov"
+
+            # one time setup stuff
+            cmd = "sudo ethtool -G %s rx 4096 tx 4096" % self.vnic_name
+            self.server.send_command(cmd)
+            self.client.send_command(cmd)
+
+            self.queue_number = self._get_sriov_queue_number()
+            self._setup_irqmapping_sriov(self.queue_number)
+        else:
+            self.vnic_type = "ovs"
+            self.queue_number = self._enable_ovs_multiqueue()
+            self._setup_irqmapping_ovs(self.queue_number)
+
+        self.multiqueue_setup_done = True
+
+    def _get_vnic_driver_name(self):
+        cmd = "readlink /sys/class/net/%s/device/driver" % self.vnic_name
+        LOG.debug("Executing command: %s", cmd)
+        status, stdout, stderr = self.server.execute(cmd)
+        if status:
+            raise RuntimeError(stderr)
+        return os.path.basename(stdout.strip())
+
+    def _is_irqbalance_disabled(self):
+        """Did we disable irqbalance already in the guest?"""
+        is_disabled = False
+        cmd = "grep ENABLED /etc/default/irqbalance"
+        status, stdout, stderr = self.server.execute(cmd)
+        if status:
+            raise RuntimeError(stderr)
+        if "0" in stdout:
+            is_disabled = True
+
+        return is_disabled
+
+    def _disable_irqbalance(self):
+        cmd = "sudo sed -i -e 's/ENABLED=\"1\"/ENABLED=\"0\"/g' " \
+              "/etc/default/irqbalance"
+        status, stdout, stderr = self.server.execute(cmd)
+        status, stdout, stderr = self.client.execute(cmd)
+        if status:
+            raise RuntimeError(stderr)
+
+        cmd = "sudo service irqbalance stop"
+        status, stdout, stderr = self.server.execute(cmd)
+        status, stdout, stderr = self.client.execute(cmd)
+        if status:
+            raise RuntimeError(stderr)
+
+        cmd = "sudo service irqbalance disable"
+        status, stdout, stderr = self.server.execute(cmd)
+        status, stdout, stderr = self.client.execute(cmd)
+        if status:
+            raise RuntimeError(stderr)
+
+    def _setup_irqmapping_ovs(self, queue_number):
+        cmd = "grep 'virtio0-input.0' /proc/interrupts |" \
+              "awk '{match($0,/ +[0-9]+/)} " \
+              "{print substr($1,RSTART,RLENGTH-1)}'"
+        status, stdout, stderr = self.server.execute(cmd)
+        if status:
+            raise RuntimeError(stderr)
+
+        cmd = "echo 1 | sudo tee /proc/irq/%s/smp_affinity" % (int(stdout))
+        status, stdout, stderr = self.server.execute(cmd)
+        status, stdout, stderr = self.client.execute(cmd)
+        if status:
+            raise RuntimeError(stderr)
+
+        cmd = "grep 'virtio0-output.0' /proc/interrupts |" \
+              "awk '{match($0,/ +[0-9]+/)} " \
+              "{print substr($1,RSTART,RLENGTH-1)}'"
+        status, stdout, stderr = self.server.execute(cmd)
+        if status:
+            raise RuntimeError(stderr)
+
+        cmd = "echo 1 | sudo tee /proc/irq/%s/smp_affinity" % (int(stdout))
+        status, stdout, stderr = self.server.execute(cmd)
+        status, stdout, stderr = self.client.execute(cmd)
+        if status:
+            raise RuntimeError(stderr)
+
+        if queue_number == 1:
+            return
+
+        for i in range(1, queue_number):
+            cmd = "grep 'virtio0-input.%s' /proc/interrupts |" \
+                  "awk '{match($0,/ +[0-9]+/)} " \
+                  "{print substr($1,RSTART,RLENGTH-1)}'" % (i)
+            status, stdout, stderr = self.server.execute(cmd)
+            if status:
+                raise RuntimeError(stderr)
+
+            cmd = "echo %s | sudo tee /proc/irq/%s/smp_affinity" \
+                % (1 << i, int(stdout))
+            status, stdout, stderr = self.server.execute(cmd)
+            status, stdout, stderr = self.client.execute(cmd)
+            if status:
+                raise RuntimeError(stderr)
+
+            cmd = "grep 'virtio0-output.%s' /proc/interrupts |" \
+                  "awk '{match($0,/ +[0-9]+/)} " \
+                  "{print substr($1,RSTART,RLENGTH-1)}'" % (i)
+            status, stdout, stderr = self.server.execute(cmd)
+            if status:
+                raise RuntimeError(stderr)
+
+            cmd = "echo %s | sudo tee /proc/irq/%s/smp_affinity" \
+                % (1 << i, int(stdout))
+            status, stdout, stderr = self.server.execute(cmd)
+            status, stdout, stderr = self.client.execute(cmd)
+            if status:
+                raise RuntimeError(stderr)
+
+    def _setup_irqmapping_sriov(self, queue_number):
+        cmd = "grep '%s-TxRx-0' /proc/interrupts |" \
+              "awk '{match($0,/ +[0-9]+/)} " \
+              "{print substr($1,RSTART,RLENGTH-1)}'" % self.vnic_name
+        status, stdout, stderr = self.server.execute(cmd)
+        if status:
+            raise RuntimeError(stderr)
+
+        cmd = "echo 1 | sudo tee /proc/irq/%s/smp_affinity" % (int(stdout))
+        status, stdout, stderr = self.server.execute(cmd)
+        status, stdout, stderr = self.client.execute(cmd)
+        if status:
+            raise RuntimeError(stderr)
+
+        if queue_number == 1:
+            return
+
+        for i in range(1, queue_number):
+            cmd = "grep '%s-TxRx-%s' /proc/interrupts |" \
+                  "awk '{match($0,/ +[0-9]+/)} " \
+                  "{print substr($1,RSTART,RLENGTH-1)}'" % (self.vnic_name, i)
+            status, stdout, stderr = self.server.execute(cmd)
+            if status:
+                raise RuntimeError(stderr)
+
+            cmd = "echo %s | sudo tee /proc/irq/%s/smp_affinity" \
+                % (1 << i, int(stdout))
+            status, stdout, stderr = self.server.execute(cmd)
+            status, stdout, stderr = self.client.execute(cmd)
+            if status:
+                raise RuntimeError(stderr)
+
+    def _get_sriov_queue_number(self):
+        """Get queue number from server as both VMs are the same"""
+        cmd = "grep %s-TxRx- /proc/interrupts | wc -l" % self.vnic_name
+        LOG.debug("Executing command: %s", cmd)
+        status, stdout, stderr = self.server.execute(cmd)
+        if status:
+            raise RuntimeError(stderr)
+        return int(stdout)
+
+    def _get_available_queue_number(self):
+        """Get queue number from client as both VMs are the same"""
+        cmd = "sudo ethtool -l %s | grep Combined | head -1 |" \
+              "awk '{printf $2}'" % self.vnic_name
+        LOG.debug("Executing command: %s", cmd)
+        status, stdout, stderr = self.server.execute(cmd)
+        if status:
+            raise RuntimeError(stderr)
+        return int(stdout)
+
+    def _get_usable_queue_number(self):
+        """Get queue number from client as both VMs are the same"""
+        cmd = "sudo ethtool -l %s | grep Combined | tail -1 |" \
+              "awk '{printf $2}'" % self.vnic_name
+        LOG.debug("Executing command: %s", cmd)
+        status, stdout, stderr = self.server.execute(cmd)
+        if status:
+            raise RuntimeError(stderr)
+        return int(stdout)
+
+    def _enable_ovs_multiqueue(self):
+        available_queue_number = self._get_available_queue_number()
+        usable_queue_number = self._get_usable_queue_number()
+        if available_queue_number > 1 and \
+           available_queue_number != usable_queue_number:
+            cmd = "sudo ethtool -L %s combined %s" % \
+                (self.vnic_name, available_queue_number)
+            LOG.debug("Executing command: %s", cmd)
+            status, stdout, stderr = self.server.execute(cmd)
+            status, stdout, stderr = self.client.execute(cmd)
+            if status:
+                raise RuntimeError(stderr)
+        return available_queue_number
+
     def _iptables_setup(self):
         """Setup iptables on server to monitor for received packets"""
         cmd = "sudo iptables -F; " \
@@ -99,6 +313,14 @@ class Pktgen(base.Scenario):
         options = self.scenario_cfg['options']
         packetsize = options.get("packetsize", 60)
         self.number_of_ports = options.get("number_of_ports", 10)
+        self.vnic_name = options.get("vnic_name", "eth0")
+        ovs_dpdk = options.get("ovs_dpdk", False)
+        pps = options.get("pps", 1000000)
+        multiqueue = options.get("multiqueue", False)
+
+        if multiqueue and not self.multiqueue_setup_done:
+            self.multiqueue_setup()
+
         # if run by a duration runner
         duration_time = self.scenario_cfg["runner"].get("duration", None) \
             if "runner" in self.scenario_cfg else None
@@ -114,8 +336,18 @@ class Pktgen(base.Scenario):
 
         self._iptables_setup()
 
-        cmd = "sudo bash pktgen.sh %s %s %s %s" \
-            % (ipaddr, self.number_of_ports, packetsize, duration)
+        queue_number = self.queue_number
+
+        # For native OVS, half of vCPUs are used by vhost kernel threads
+        # hence set the queue_number to half number of vCPUs
+        # e.g. set queue_number to 2 if there are 4 vCPUs
+        if self.vnic_type == "ovs" and not ovs_dpdk and self.queue_number > 1:
+            queue_number = self.queue_number / 2
+
+        cmd = "sudo bash pktgen.sh %s %s %s %s %s %s" \
+            % (ipaddr, self.number_of_ports, packetsize,
+               duration, queue_number, pps)
+
         LOG.debug("Executing command: %s", cmd)
         status, stdout, stderr = self.client.execute(cmd)
 
@@ -131,12 +363,15 @@ class Pktgen(base.Scenario):
             sent = result['packets_sent']
             received = result['packets_received']
             ppm = 1000000 * (sent - received) / sent
+            # if ppm is 1, then 11 out of 10 million is no pass
+            ppm += (sent - received) % sent > 0
+            LOG.debug("Lost packets %d - Lost ppm %d", (sent - received), ppm)
             sla_max_ppm = int(self.scenario_cfg["sla"]["max_ppm"])
             assert ppm <= sla_max_ppm, "ppm %d > sla_max_ppm %d; " \
                 % (ppm, sla_max_ppm)
 
 
-def _test():
+def _test():  # pragma: no cover
     """internal test function"""
     key_filename = pkg_resources.resource_filename('yardstick.resources',
                                                    'files/yardstick_key')
@@ -165,6 +400,5 @@ def _test():
     p.run(result)
     print(result)
 
-
 if __name__ == '__main__':
     _test()
diff --git a/yardstick/benchmark/scenarios/networking/pktgen_benchmark.bash b/yardstick/benchmark/scenarios/networking/pktgen_benchmark.bash
index 4224c5abf..e338a1b09 100644
--- a/yardstick/benchmark/scenarios/networking/pktgen_benchmark.bash
+++ b/yardstick/benchmark/scenarios/networking/pktgen_benchmark.bash
@@ -16,6 +16,8 @@ DST_IP=$1         # destination IP address
 NUM_PORTS=$2      # number of source ports
 PKT_SIZE=$3       # packet size
 DURATION=$4       # test duration (seconds)
+TRXQUEUE=$5       # number of RX/TX queues to use
+PPS=$6            # packets per second to send
 
 # Configuration
 UDP_SRC_MIN=1000                               # UDP source port min
@@ -37,62 +39,100 @@ pgset()
     fi
 }
 
+# remove all devices from thread
+pgclean()
+{
+    COUNTER=0
+    while [ ${COUNTER} -lt ${TRXQUEUE} ]; do
+        #
+        # Thread commands
+        #
+
+        PGDEV=/proc/net/pktgen/kpktgend_${COUNTER}
+
+        # Remove all devices from this thread
+        pgset "rem_device_all"
+        let COUNTER=COUNTER+1
+    done
+}
+
 # configure pktgen (see pktgen doc for details)
 pgconfig()
 {
-    #
-    # Thread commands
-    #
+    pps=$(( PPS / TRXQUEUE ))
+    COUNTER=0
+    while [ ${COUNTER} -lt ${TRXQUEUE} ]; do
+        #
+        # Thread commands
+        #
 
-    PGDEV=/proc/net/pktgen/kpktgend_0
+        PGDEV=/proc/net/pktgen/kpktgend_${COUNTER}
 
-    # Remove all devices from this thread
-    pgset "rem_device_all"
+        # Add device to thread
+        pgset "add_device $DEV@${COUNTER}"
 
-    # Add device to thread
-    pgset "add_device $DEV"
+        #
+        # Device commands
+        #
 
-    #
-    # Device commands
-    #
+        PGDEV=/proc/net/pktgen/$DEV@${COUNTER}
 
-    PGDEV=/proc/net/pktgen/$DEV
+        # 0 means continious sends untill explicitly stopped
+        pgset "count 0"
 
-    # 0 means continious sends untill explicitly stopped
-    pgset "count 0"
+        # set pps count to test with an explicit number. if 0 will try with bandwidth
+        if [ ${pps} -gt 0 ]
+        then
+            pgset "ratep ${pps}"
+        fi
 
-    # use single SKB for all transmits
-    pgset "clone_skb 0"
+        pgset "clone_skb 10"
 
-    # packet size, NIC adds 4 bytes CRC
-    pgset "pkt_size $PKT_SIZE"
+        # use different queue per thread
+        pgset "queue_map_min ${COUNTER}"
+        pgset "queue_map_max ${COUNTER}"
 
-    # random address within the min-max range
-    pgset "flag IPDST_RND UDPSRC_RND UDPDST_RND"
+        # packet size, NIC adds 4 bytes CRC
+        pgset "pkt_size $PKT_SIZE"
 
-    # destination IP
-    pgset "dst_min $DST_IP"
-    pgset "dst_max $DST_IP"
+        # random address within the min-max range
+        pgset "flag UDPDST_RND"
+        pgset "flag UDPSRC_RND"
+        pgset "flag IPDST_RND"
 
-    # destination MAC address
-    pgset "dst_mac $MAC"
+        # destination IP
+        pgset "dst_min $DST_IP"
+        pgset "dst_max $DST_IP"
+
+        # destination MAC address
+        pgset "dst_mac $MAC"
+
+        # source UDP port range
+        pgset "udp_src_min $UDP_SRC_MIN"
+        pgset "udp_src_max $UDP_SRC_MAX"
 
-    # source UDP port range
-    pgset "udp_src_min $UDP_SRC_MIN"
-    pgset "udp_src_max $UDP_SRC_MAX"
+        # destination UDP port range
+        pgset "udp_dst_min $UDP_DST_MIN"
+        pgset "udp_dst_max $UDP_DST_MAX"
 
-    # destination UDP port range
-    pgset "udp_dst_min $UDP_DST_MIN"
-    pgset "udp_dst_max $UDP_DST_MAX"
+        let COUNTER=COUNTER+1
+
+    done
 }
 
 # run pktgen
 pgrun()
 {
-    # Time to run, result can be vieved in /proc/net/pktgen/$DEV
+    # Time to run, result can be viewed in /proc/net/pktgen/$DEV
     PGDEV=/proc/net/pktgen/pgctrl
     # Will hang, Ctrl-C or SIGINT to stop
     pgset "start" start
+
+    COUNTER=0
+    while [ ${COUNTER} -lt ${TRXQUEUE} ]; do
+        taskset -c ${COUNTER} kpktgend_${COUNTER}
+        let COUNTER=COUNTER+1
+    done
 }
 
 # run pktgen for ${DURATION} seconds
@@ -111,19 +151,28 @@ run_test()
 # write the result to stdout in json format
 output_json()
 {
-    sent=$(awk '/^Result:/{print $5}' <$PGDEV)
-    pps=$(awk 'match($0,/'\([0-9]+\)pps'/, a) {print a[1]}' <$PGDEV)
-    errors=$(awk '/errors:/{print $5}' <$PGDEV)
+    sent=0
+    result_pps=0
+    errors=0
+    PGDEV=/proc/net/pktgen/$DEV@
+    COUNTER=0
+    while [ ${COUNTER} -lt ${TRXQUEUE} ]; do
+        sent=$(($sent + $(awk '/^Result:/{print $5}' <$PGDEV${COUNTER})))
+        result_pps=$(($result_pps + $(awk 'match($0,/'\([0-9]+\)pps'/, a) {print a[1]}' <$PGDEV${COUNTER})))
+        errors=$(($errors + $(awk '/errors:/{print $5}' <$PGDEV${COUNTER})))
+        let COUNTER=COUNTER+1
+    done
 
     flows=$(( NUM_PORTS * (NUM_PORTS + 1) ))
 
-    echo { '"packets_sent"':$sent , '"packets_per_second"':$pps, '"flows"':$flows, '"errors"':$errors }
+    echo '{ "packets_sent"':${sent} , '"packets_per_second"':${result_pps}, '"flows"':${flows}, '"errors"':${errors} '}'
 }
 
 # main entry
 main()
 {
     modprobe pktgen
+    pgclean
 
     ping -c 3 $DST_IP >/dev/null
 
@@ -137,16 +186,20 @@ main()
     pgconfig
 
     # run the test
-    run_test >/dev/null
+    run_test
 
-    PGDEV=/proc/net/pktgen/$DEV
+    PGDEV=/proc/net/pktgen/$DEV@
 
     # check result
-    result=$(cat $PGDEV | fgrep "Result: OK:")
-    if [ "$result" = "" ]; then
-         cat $PGDEV | fgrep Result: >/dev/stderr
-         exit 1
-    fi
+    COUNTER=0
+    while [  ${COUNTER} -lt ${TRXQUEUE} ]; do
+        result=$(cat $PGDEV${COUNTER} | fgrep "Result: OK:")
+        if [ "$result" = "" ]; then
+            cat $PGDEV${COUNTER} | fgrep Result: >/dev/stderr
+            exit 1
+        fi
+        let COUNTER=COUNTER+1
+    done
 
     # output result
     output_json