Add multiqueue support

Problem:
Neither OVS nor SRIOV multi-queue is not supported. Guest VM lacks tuning to reach high throughput.
Solution:
(1) Build SRIOV multi-queue capable guest image by recompiling the igxbevf driver (make CFLAGS_EXTRA=-DIXGBE_ENABLE_VF_MQ install).
(2) Change pktgen to send on multiple queues.
(3) Add tuning in guest VM (Disable irqbalance and setup vNIC interrupt affinity to vCPUs)

Update 1: Takes care comments plus adds unit test cases
Update 2: Jenkins reports code coverage 81%, local reports 96%, add more unit test cases
Update 3: Manually rebased to adapt to SSH.from_node()
Update 4: Takes care comments for vnic_type, vnic_name

Change-Id: Ieb15381c653b13697487d095efa4be6c3c49fa42
JIRA: YARDSTICK-619
Signed-off-by: Jing Zhang <jing.c.zhang@nokia.com>

2 files changed, 543 insertions, 0 deletions

diff --git a/tests/opnfv/test_cases/opnfv_yardstick_tc008.yaml b/tests/opnfv/test_cases/opnfv_yardstick_tc008.yaml
index 4c7fdab90..f5ccb255a 100644
--- a/tests/opnfv/test_cases/opnfv_yardstick_tc008.yaml
+++ b/tests/opnfv/test_cases/opnfv_yardstick_tc008.yaml
@@ -29,12 +29,21 @@ scenarios:
     packetsize: {{pkt_size}}
     number_of_ports: {{num_ports}}
     duration: 20
+    # choose vnic name: default to eth0
+    # vnic_name: 'ens3'
+    # turn on multiqueue inside VM
+    # multiqueue: True
+    # choose starting pps: default 1M;
+    # works with binary search runner Dynamictp to find max throughput per sla
+    # pps: 3000000
 
   host: demeter.yardstick-TC008
   target: poseidon.yardstick-TC008
 
   runner:
     type: Iteration
+    # binary search runner
+    # type: Dynamictp
     iterations: 10
     interval: 1
 
diff --git a/tests/unit/benchmark/scenarios/networking/test_pktgen.py b/tests/unit/benchmark/scenarios/networking/test_pktgen.py
index d4eb1246f..2914c8e02 100644
--- a/tests/unit/benchmark/scenarios/networking/test_pktgen.py
+++ b/tests/unit/benchmark/scenarios/networking/test_pktgen.py
@@ -138,6 +138,7 @@ class PktgenTestCase(unittest.TestCase):
         p.run(result)
         expected_result = jsonutils.loads(sample_output)
         expected_result["packets_received"] = 149300
+        expected_result["packetsize"] = 60
         self.assertEqual(result, expected_result)
 
     def test_pktgen_successful_sla(self, mock_ssh):
@@ -164,6 +165,7 @@ class PktgenTestCase(unittest.TestCase):
         p.run(result)
         expected_result = jsonutils.loads(sample_output)
         expected_result["packets_received"] = 149300
+        expected_result["packetsize"] = 60
         self.assertEqual(result, expected_result)
 
     def test_pktgen_unsuccessful_sla(self, mock_ssh):
@@ -204,6 +206,538 @@ class PktgenTestCase(unittest.TestCase):
         mock_ssh.SSH.from_node().execute.return_value = (1, '', 'FOOBAR')
         self.assertRaises(RuntimeError, p.run, result)
 
+    def test_pktgen_get_vnic_driver_name(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, 'ixgbevf', '')
+
+        vnic_driver_name = p._get_vnic_driver_name()
+        self.assertEqual(vnic_driver_name, 'ixgbevf')
+
+    def test_pktgen_unsuccessful_get_vnic_driver_name(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (1, '', '')
+
+        self.assertRaises(RuntimeError, p._get_vnic_driver_name)
+
+    def test_pktgen_get_sriov_queue_number(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, '2', '')
+
+        p.queue_number = p._get_sriov_queue_number()
+        self.assertEqual(p.queue_number, 2)
+
+    def test_pktgen_unsuccessful_get_sriov_queue_number(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (1, '', '')
+
+        self.assertRaises(RuntimeError, p._get_sriov_queue_number)
+
+    def test_pktgen_get_available_queue_number(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, '4', '')
+
+        p._get_available_queue_number()
+
+        mock_ssh.SSH.from_node().execute.assert_called_with(
+            "sudo ethtool -l eth0 | grep Combined | head -1 |" \
+            "awk '{printf $2}'")
+
+    def test_pktgen_unsuccessful_get_available_queue_number(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (1, '', '')
+
+        self.assertRaises(RuntimeError, p._get_available_queue_number)
+
+    def test_pktgen_get_usable_queue_number(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, '1', '')
+
+        p._get_usable_queue_number()
+
+        mock_ssh.SSH.from_node().execute.assert_called_with(
+            "sudo ethtool -l eth0 | grep Combined | tail -1 |" \
+            "awk '{printf $2}'")
+
+    def test_pktgen_unsuccessful_get_usable_queue_number(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (1, '', '')
+
+        self.assertRaises(RuntimeError, p._get_usable_queue_number)
+
+    def test_pktgen_enable_ovs_multiqueue(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, '4', '')
+
+        mock_result1 = mock.Mock()
+        mock_result1.return_value = 1
+        p._get_usable_queue_number = mock_result1
+
+        mock_result2 = mock.Mock()
+        mock_result2.return_value = 4
+        p._get_available_queue_number = mock_result2
+
+        p.queue_number = p._enable_ovs_multiqueue()
+        self.assertEqual(p.queue_number, 4)
+
+    def test_pktgen_enable_ovs_multiqueue_1q(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, '1', '')
+
+        mock_result1 = mock.Mock()
+        mock_result1.return_value = 1
+        p._get_usable_queue_number = mock_result1
+
+        mock_result2 = mock.Mock()
+        mock_result2.return_value = 1
+        p._get_available_queue_number = mock_result2
+
+        p.queue_number = p._enable_ovs_multiqueue()
+        self.assertEqual(p.queue_number, 1)
+
+    def test_pktgen_unsuccessful_enable_ovs_multiqueue(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (1, '', '')
+
+        mock_result1 = mock.Mock()
+        mock_result1.return_value = 1
+        p._get_usable_queue_number = mock_result1
+
+        mock_result2 = mock.Mock()
+        mock_result2.return_value = 4
+        p._get_available_queue_number = mock_result2
+
+        self.assertRaises(RuntimeError, p._enable_ovs_multiqueue)
+
+    def test_pktgen_setup_irqmapping_ovs(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, '10', '')
+
+        p._setup_irqmapping_ovs(4)
+
+        mock_ssh.SSH.from_node().execute.assert_called_with(
+            "echo 8 | sudo tee /proc/irq/10/smp_affinity")
+
+    def test_pktgen_setup_irqmapping_ovs_1q(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, '10', '')
+
+        p._setup_irqmapping_ovs(1)
+
+        mock_ssh.SSH.from_node().execute.assert_called_with(
+            "echo 1 | sudo tee /proc/irq/10/smp_affinity")
+
+    def test_pktgen_unsuccessful_setup_irqmapping_ovs(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (1, '', '')
+
+        self.assertRaises(RuntimeError, p._setup_irqmapping_ovs, 4)
+
+    def test_pktgen_unsuccessful_setup_irqmapping_ovs_1q(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (1, '', '')
+
+        self.assertRaises(RuntimeError, p._setup_irqmapping_ovs, 1)
+
+    def test_pktgen_setup_irqmapping_sriov(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, '10', '')
+
+        p._setup_irqmapping_sriov(2)
+
+        mock_ssh.SSH.from_node().execute.assert_called_with(
+            "echo 2 | sudo tee /proc/irq/10/smp_affinity")
+
+    def test_pktgen_setup_irqmapping_sriov_1q(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, '10', '')
+
+        p._setup_irqmapping_sriov(1)
+
+        mock_ssh.SSH.from_node().execute.assert_called_with(
+            "echo 1 | sudo tee /proc/irq/10/smp_affinity")
+
+    def test_pktgen_unsuccessful_setup_irqmapping_sriov(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (1, '', '')
+
+        self.assertRaises(RuntimeError, p._setup_irqmapping_sriov, 2)
+
+    def test_pktgen_unsuccessful_setup_irqmapping_sriov_1q(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (1, '', '')
+
+        self.assertRaises(RuntimeError, p._setup_irqmapping_sriov, 1)
+
+    def test_pktgen_is_irqbalance_disabled(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, '', '')
+
+        p._is_irqbalance_disabled()
+
+        mock_ssh.SSH.from_node().execute.assert_called_with(
+            "grep ENABLED /etc/default/irqbalance")
+
+    def test_pktgen_unsuccessful_is_irqbalance_disabled(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (1, '', '')
+
+        self.assertRaises(RuntimeError, p._is_irqbalance_disabled)
+
+    def test_pktgen_disable_irqbalance(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, '', '')
+
+        p._disable_irqbalance()
+
+        mock_ssh.SSH.from_node().execute.assert_called_with(
+            "sudo service irqbalance disable")
+
+    def test_pktgen_unsuccessful_disable_irqbalance(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (1, '', '')
+
+        self.assertRaises(RuntimeError, p._disable_irqbalance)
+
+    def test_pktgen_multiqueue_setup_ovs(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60, 'multiqueue': True},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, '4', '')
+
+        mock_result1 = mock.Mock()
+        mock_result1.return_value = False
+        p._is_irqbalance_disabled = mock_result1
+
+        mock_result2 = mock.Mock()
+        mock_result2.return_value = "virtio_net" 
+        p._get_vnic_driver_name = mock_result2
+
+        mock_result3 = mock.Mock()
+        mock_result3.return_value = 1
+        p._get_usable_queue_number = mock_result3
+
+        mock_result4 = mock.Mock()
+        mock_result4.return_value = 4
+        p._get_available_queue_number = mock_result4
+
+        p.multiqueue_setup()
+
+        self.assertEqual(p.queue_number, 4)
+
+    def test_pktgen_multiqueue_setup_ovs_1q(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60, 'multiqueue': True},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, '1', '')
+
+        mock_result1 = mock.Mock()
+        mock_result1.return_value = False
+        p._is_irqbalance_disabled = mock_result1
+
+        mock_result2 = mock.Mock()
+        mock_result2.return_value = "virtio_net" 
+        p._get_vnic_driver_name = mock_result2
+
+        mock_result3 = mock.Mock()
+        mock_result3.return_value = 1
+        p._get_usable_queue_number = mock_result3
+
+        mock_result4 = mock.Mock()
+        mock_result4.return_value = 1
+        p._get_available_queue_number = mock_result4
+
+        p.multiqueue_setup()
+
+        self.assertEqual(p.queue_number, 1)
+
+    def test_pktgen_multiqueue_setup_sriov(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60, 'multiqueue': True},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, '2', '')
+
+        mock_result1 = mock.Mock()
+        mock_result1.return_value = False
+        p._is_irqbalance_disabled = mock_result1
+
+        mock_result2 = mock.Mock()
+        mock_result2.return_value = "ixgbevf" 
+        p._get_vnic_driver_name = mock_result2
+
+        p.multiqueue_setup()
+
+        self.assertEqual(p.queue_number, 2)
+
+    def test_pktgen_multiqueue_setup_sriov_1q(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60, 'multiqueue': True},
+        }
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_ssh.SSH.from_node().execute.return_value = (0, '1', '')
+
+        mock_result1 = mock.Mock()
+        mock_result1.return_value = False
+        p._is_irqbalance_disabled = mock_result1
+
+        mock_result2 = mock.Mock()
+        mock_result2.return_value = "ixgbevf" 
+        p._get_vnic_driver_name = mock_result2
+
+        p.multiqueue_setup()
+
+        self.assertEqual(p.queue_number, 1)
+
+    def test_pktgen_run_with_setup_done(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60, 'number_of_ports': 10, 'duration': 20, 'multiqueue': True},
+            'sla': {'max_ppm': 1}
+        }
+        result = {}
+        p = pktgen.Pktgen(args, self.ctx)
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        p.setup_done = True
+        p.multiqueue_setup_done = True
+
+        mock_iptables_result = mock.Mock()
+        mock_iptables_result.return_value = 149300
+        p._iptables_get_result = mock_iptables_result
+
+        sample_output = '{"packets_per_second": 9753, "errors": 0, \
+            "packets_sent": 149300, "flows": 110}'
+        mock_ssh.SSH.from_node().execute.return_value = (0, sample_output, '')
+
+        p.run(result)
+        expected_result = jsonutils.loads(sample_output)
+        expected_result["packets_received"] = 149300
+        expected_result["packetsize"] = 60
+        self.assertEqual(result, expected_result)
+
+    def test_pktgen_run_with_ovs_multiqueque(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60, 'number_of_ports': 10, 'duration': 20, 'multiqueue': True},
+            'sla': {'max_ppm': 1}
+        }
+        result = {}
+
+        p = pktgen.Pktgen(args, self.ctx)
+
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_result = mock.Mock()
+        mock_result.return_value = "virtio_net" 
+        p._get_vnic_driver_name = mock_result
+
+        mock_result1 = mock.Mock()
+        mock_result1.return_value = 1
+        p._get_usable_queue_number = mock_result1
+
+        mock_result2 = mock.Mock()
+        mock_result2.return_value = 4
+        p._get_available_queue_number = mock_result2
+
+        mock_result3 = mock.Mock()
+        mock_result3.return_value = 4
+        p._enable_ovs_multiqueue  = mock_result3
+
+        mock_result4 = mock.Mock()
+        p._setup_irqmapping_ovs = mock_result4
+
+        mock_iptables_result = mock.Mock()
+        mock_iptables_result.return_value = 149300
+        p._iptables_get_result = mock_iptables_result
+
+        sample_output = '{"packets_per_second": 9753, "errors": 0, \
+            "packets_sent": 149300, "flows": 110}'
+        mock_ssh.SSH.from_node().execute.return_value = (0, sample_output, '')
+
+        p.run(result)
+        expected_result = jsonutils.loads(sample_output)
+        expected_result["packets_received"] = 149300
+        expected_result["packetsize"] = 60
+        self.assertEqual(result, expected_result)
+
+    def test_pktgen_run_with_sriov_multiqueque(self, mock_ssh):
+        args = {
+            'options': {'packetsize': 60, 'number_of_ports': 10, 'duration': 20, 'multiqueue': True},
+            'sla': {'max_ppm': 1}
+        }
+        result = {}
+
+        p = pktgen.Pktgen(args, self.ctx)
+
+        p.server = mock_ssh.SSH.from_node()
+        p.client = mock_ssh.SSH.from_node()
+
+        mock_result1 = mock.Mock()
+        mock_result1.return_value = "ixgbevf"
+        p._get_vnic_driver_name = mock_result1
+
+        mock_result2 = mock.Mock()
+        mock_result2.return_value = 2
+        p._get_sriov_queue_number = mock_result2
+
+        mock_result3 = mock.Mock()
+        p._setup_irqmapping_sriov = mock_result3
+
+        mock_iptables_result = mock.Mock()
+        mock_iptables_result.return_value = 149300
+        p._iptables_get_result = mock_iptables_result
+
+        sample_output = '{"packets_per_second": 9753, "errors": 0, \
+            "packets_sent": 149300, "flows": 110}'
+        mock_ssh.SSH.from_node().execute.return_value = (0, sample_output, '')
+
+        p.run(result)
+        expected_result = jsonutils.loads(sample_output)
+        expected_result["packets_received"] = 149300
+        expected_result["packetsize"] = 60
+        self.assertEqual(result, expected_result)
 
 def main():
     unittest.main()