diff options
4 files changed, 877 insertions, 47 deletions
diff --git a/tests/opnfv/test_cases/opnfv_yardstick_tc008.yaml b/tests/opnfv/test_cases/opnfv_yardstick_tc008.yaml index 4c7fdab90..f5ccb255a 100644 --- a/tests/opnfv/test_cases/opnfv_yardstick_tc008.yaml +++ b/tests/opnfv/test_cases/opnfv_yardstick_tc008.yaml @@ -29,12 +29,21 @@ scenarios: packetsize: {{pkt_size}} number_of_ports: {{num_ports}} duration: 20 + # choose vnic name: default to eth0 + # vnic_name: 'ens3' + # turn on multiqueue inside VM + # multiqueue: True + # choose starting pps: default 1M; + # works with binary search runner Dynamictp to find max throughput per sla + # pps: 3000000 host: demeter.yardstick-TC008 target: poseidon.yardstick-TC008 runner: type: Iteration + # binary search runner + # type: Dynamictp iterations: 10 interval: 1 diff --git a/tests/unit/benchmark/scenarios/networking/test_pktgen.py b/tests/unit/benchmark/scenarios/networking/test_pktgen.py index d4eb1246f..2914c8e02 100644 --- a/tests/unit/benchmark/scenarios/networking/test_pktgen.py +++ b/tests/unit/benchmark/scenarios/networking/test_pktgen.py @@ -138,6 +138,7 @@ class PktgenTestCase(unittest.TestCase): p.run(result) expected_result = jsonutils.loads(sample_output) expected_result["packets_received"] = 149300 + expected_result["packetsize"] = 60 self.assertEqual(result, expected_result) def test_pktgen_successful_sla(self, mock_ssh): @@ -164,6 +165,7 @@ class PktgenTestCase(unittest.TestCase): p.run(result) expected_result = jsonutils.loads(sample_output) expected_result["packets_received"] = 149300 + expected_result["packetsize"] = 60 self.assertEqual(result, expected_result) def test_pktgen_unsuccessful_sla(self, mock_ssh): @@ -204,6 +206,538 @@ class PktgenTestCase(unittest.TestCase): mock_ssh.SSH.from_node().execute.return_value = (1, '', 'FOOBAR') self.assertRaises(RuntimeError, p.run, result) + def test_pktgen_get_vnic_driver_name(self, mock_ssh): + args = { + 'options': {'packetsize': 60}, + } + p = pktgen.Pktgen(args, self.ctx) + p.server = mock_ssh.SSH.from_node() + + mock_ssh.SSH.from_node().execute.return_value = (0, 'ixgbevf', '') + + vnic_driver_name = p._get_vnic_driver_name() + self.assertEqual(vnic_driver_name, 'ixgbevf') + + def test_pktgen_unsuccessful_get_vnic_driver_name(self, mock_ssh): + args = { + 'options': {'packetsize': 60}, + } + p = pktgen.Pktgen(args, self.ctx) + p.server = mock_ssh.SSH.from_node() + + mock_ssh.SSH.from_node().execute.return_value = (1, '', '') + + self.assertRaises(RuntimeError, p._get_vnic_driver_name) + + def test_pktgen_get_sriov_queue_number(self, mock_ssh): + args = { + 'options': {'packetsize': 60}, + } + p = pktgen.Pktgen(args, self.ctx) + p.server = mock_ssh.SSH.from_node() + + mock_ssh.SSH.from_node().execute.return_value = (0, '2', '') + + p.queue_number = p._get_sriov_queue_number() + self.assertEqual(p.queue_number, 2) + + def test_pktgen_unsuccessful_get_sriov_queue_number(self, mock_ssh): + args = { + 'options': {'packetsize': 60}, + } + p = pktgen.Pktgen(args, self.ctx) + p.server = mock_ssh.SSH.from_node() + + mock_ssh.SSH.from_node().execute.return_value = (1, '', '') + + self.assertRaises(RuntimeError, p._get_sriov_queue_number) + + def test_pktgen_get_available_queue_number(self, mock_ssh): + args = { + 'options': {'packetsize': 60}, + } + p = pktgen.Pktgen(args, self.ctx) + p.server = mock_ssh.SSH.from_node() + + mock_ssh.SSH.from_node().execute.return_value = (0, '4', '') + + p._get_available_queue_number() + + mock_ssh.SSH.from_node().execute.assert_called_with( + "sudo ethtool -l eth0 | grep Combined | head -1 |" \ + "awk '{printf $2}'") + + def test_pktgen_unsuccessful_get_available_queue_number(self, mock_ssh): + args = { + 'options': {'packetsize': 60}, + } + p = pktgen.Pktgen(args, self.ctx) + p.server = mock_ssh.SSH.from_node() + + mock_ssh.SSH.from_node().execute.return_value = (1, '', '') + + self.assertRaises(RuntimeError, p._get_available_queue_number) + + def test_pktgen_get_usable_queue_number(self, mock_ssh): + args = { + 'options': {'packetsize': 60}, + } + p = pktgen.Pktgen(args, self.ctx) + p.server = mock_ssh.SSH.from_node() + + mock_ssh.SSH.from_node().execute.return_value = (0, '1', '') + + p._get_usable_queue_number() + + mock_ssh.SSH.from_node().execute.assert_called_with( + "sudo ethtool -l eth0 | grep Combined | tail -1 |" \ + "awk '{printf $2}'") + + def test_pktgen_unsuccessful_get_usable_queue_number(self, mock_ssh): + args = { + 'options': {'packetsize': 60}, + } + p = pktgen.Pktgen(args, self.ctx) + p.server = mock_ssh.SSH.from_node() + + mock_ssh.SSH.from_node().execute.return_value = (1, '', '') + + self.assertRaises(RuntimeError, p._get_usable_queue_number) + + def test_pktgen_enable_ovs_multiqueue(self, mock_ssh): + args = { + 'options': {'packetsize': 60}, + } + p = pktgen.Pktgen(args, self.ctx) + p.server = mock_ssh.SSH.from_node() + p.client = mock_ssh.SSH.from_node() + + mock_ssh.SSH.from_node().execute.return_value = (0, '4', '') + + mock_result1 = mock.Mock() + mock_result1.return_value = 1 + p._get_usable_queue_number = mock_result1 + + mock_result2 = mock.Mock() + mock_result2.return_value = 4 + p._get_available_queue_number = mock_result2 + + p.queue_number = p._enable_ovs_multiqueue() + self.assertEqual(p.queue_number, 4) + + def test_pktgen_enable_ovs_multiqueue_1q(self, mock_ssh): + args = { + 'options': {'packetsize': 60}, + } + p = pktgen.Pktgen(args, self.ctx) + p.server = mock_ssh.SSH.from_node() + p.client = mock_ssh.SSH.from_node() + + mock_ssh.SSH.from_node().execute.return_value = (0, '1', '') + + mock_result1 = mock.Mock() + mock_result1.return_value = 1 + p._get_usable_queue_number = mock_result1 + + mock_result2 = mock.Mock() + mock_result2.return_value = 1 + p._get_available_queue_number = mock_result2 + + p.queue_number = p._enable_ovs_multiqueue() + self.assertEqual(p.queue_number, 1) + + def test_pktgen_unsuccessful_enable_ovs_multiqueue(self, mock_ssh): + args = { + 'options': {'packetsize': 60}, + } + p = pktgen.Pktgen(args, self.ctx) + p.server = mock_ssh.SSH.from_node() + p.client = mock_ssh.SSH.from_node() + + mock_ssh.SSH.from_node().execute.return_value = (1, '', '') + + mock_result1 = mock.Mock() + mock_result1.return_value = 1 + p._get_usable_queue_number = mock_result1 + + mock_result2 = mock.Mock() + mock_result2.return_value = 4 + p._get_available_queue_number = mock_result2 + + self.assertRaises(RuntimeError, p._enable_ovs_multiqueue) + + def test_pktgen_setup_irqmapping_ovs(self, mock_ssh): + args = { + 'options': {'packetsize': 60}, + } + p = pktgen.Pktgen(args, self.ctx) + p.server = mock_ssh.SSH.from_node() + p.client = mock_ssh.SSH.from_node() + + mock_ssh.SSH.from_node().execute.return_value = (0, '10', '') + + p._setup_irqmapping_ovs(4) + + mock_ssh.SSH.from_node().execute.assert_called_with( + "echo 8 | sudo tee /proc/irq/10/smp_affinity") + + def test_pktgen_setup_irqmapping_ovs_1q(self, mock_ssh): + args = { + 'options': {'packetsize': 60}, + } + p = pktgen.Pktgen(args, self.ctx) + p.server = mock_ssh.SSH.from_node() + p.client = mock_ssh.SSH.from_node() + + mock_ssh.SSH.from_node().execute.return_value = (0, '10', '') + + p._setup_irqmapping_ovs(1) + + mock_ssh.SSH.from_node().execute.assert_called_with( + "echo 1 | sudo tee /proc/irq/10/smp_affinity") + + def test_pktgen_unsuccessful_setup_irqmapping_ovs(self, mock_ssh): + args = { + 'options': {'packetsize': 60}, + } + p = pktgen.Pktgen(args, self.ctx) + p.server = mock_ssh.SSH.from_node() + p.client = mock_ssh.SSH.from_node() + + mock_ssh.SSH.from_node().execute.return_value = (1, '', '') + + self.assertRaises(RuntimeError, p._setup_irqmapping_ovs, 4) + + def test_pktgen_unsuccessful_setup_irqmapping_ovs_1q(self, mock_ssh): + args = { + 'options': {'packetsize': 60}, + } + p = pktgen.Pktgen(args, self.ctx) + p.server = mock_ssh.SSH.from_node() + p.client = mock_ssh.SSH.from_node() + + mock_ssh.SSH.from_node().execute.return_value = (1, '', '') + + self.assertRaises(RuntimeError, p._setup_irqmapping_ovs, 1) + + def test_pktgen_setup_irqmapping_sriov(self, mock_ssh): + args = { + 'options': {'packetsize': 60}, + } + p = pktgen.Pktgen(args, self.ctx) + p.server = mock_ssh.SSH.from_node() + p.client = mock_ssh.SSH.from_node() + + mock_ssh.SSH.from_node().execute.return_value = (0, '10', '') + + p._setup_irqmapping_sriov(2) + + mock_ssh.SSH.from_node().execute.assert_called_with( + "echo 2 | sudo tee /proc/irq/10/smp_affinity") + + def test_pktgen_setup_irqmapping_sriov_1q(self, mock_ssh): + args = { + 'options': {'packetsize': 60}, + } + p = pktgen.Pktgen(args, self.ctx) + p.server = mock_ssh.SSH.from_node() + p.client = mock_ssh.SSH.from_node() + + mock_ssh.SSH.from_node().execute.return_value = (0, '10', '') + + p._setup_irqmapping_sriov(1) + + mock_ssh.SSH.from_node().execute.assert_called_with( + "echo 1 | sudo tee /proc/irq/10/smp_affinity") + + def test_pktgen_unsuccessful_setup_irqmapping_sriov(self, mock_ssh): + args = { + 'options': {'packetsize': 60}, + } + p = pktgen.Pktgen(args, self.ctx) + p.server = mock_ssh.SSH.from_node() + p.client = mock_ssh.SSH.from_node() + + mock_ssh.SSH.from_node().execute.return_value = (1, '', '') + + self.assertRaises(RuntimeError, p._setup_irqmapping_sriov, 2) + + def test_pktgen_unsuccessful_setup_irqmapping_sriov_1q(self, mock_ssh): + args = { + 'options': {'packetsize': 60}, + } + p = pktgen.Pktgen(args, self.ctx) + p.server = mock_ssh.SSH.from_node() + p.client = mock_ssh.SSH.from_node() + + mock_ssh.SSH.from_node().execute.return_value = (1, '', '') + + self.assertRaises(RuntimeError, p._setup_irqmapping_sriov, 1) + + def test_pktgen_is_irqbalance_disabled(self, mock_ssh): + args = { + 'options': {'packetsize': 60}, + } + p = pktgen.Pktgen(args, self.ctx) + p.server = mock_ssh.SSH.from_node() + + mock_ssh.SSH.from_node().execute.return_value = (0, '', '') + + p._is_irqbalance_disabled() + + mock_ssh.SSH.from_node().execute.assert_called_with( + "grep ENABLED /etc/default/irqbalance") + + def test_pktgen_unsuccessful_is_irqbalance_disabled(self, mock_ssh): + args = { + 'options': {'packetsize': 60}, + } + p = pktgen.Pktgen(args, self.ctx) + p.server = mock_ssh.SSH.from_node() + + mock_ssh.SSH.from_node().execute.return_value = (1, '', '') + + self.assertRaises(RuntimeError, p._is_irqbalance_disabled) + + def test_pktgen_disable_irqbalance(self, mock_ssh): + args = { + 'options': {'packetsize': 60}, + } + p = pktgen.Pktgen(args, self.ctx) + p.server = mock_ssh.SSH.from_node() + p.client = mock_ssh.SSH.from_node() + + mock_ssh.SSH.from_node().execute.return_value = (0, '', '') + + p._disable_irqbalance() + + mock_ssh.SSH.from_node().execute.assert_called_with( + "sudo service irqbalance disable") + + def test_pktgen_unsuccessful_disable_irqbalance(self, mock_ssh): + args = { + 'options': {'packetsize': 60}, + } + p = pktgen.Pktgen(args, self.ctx) + p.server = mock_ssh.SSH.from_node() + p.client = mock_ssh.SSH.from_node() + + mock_ssh.SSH.from_node().execute.return_value = (1, '', '') + + self.assertRaises(RuntimeError, p._disable_irqbalance) + + def test_pktgen_multiqueue_setup_ovs(self, mock_ssh): + args = { + 'options': {'packetsize': 60, 'multiqueue': True}, + } + p = pktgen.Pktgen(args, self.ctx) + p.server = mock_ssh.SSH.from_node() + p.client = mock_ssh.SSH.from_node() + + mock_ssh.SSH.from_node().execute.return_value = (0, '4', '') + + mock_result1 = mock.Mock() + mock_result1.return_value = False + p._is_irqbalance_disabled = mock_result1 + + mock_result2 = mock.Mock() + mock_result2.return_value = "virtio_net" + p._get_vnic_driver_name = mock_result2 + + mock_result3 = mock.Mock() + mock_result3.return_value = 1 + p._get_usable_queue_number = mock_result3 + + mock_result4 = mock.Mock() + mock_result4.return_value = 4 + p._get_available_queue_number = mock_result4 + + p.multiqueue_setup() + + self.assertEqual(p.queue_number, 4) + + def test_pktgen_multiqueue_setup_ovs_1q(self, mock_ssh): + args = { + 'options': {'packetsize': 60, 'multiqueue': True}, + } + p = pktgen.Pktgen(args, self.ctx) + p.server = mock_ssh.SSH.from_node() + p.client = mock_ssh.SSH.from_node() + + mock_ssh.SSH.from_node().execute.return_value = (0, '1', '') + + mock_result1 = mock.Mock() + mock_result1.return_value = False + p._is_irqbalance_disabled = mock_result1 + + mock_result2 = mock.Mock() + mock_result2.return_value = "virtio_net" + p._get_vnic_driver_name = mock_result2 + + mock_result3 = mock.Mock() + mock_result3.return_value = 1 + p._get_usable_queue_number = mock_result3 + + mock_result4 = mock.Mock() + mock_result4.return_value = 1 + p._get_available_queue_number = mock_result4 + + p.multiqueue_setup() + + self.assertEqual(p.queue_number, 1) + + def test_pktgen_multiqueue_setup_sriov(self, mock_ssh): + args = { + 'options': {'packetsize': 60, 'multiqueue': True}, + } + p = pktgen.Pktgen(args, self.ctx) + p.server = mock_ssh.SSH.from_node() + p.client = mock_ssh.SSH.from_node() + + mock_ssh.SSH.from_node().execute.return_value = (0, '2', '') + + mock_result1 = mock.Mock() + mock_result1.return_value = False + p._is_irqbalance_disabled = mock_result1 + + mock_result2 = mock.Mock() + mock_result2.return_value = "ixgbevf" + p._get_vnic_driver_name = mock_result2 + + p.multiqueue_setup() + + self.assertEqual(p.queue_number, 2) + + def test_pktgen_multiqueue_setup_sriov_1q(self, mock_ssh): + args = { + 'options': {'packetsize': 60, 'multiqueue': True}, + } + p = pktgen.Pktgen(args, self.ctx) + p.server = mock_ssh.SSH.from_node() + p.client = mock_ssh.SSH.from_node() + + mock_ssh.SSH.from_node().execute.return_value = (0, '1', '') + + mock_result1 = mock.Mock() + mock_result1.return_value = False + p._is_irqbalance_disabled = mock_result1 + + mock_result2 = mock.Mock() + mock_result2.return_value = "ixgbevf" + p._get_vnic_driver_name = mock_result2 + + p.multiqueue_setup() + + self.assertEqual(p.queue_number, 1) + + def test_pktgen_run_with_setup_done(self, mock_ssh): + args = { + 'options': {'packetsize': 60, 'number_of_ports': 10, 'duration': 20, 'multiqueue': True}, + 'sla': {'max_ppm': 1} + } + result = {} + p = pktgen.Pktgen(args, self.ctx) + p.server = mock_ssh.SSH.from_node() + p.client = mock_ssh.SSH.from_node() + + p.setup_done = True + p.multiqueue_setup_done = True + + mock_iptables_result = mock.Mock() + mock_iptables_result.return_value = 149300 + p._iptables_get_result = mock_iptables_result + + sample_output = '{"packets_per_second": 9753, "errors": 0, \ + "packets_sent": 149300, "flows": 110}' + mock_ssh.SSH.from_node().execute.return_value = (0, sample_output, '') + + p.run(result) + expected_result = jsonutils.loads(sample_output) + expected_result["packets_received"] = 149300 + expected_result["packetsize"] = 60 + self.assertEqual(result, expected_result) + + def test_pktgen_run_with_ovs_multiqueque(self, mock_ssh): + args = { + 'options': {'packetsize': 60, 'number_of_ports': 10, 'duration': 20, 'multiqueue': True}, + 'sla': {'max_ppm': 1} + } + result = {} + + p = pktgen.Pktgen(args, self.ctx) + + p.server = mock_ssh.SSH.from_node() + p.client = mock_ssh.SSH.from_node() + + mock_result = mock.Mock() + mock_result.return_value = "virtio_net" + p._get_vnic_driver_name = mock_result + + mock_result1 = mock.Mock() + mock_result1.return_value = 1 + p._get_usable_queue_number = mock_result1 + + mock_result2 = mock.Mock() + mock_result2.return_value = 4 + p._get_available_queue_number = mock_result2 + + mock_result3 = mock.Mock() + mock_result3.return_value = 4 + p._enable_ovs_multiqueue = mock_result3 + + mock_result4 = mock.Mock() + p._setup_irqmapping_ovs = mock_result4 + + mock_iptables_result = mock.Mock() + mock_iptables_result.return_value = 149300 + p._iptables_get_result = mock_iptables_result + + sample_output = '{"packets_per_second": 9753, "errors": 0, \ + "packets_sent": 149300, "flows": 110}' + mock_ssh.SSH.from_node().execute.return_value = (0, sample_output, '') + + p.run(result) + expected_result = jsonutils.loads(sample_output) + expected_result["packets_received"] = 149300 + expected_result["packetsize"] = 60 + self.assertEqual(result, expected_result) + + def test_pktgen_run_with_sriov_multiqueque(self, mock_ssh): + args = { + 'options': {'packetsize': 60, 'number_of_ports': 10, 'duration': 20, 'multiqueue': True}, + 'sla': {'max_ppm': 1} + } + result = {} + + p = pktgen.Pktgen(args, self.ctx) + + p.server = mock_ssh.SSH.from_node() + p.client = mock_ssh.SSH.from_node() + + mock_result1 = mock.Mock() + mock_result1.return_value = "ixgbevf" + p._get_vnic_driver_name = mock_result1 + + mock_result2 = mock.Mock() + mock_result2.return_value = 2 + p._get_sriov_queue_number = mock_result2 + + mock_result3 = mock.Mock() + p._setup_irqmapping_sriov = mock_result3 + + mock_iptables_result = mock.Mock() + mock_iptables_result.return_value = 149300 + p._iptables_get_result = mock_iptables_result + + sample_output = '{"packets_per_second": 9753, "errors": 0, \ + "packets_sent": 149300, "flows": 110}' + mock_ssh.SSH.from_node().execute.return_value = (0, sample_output, '') + + p.run(result) + expected_result = jsonutils.loads(sample_output) + expected_result["packets_received"] = 149300 + expected_result["packetsize"] = 60 + self.assertEqual(result, expected_result) def main(): unittest.main() diff --git a/yardstick/benchmark/scenarios/networking/pktgen.py b/yardstick/benchmark/scenarios/networking/pktgen.py index e6aa7e5fb..8ca1ca60e 100644 --- a/yardstick/benchmark/scenarios/networking/pktgen.py +++ b/yardstick/benchmark/scenarios/networking/pktgen.py @@ -9,6 +9,7 @@ from __future__ import absolute_import from __future__ import print_function +import os import logging import pkg_resources @@ -19,6 +20,9 @@ from yardstick.benchmark.scenarios import base LOG = logging.getLogger(__name__) +VNIC_TYPE_LIST = ["ovs", "sriov"] +SRIOV_DRIVER_LIST = ["ixgbevf", "i40evf"] + class Pktgen(base.Scenario): """Execute pktgen between two hosts @@ -44,7 +48,11 @@ class Pktgen(base.Scenario): def __init__(self, scenario_cfg, context_cfg): self.scenario_cfg = scenario_cfg self.context_cfg = context_cfg + self.vnic_name = "eth0" + self.vnic_type = "ovs" + self.queue_number = 1 self.setup_done = False + self.multiqueue_setup_done = False def setup(self): """scenario setup""" @@ -67,6 +75,212 @@ class Pktgen(base.Scenario): self.setup_done = True + def multiqueue_setup(self): + # one time setup stuff + cmd = "sudo sysctl -w net.core.netdev_budget=3000" + self.server.send_command(cmd) + self.client.send_command(cmd) + + cmd = "sudo sysctl -w net.core.netdev_max_backlog=100000" + self.server.send_command(cmd) + self.client.send_command(cmd) + + """multiqueue setup""" + if not self._is_irqbalance_disabled(): + self._disable_irqbalance() + + vnic_driver_name = self._get_vnic_driver_name() + if vnic_driver_name in SRIOV_DRIVER_LIST: + self.vnic_type = "sriov" + + # one time setup stuff + cmd = "sudo ethtool -G %s rx 4096 tx 4096" % self.vnic_name + self.server.send_command(cmd) + self.client.send_command(cmd) + + self.queue_number = self._get_sriov_queue_number() + self._setup_irqmapping_sriov(self.queue_number) + else: + self.vnic_type = "ovs" + self.queue_number = self._enable_ovs_multiqueue() + self._setup_irqmapping_ovs(self.queue_number) + + self.multiqueue_setup_done = True + + def _get_vnic_driver_name(self): + cmd = "readlink /sys/class/net/%s/device/driver" % self.vnic_name + LOG.debug("Executing command: %s", cmd) + status, stdout, stderr = self.server.execute(cmd) + if status: + raise RuntimeError(stderr) + return os.path.basename(stdout.strip()) + + def _is_irqbalance_disabled(self): + """Did we disable irqbalance already in the guest?""" + is_disabled = False + cmd = "grep ENABLED /etc/default/irqbalance" + status, stdout, stderr = self.server.execute(cmd) + if status: + raise RuntimeError(stderr) + if "0" in stdout: + is_disabled = True + + return is_disabled + + def _disable_irqbalance(self): + cmd = "sudo sed -i -e 's/ENABLED=\"1\"/ENABLED=\"0\"/g' " \ + "/etc/default/irqbalance" + status, stdout, stderr = self.server.execute(cmd) + status, stdout, stderr = self.client.execute(cmd) + if status: + raise RuntimeError(stderr) + + cmd = "sudo service irqbalance stop" + status, stdout, stderr = self.server.execute(cmd) + status, stdout, stderr = self.client.execute(cmd) + if status: + raise RuntimeError(stderr) + + cmd = "sudo service irqbalance disable" + status, stdout, stderr = self.server.execute(cmd) + status, stdout, stderr = self.client.execute(cmd) + if status: + raise RuntimeError(stderr) + + def _setup_irqmapping_ovs(self, queue_number): + cmd = "grep 'virtio0-input.0' /proc/interrupts |" \ + "awk '{match($0,/ +[0-9]+/)} " \ + "{print substr($1,RSTART,RLENGTH-1)}'" + status, stdout, stderr = self.server.execute(cmd) + if status: + raise RuntimeError(stderr) + + cmd = "echo 1 | sudo tee /proc/irq/%s/smp_affinity" % (int(stdout)) + status, stdout, stderr = self.server.execute(cmd) + status, stdout, stderr = self.client.execute(cmd) + if status: + raise RuntimeError(stderr) + + cmd = "grep 'virtio0-output.0' /proc/interrupts |" \ + "awk '{match($0,/ +[0-9]+/)} " \ + "{print substr($1,RSTART,RLENGTH-1)}'" + status, stdout, stderr = self.server.execute(cmd) + if status: + raise RuntimeError(stderr) + + cmd = "echo 1 | sudo tee /proc/irq/%s/smp_affinity" % (int(stdout)) + status, stdout, stderr = self.server.execute(cmd) + status, stdout, stderr = self.client.execute(cmd) + if status: + raise RuntimeError(stderr) + + if queue_number == 1: + return + + for i in range(1, queue_number): + cmd = "grep 'virtio0-input.%s' /proc/interrupts |" \ + "awk '{match($0,/ +[0-9]+/)} " \ + "{print substr($1,RSTART,RLENGTH-1)}'" % (i) + status, stdout, stderr = self.server.execute(cmd) + if status: + raise RuntimeError(stderr) + + cmd = "echo %s | sudo tee /proc/irq/%s/smp_affinity" \ + % (1 << i, int(stdout)) + status, stdout, stderr = self.server.execute(cmd) + status, stdout, stderr = self.client.execute(cmd) + if status: + raise RuntimeError(stderr) + + cmd = "grep 'virtio0-output.%s' /proc/interrupts |" \ + "awk '{match($0,/ +[0-9]+/)} " \ + "{print substr($1,RSTART,RLENGTH-1)}'" % (i) + status, stdout, stderr = self.server.execute(cmd) + if status: + raise RuntimeError(stderr) + + cmd = "echo %s | sudo tee /proc/irq/%s/smp_affinity" \ + % (1 << i, int(stdout)) + status, stdout, stderr = self.server.execute(cmd) + status, stdout, stderr = self.client.execute(cmd) + if status: + raise RuntimeError(stderr) + + def _setup_irqmapping_sriov(self, queue_number): + cmd = "grep '%s-TxRx-0' /proc/interrupts |" \ + "awk '{match($0,/ +[0-9]+/)} " \ + "{print substr($1,RSTART,RLENGTH-1)}'" % self.vnic_name + status, stdout, stderr = self.server.execute(cmd) + if status: + raise RuntimeError(stderr) + + cmd = "echo 1 | sudo tee /proc/irq/%s/smp_affinity" % (int(stdout)) + status, stdout, stderr = self.server.execute(cmd) + status, stdout, stderr = self.client.execute(cmd) + if status: + raise RuntimeError(stderr) + + if queue_number == 1: + return + + for i in range(1, queue_number): + cmd = "grep '%s-TxRx-%s' /proc/interrupts |" \ + "awk '{match($0,/ +[0-9]+/)} " \ + "{print substr($1,RSTART,RLENGTH-1)}'" % (self.vnic_name, i) + status, stdout, stderr = self.server.execute(cmd) + if status: + raise RuntimeError(stderr) + + cmd = "echo %s | sudo tee /proc/irq/%s/smp_affinity" \ + % (1 << i, int(stdout)) + status, stdout, stderr = self.server.execute(cmd) + status, stdout, stderr = self.client.execute(cmd) + if status: + raise RuntimeError(stderr) + + def _get_sriov_queue_number(self): + """Get queue number from server as both VMs are the same""" + cmd = "grep %s-TxRx- /proc/interrupts | wc -l" % self.vnic_name + LOG.debug("Executing command: %s", cmd) + status, stdout, stderr = self.server.execute(cmd) + if status: + raise RuntimeError(stderr) + return int(stdout) + + def _get_available_queue_number(self): + """Get queue number from client as both VMs are the same""" + cmd = "sudo ethtool -l %s | grep Combined | head -1 |" \ + "awk '{printf $2}'" % self.vnic_name + LOG.debug("Executing command: %s", cmd) + status, stdout, stderr = self.server.execute(cmd) + if status: + raise RuntimeError(stderr) + return int(stdout) + + def _get_usable_queue_number(self): + """Get queue number from client as both VMs are the same""" + cmd = "sudo ethtool -l %s | grep Combined | tail -1 |" \ + "awk '{printf $2}'" % self.vnic_name + LOG.debug("Executing command: %s", cmd) + status, stdout, stderr = self.server.execute(cmd) + if status: + raise RuntimeError(stderr) + return int(stdout) + + def _enable_ovs_multiqueue(self): + available_queue_number = self._get_available_queue_number() + usable_queue_number = self._get_usable_queue_number() + if available_queue_number > 1 and \ + available_queue_number != usable_queue_number: + cmd = "sudo ethtool -L %s combined %s" % \ + (self.vnic_name, available_queue_number) + LOG.debug("Executing command: %s", cmd) + status, stdout, stderr = self.server.execute(cmd) + status, stdout, stderr = self.client.execute(cmd) + if status: + raise RuntimeError(stderr) + return available_queue_number + def _iptables_setup(self): """Setup iptables on server to monitor for received packets""" cmd = "sudo iptables -F; " \ @@ -99,6 +313,14 @@ class Pktgen(base.Scenario): options = self.scenario_cfg['options'] packetsize = options.get("packetsize", 60) self.number_of_ports = options.get("number_of_ports", 10) + self.vnic_name = options.get("vnic_name", "eth0") + ovs_dpdk = options.get("ovs_dpdk", False) + pps = options.get("pps", 1000000) + multiqueue = options.get("multiqueue", False) + + if multiqueue and not self.multiqueue_setup_done: + self.multiqueue_setup() + # if run by a duration runner duration_time = self.scenario_cfg["runner"].get("duration", None) \ if "runner" in self.scenario_cfg else None @@ -114,8 +336,18 @@ class Pktgen(base.Scenario): self._iptables_setup() - cmd = "sudo bash pktgen.sh %s %s %s %s" \ - % (ipaddr, self.number_of_ports, packetsize, duration) + queue_number = self.queue_number + + # For native OVS, half of vCPUs are used by vhost kernel threads + # hence set the queue_number to half number of vCPUs + # e.g. set queue_number to 2 if there are 4 vCPUs + if self.vnic_type == "ovs" and not ovs_dpdk and self.queue_number > 1: + queue_number = self.queue_number / 2 + + cmd = "sudo bash pktgen.sh %s %s %s %s %s %s" \ + % (ipaddr, self.number_of_ports, packetsize, + duration, queue_number, pps) + LOG.debug("Executing command: %s", cmd) status, stdout, stderr = self.client.execute(cmd) @@ -131,12 +363,15 @@ class Pktgen(base.Scenario): sent = result['packets_sent'] received = result['packets_received'] ppm = 1000000 * (sent - received) / sent + # if ppm is 1, then 11 out of 10 million is no pass + ppm += (sent - received) % sent > 0 + LOG.debug("Lost packets %d - Lost ppm %d", (sent - received), ppm) sla_max_ppm = int(self.scenario_cfg["sla"]["max_ppm"]) assert ppm <= sla_max_ppm, "ppm %d > sla_max_ppm %d; " \ % (ppm, sla_max_ppm) -def _test(): +def _test(): # pragma: no cover """internal test function""" key_filename = pkg_resources.resource_filename('yardstick.resources', 'files/yardstick_key') @@ -165,6 +400,5 @@ def _test(): p.run(result) print(result) - if __name__ == '__main__': _test() diff --git a/yardstick/benchmark/scenarios/networking/pktgen_benchmark.bash b/yardstick/benchmark/scenarios/networking/pktgen_benchmark.bash index 4224c5abf..e338a1b09 100644 --- a/yardstick/benchmark/scenarios/networking/pktgen_benchmark.bash +++ b/yardstick/benchmark/scenarios/networking/pktgen_benchmark.bash @@ -16,6 +16,8 @@ DST_IP=$1 # destination IP address NUM_PORTS=$2 # number of source ports PKT_SIZE=$3 # packet size DURATION=$4 # test duration (seconds) +TRXQUEUE=$5 # number of RX/TX queues to use +PPS=$6 # packets per second to send # Configuration UDP_SRC_MIN=1000 # UDP source port min @@ -37,62 +39,100 @@ pgset() fi } +# remove all devices from thread +pgclean() +{ + COUNTER=0 + while [ ${COUNTER} -lt ${TRXQUEUE} ]; do + # + # Thread commands + # + + PGDEV=/proc/net/pktgen/kpktgend_${COUNTER} + + # Remove all devices from this thread + pgset "rem_device_all" + let COUNTER=COUNTER+1 + done +} + # configure pktgen (see pktgen doc for details) pgconfig() { - # - # Thread commands - # + pps=$(( PPS / TRXQUEUE )) + COUNTER=0 + while [ ${COUNTER} -lt ${TRXQUEUE} ]; do + # + # Thread commands + # - PGDEV=/proc/net/pktgen/kpktgend_0 + PGDEV=/proc/net/pktgen/kpktgend_${COUNTER} - # Remove all devices from this thread - pgset "rem_device_all" + # Add device to thread + pgset "add_device $DEV@${COUNTER}" - # Add device to thread - pgset "add_device $DEV" + # + # Device commands + # - # - # Device commands - # + PGDEV=/proc/net/pktgen/$DEV@${COUNTER} - PGDEV=/proc/net/pktgen/$DEV + # 0 means continious sends untill explicitly stopped + pgset "count 0" - # 0 means continious sends untill explicitly stopped - pgset "count 0" + # set pps count to test with an explicit number. if 0 will try with bandwidth + if [ ${pps} -gt 0 ] + then + pgset "ratep ${pps}" + fi - # use single SKB for all transmits - pgset "clone_skb 0" + pgset "clone_skb 10" - # packet size, NIC adds 4 bytes CRC - pgset "pkt_size $PKT_SIZE" + # use different queue per thread + pgset "queue_map_min ${COUNTER}" + pgset "queue_map_max ${COUNTER}" - # random address within the min-max range - pgset "flag IPDST_RND UDPSRC_RND UDPDST_RND" + # packet size, NIC adds 4 bytes CRC + pgset "pkt_size $PKT_SIZE" - # destination IP - pgset "dst_min $DST_IP" - pgset "dst_max $DST_IP" + # random address within the min-max range + pgset "flag UDPDST_RND" + pgset "flag UDPSRC_RND" + pgset "flag IPDST_RND" - # destination MAC address - pgset "dst_mac $MAC" + # destination IP + pgset "dst_min $DST_IP" + pgset "dst_max $DST_IP" + + # destination MAC address + pgset "dst_mac $MAC" + + # source UDP port range + pgset "udp_src_min $UDP_SRC_MIN" + pgset "udp_src_max $UDP_SRC_MAX" - # source UDP port range - pgset "udp_src_min $UDP_SRC_MIN" - pgset "udp_src_max $UDP_SRC_MAX" + # destination UDP port range + pgset "udp_dst_min $UDP_DST_MIN" + pgset "udp_dst_max $UDP_DST_MAX" - # destination UDP port range - pgset "udp_dst_min $UDP_DST_MIN" - pgset "udp_dst_max $UDP_DST_MAX" + let COUNTER=COUNTER+1 + + done } # run pktgen pgrun() { - # Time to run, result can be vieved in /proc/net/pktgen/$DEV + # Time to run, result can be viewed in /proc/net/pktgen/$DEV PGDEV=/proc/net/pktgen/pgctrl # Will hang, Ctrl-C or SIGINT to stop pgset "start" start + + COUNTER=0 + while [ ${COUNTER} -lt ${TRXQUEUE} ]; do + taskset -c ${COUNTER} kpktgend_${COUNTER} + let COUNTER=COUNTER+1 + done } # run pktgen for ${DURATION} seconds @@ -111,19 +151,28 @@ run_test() # write the result to stdout in json format output_json() { - sent=$(awk '/^Result:/{print $5}' <$PGDEV) - pps=$(awk 'match($0,/'\([0-9]+\)pps'/, a) {print a[1]}' <$PGDEV) - errors=$(awk '/errors:/{print $5}' <$PGDEV) + sent=0 + result_pps=0 + errors=0 + PGDEV=/proc/net/pktgen/$DEV@ + COUNTER=0 + while [ ${COUNTER} -lt ${TRXQUEUE} ]; do + sent=$(($sent + $(awk '/^Result:/{print $5}' <$PGDEV${COUNTER}))) + result_pps=$(($result_pps + $(awk 'match($0,/'\([0-9]+\)pps'/, a) {print a[1]}' <$PGDEV${COUNTER}))) + errors=$(($errors + $(awk '/errors:/{print $5}' <$PGDEV${COUNTER}))) + let COUNTER=COUNTER+1 + done flows=$(( NUM_PORTS * (NUM_PORTS + 1) )) - echo { '"packets_sent"':$sent , '"packets_per_second"':$pps, '"flows"':$flows, '"errors"':$errors } + echo '{ "packets_sent"':${sent} , '"packets_per_second"':${result_pps}, '"flows"':${flows}, '"errors"':${errors} '}' } # main entry main() { modprobe pktgen + pgclean ping -c 3 $DST_IP >/dev/null @@ -137,16 +186,20 @@ main() pgconfig # run the test - run_test >/dev/null + run_test - PGDEV=/proc/net/pktgen/$DEV + PGDEV=/proc/net/pktgen/$DEV@ # check result - result=$(cat $PGDEV | fgrep "Result: OK:") - if [ "$result" = "" ]; then - cat $PGDEV | fgrep Result: >/dev/stderr - exit 1 - fi + COUNTER=0 + while [ ${COUNTER} -lt ${TRXQUEUE} ]; do + result=$(cat $PGDEV${COUNTER} | fgrep "Result: OK:") + if [ "$result" = "" ]; then + cat $PGDEV${COUNTER} | fgrep Result: >/dev/stderr + exit 1 + fi + let COUNTER=COUNTER+1 + done # output result output_json |