From 312a77a1ac353c1a135ac0b901863bbe8ee66f20 Mon Sep 17 00:00:00 2001 From: Karthik S Date: Wed, 5 Jul 2017 06:52:36 -0400 Subject: OvS 2.7 support - dpdk-devargs needs to provided for DPDK devices The pci address of the DPDK NIC needs to be specified as dpdk-devargs when adding the DPDK port. This is required for using DPDK with OvS 2.7 Change-Id: I4975130961199ee04dd002ec987081299e3ddd4a Closes-Bug: #1702457 Signed-off-by: Karthik S --- os_net_config/impl_ifcfg.py | 18 +++++++++ os_net_config/tests/test_impl_ifcfg.py | 38 ++++++++++++++++-- os_net_config/tests/test_utils.py | 72 +++++++++++++++++++++++++++++++++- os_net_config/utils.py | 19 +++++++++ 4 files changed, 141 insertions(+), 6 deletions(-) diff --git a/os_net_config/impl_ifcfg.py b/os_net_config/impl_ifcfg.py index f065f63..3a82597 100644 --- a/os_net_config/impl_ifcfg.py +++ b/os_net_config/impl_ifcfg.py @@ -299,6 +299,14 @@ class IfcfgNetConfig(os_net_config.NetConfig): data += "DEVICETYPE=ovs\n" data += "TYPE=OVSDPDKPort\n" data += "OVS_BRIDGE=%s\n" % base_opt.bridge_name + # Validation of DPDK port having only one interface is done prior + # to this. So accesing the interface name statically. + # Also pci_address would be valid here, since + # bind_dpdk_interfaces() is invoked before this. + pci_address = utils.get_stored_pci_address( + base_opt.members[0].name, self.noop) + ovs_extra.append("set Interface $DEVICE options:dpdk-devargs=" + "%s" % pci_address) if base_opt.mtu: ovs_extra.append("set Interface $DEVICE mtu_request=$MTU") if base_opt.rx_queue: @@ -318,6 +326,16 @@ class IfcfgNetConfig(os_net_config.NetConfig): data += "TYPE=OVSDPDKBond\n" data += "OVS_BRIDGE=%s\n" % base_opt.bridge_name if base_opt.members: + for bond_member in base_opt.members: + # Validation of DPDK port having only one interface is done + # prior to this. So accesing the interface name statically. + # Also pci_address would be valid here, since + # bind_dpdk_interfaces () is invoked before this. + pci_address = utils.get_stored_pci_address( + bond_member.members[0].name, self.noop) + ovs_extra.append("set Interface %s options:" + "dpdk-devargs=%s" + % (bond_member.name, pci_address)) members = [member.name for member in base_opt.members] data += ("BOND_IFACES=\"%s\"\n" % " ".join(members)) # MTU configuration given for the OvsDpdkbond shall be applied diff --git a/os_net_config/tests/test_impl_ifcfg.py b/os_net_config/tests/test_impl_ifcfg.py index 3a56f4d..d41cf60 100644 --- a/os_net_config/tests/test_impl_ifcfg.py +++ b/os_net_config/tests/test_impl_ifcfg.py @@ -413,6 +413,14 @@ class TestIfcfgNetConfig(base.TestCase): def get_route6_config(self, name='em1'): return self.provider.route6_data.get(name, '') + def stub_get_stored_pci_address(self, ifname, noop): + if 'eth0' in ifname: + return "0000:00:07.0" + if 'eth1' in ifname: + return "0000:00:08.0" + if 'eth2' in ifname: + return "0000:00:09.0" + def test_add_base_interface(self): interface = objects.Interface('em1') self.provider.add_interface(interface) @@ -894,6 +902,8 @@ DNS2=5.6.7.8 self.assertEqual(driver, 'vfio-pci') self.stubs.Set(utils, 'bind_dpdk_interfaces', test_bind_dpdk_interfaces) + self.stubs.Set(utils, 'get_stored_pci_address', + self.stub_get_stored_pci_address) self.provider.add_ovs_dpdk_port(dpdk_port) self.provider.add_ovs_user_bridge(bridge) @@ -915,6 +925,7 @@ PEERDNS=no DEVICETYPE=ovs TYPE=OVSDPDKPort OVS_BRIDGE=br-link +OVS_EXTRA="set Interface $DEVICE options:dpdk-devargs=0000:00:09.0" """ self.assertEqual(br_link_config, self.provider.bridge_data['br-link']) @@ -934,6 +945,8 @@ OVS_BRIDGE=br-link self.assertEqual(driver, 'vfio-pci') self.stubs.Set(utils, 'bind_dpdk_interfaces', test_bind_dpdk_interfaces) + self.stubs.Set(utils, 'get_stored_pci_address', + self.stub_get_stored_pci_address) self.provider.add_ovs_dpdk_port(dpdk_port) self.provider.add_ovs_user_bridge(bridge) @@ -957,7 +970,8 @@ TYPE=OVSDPDKPort OVS_BRIDGE=br-link RX_QUEUE=4 MTU=9000 -OVS_EXTRA="set Interface $DEVICE mtu_request=$MTU \ +OVS_EXTRA="set Interface $DEVICE options:dpdk-devargs=0000:00:09.0 \ +-- set Interface $DEVICE mtu_request=$MTU \ -- set Interface $DEVICE options:n_rxq=$RX_QUEUE" """ self.assertEqual(br_link_config, @@ -980,6 +994,8 @@ OVS_EXTRA="set Interface $DEVICE mtu_request=$MTU \ self.assertEqual(driver, 'vfio-pci') self.stubs.Set(utils, 'bind_dpdk_interfaces', test_bind_dpdk_interfaces) + self.stubs.Set(utils, 'get_stored_pci_address', + self.stub_get_stored_pci_address) self.provider.add_ovs_dpdk_bond(bond) self.provider.add_ovs_user_bridge(bridge) @@ -994,6 +1010,8 @@ DEVICETYPE=ovs TYPE=OVSDPDKBond OVS_BRIDGE=br-link BOND_IFACES="dpdk0 dpdk1" +OVS_EXTRA="set Interface dpdk0 options:dpdk-devargs=0000:00:08.0 \ +-- set Interface dpdk1 options:dpdk-devargs=0000:00:09.0" """ self.assertEqual(dpdk_bond_config, self.get_interface_config('dpdkbond0')) @@ -1015,6 +1033,8 @@ BOND_IFACES="dpdk0 dpdk1" self.assertEqual(driver, 'vfio-pci') self.stubs.Set(utils, 'bind_dpdk_interfaces', test_bind_dpdk_interfaces) + self.stubs.Set(utils, 'get_stored_pci_address', + self.stub_get_stored_pci_address) self.provider.add_ovs_dpdk_bond(bond) self.provider.add_ovs_user_bridge(bridge) @@ -1030,7 +1050,9 @@ TYPE=OVSDPDKBond OVS_BRIDGE=br-link BOND_IFACES="dpdk0 dpdk1" MTU=9000 -OVS_EXTRA="set Interface dpdk0 mtu_request=$MTU \ +OVS_EXTRA="set Interface dpdk0 options:dpdk-devargs=0000:00:08.0 \ +-- set Interface dpdk1 options:dpdk-devargs=0000:00:09.0 \ +-- set Interface dpdk0 mtu_request=$MTU \ -- set Interface dpdk1 mtu_request=$MTU" """ self.assertEqual(dpdk_bond_config, @@ -1053,6 +1075,8 @@ OVS_EXTRA="set Interface dpdk0 mtu_request=$MTU \ self.assertEqual(driver, 'vfio-pci') self.stubs.Set(utils, 'bind_dpdk_interfaces', test_bind_dpdk_interfaces) + self.stubs.Set(utils, 'get_stored_pci_address', + self.stub_get_stored_pci_address) self.provider.add_ovs_dpdk_bond(bond) self.provider.add_ovs_user_bridge(bridge) @@ -1068,7 +1092,9 @@ TYPE=OVSDPDKBond OVS_BRIDGE=br-link BOND_IFACES="dpdk0 dpdk1" RX_QUEUE=4 -OVS_EXTRA="set Interface dpdk0 options:n_rxq=$RX_QUEUE \ +OVS_EXTRA="set Interface dpdk0 options:dpdk-devargs=0000:00:08.0 \ +-- set Interface dpdk1 options:dpdk-devargs=0000:00:09.0 \ +-- set Interface dpdk0 options:n_rxq=$RX_QUEUE \ -- set Interface dpdk1 options:n_rxq=$RX_QUEUE" """ self.assertEqual(dpdk_bond_config, @@ -1091,6 +1117,8 @@ OVS_EXTRA="set Interface dpdk0 options:n_rxq=$RX_QUEUE \ self.assertEqual(driver, 'vfio-pci') self.stubs.Set(utils, 'bind_dpdk_interfaces', test_bind_dpdk_interfaces) + self.stubs.Set(utils, 'get_stored_pci_address', + self.stub_get_stored_pci_address) self.provider.add_ovs_dpdk_bond(bond) self.provider.add_ovs_user_bridge(bridge) @@ -1107,7 +1135,9 @@ OVS_BRIDGE=br-link BOND_IFACES="dpdk0 dpdk1" RX_QUEUE=4 MTU=9000 -OVS_EXTRA="set Interface dpdk0 mtu_request=$MTU \ +OVS_EXTRA="set Interface dpdk0 options:dpdk-devargs=0000:00:08.0 \ +-- set Interface dpdk1 options:dpdk-devargs=0000:00:09.0 \ +-- set Interface dpdk0 mtu_request=$MTU \ -- set Interface dpdk1 mtu_request=$MTU \ -- set Interface dpdk0 options:n_rxq=$RX_QUEUE \ -- set Interface dpdk1 options:n_rxq=$RX_QUEUE" diff --git a/os_net_config/tests/test_utils.py b/os_net_config/tests/test_utils.py index dcd6045..e09b6f7 100644 --- a/os_net_config/tests/test_utils.py +++ b/os_net_config/tests/test_utils.py @@ -130,6 +130,24 @@ class TestUtils(base.TestCase): pci = utils.get_pci_address('nic2', False) self.assertEqual(None, pci) + def test_get_stored_pci_address_success(self): + def test_get_dpdk_map(): + return [{'name': 'eth1', 'pci_address': '0000:00:09.0', + 'mac_address': '01:02:03:04:05:06', + 'driver': 'vfio-pci'}] + + self.stubs.Set(utils, '_get_dpdk_map', test_get_dpdk_map) + pci = utils.get_stored_pci_address('eth1', False) + self.assertEqual('0000:00:09.0', pci) + + def test_get_stored_pci_address_empty(self): + def test_get_dpdk_map(): + return [] + + self.stubs.Set(utils, '_get_dpdk_map', test_get_dpdk_map) + pci = utils.get_stored_pci_address('eth1', False) + self.assertEqual(None, pci) + def test_bind_dpdk_interfaces(self): def test_execute(name, dummy1, dummy2=None, dummy3=None): if 'ethtool' in name: @@ -143,8 +161,10 @@ class TestUtils(base.TestCase): self.stubs.Set(processutils, 'execute', test_execute) self.stubs.Set(utils, '_get_dpdk_mac_address', test_get_dpdk_mac_address) - - utils.bind_dpdk_interfaces('nic2', 'vfio-pci', False) + try: + utils.bind_dpdk_interfaces('nic2', 'vfio-pci', False) + except utils.OvsDpdkBindException: + self.fail("Received OvsDpdkBindException unexpectedly") def test_bind_dpdk_interfaces_fail(self): def test_execute(name, dummy1, dummy2=None, dummy3=None): @@ -164,6 +184,54 @@ class TestUtils(base.TestCase): utils.bind_dpdk_interfaces, 'eth1', 'vfio-pci', False) + def test_bind_dpdk_interfaces_skip_valid_device(self): + def test_execute(name, dummy1, dummy2=None, dummy3=None): + if 'ethtool' in name: + return None, 'Error' + if 'driverctl' in name: + return None, None + + def test_get_dpdk_mac_address(name): + return '01:02:03:04:05:06' + + def test_get_dpdk_map(): + return [{'name': 'eth1', 'pci_address': '0000:00:09.0', + 'mac_address': '01:02:03:04:05:06', + 'driver': 'vfio-pci'}] + + self.stubs.Set(utils, '_get_dpdk_map', test_get_dpdk_map) + self.stubs.Set(processutils, 'execute', test_execute) + self.stubs.Set(utils, '_get_dpdk_mac_address', + test_get_dpdk_mac_address) + try: + utils.bind_dpdk_interfaces('eth1', 'vfio-pci', False) + except utils.OvsDpdkBindException: + self.fail("Received OvsDpdkBindException unexpectedly") + + def test_bind_dpdk_interfaces_fail_invalid_device(self): + def test_execute(name, dummy1, dummy2=None, dummy3=None): + if 'ethtool' in name: + return None, 'Error' + if 'driverctl' in name: + return None, None + + def test_get_dpdk_mac_address(name): + return '01:02:03:04:05:06' + + def test_get_dpdk_map(): + return [{'name': 'eth1', 'pci_address': '0000:00:09.0', + 'mac_address': '01:02:03:04:05:06', + 'driver': 'vfio-pci'}] + + self.stubs.Set(utils, '_get_dpdk_map', test_get_dpdk_map) + self.stubs.Set(processutils, 'execute', test_execute) + self.stubs.Set(utils, '_get_dpdk_mac_address', + test_get_dpdk_mac_address) + + self.assertRaises(utils.OvsDpdkBindException, + utils.bind_dpdk_interfaces, 'eth2', 'vfio-pci', + False) + def test__update_dpdk_map_new(self): utils._update_dpdk_map('eth1', '0000:03:00.0', '01:02:03:04:05:06', 'vfio-pci') diff --git a/os_net_config/utils.py b/os_net_config/utils.py index b7a54bf..27e888d 100644 --- a/os_net_config/utils.py +++ b/os_net_config/utils.py @@ -238,6 +238,14 @@ def bind_dpdk_interfaces(ifname, driver, noop): except processutils.ProcessExecutionError: msg = "Failed to bind interface %s with dpdk" % ifname raise OvsDpdkBindException(msg) + else: + # Check if the pci address is already fetched and stored. + # If the pci address could not be fetched from dpdk_mapping.yaml + # raise OvsDpdkBindException, since the interface is neither + # available nor bound with dpdk. + if not get_stored_pci_address(ifname, noop): + msg = "Interface %s cannot be found" % ifname + raise OvsDpdkBindException(msg) else: logger.info('Interface %(name)s bound to DPDK driver %(driver)s ' 'using driverctl command' % @@ -264,6 +272,17 @@ def get_pci_address(ifname, noop): 'ethtool' % ifname) +def get_stored_pci_address(ifname, noop): + if not noop: + dpdk_map = _get_dpdk_map() + for dpdk_nic in dpdk_map: + if dpdk_nic['name'] == ifname: + return dpdk_nic['pci_address'] + else: + logger.info('Fetch the PCI address of the interface %s using ' + 'ethtool' % ifname) + + # Once the interface is bound to a DPDK driver, all the references to the # interface including '/sys' and '/proc', will be removed. And there is no # way to identify the nic name after it is bound. So, the DPDK bound nic info -- cgit 1.2.3-korg