From c82cc955983065dba7b4d60858082150834ac356 Mon Sep 17 00:00:00 2001 From: Saravanan KR Date: Wed, 31 Aug 2016 17:10:56 +0530 Subject: Fixed nic numbering issue of DPDK nics after the nic has bound * os-net-config is called multiple times during the deploy. Once the interface is bound to a driver, it will not be listed for ethtool to get the pci address, which will through exception. Handled this exception. * Stored the DPDK bound nic configs at '/var/lib/os-net-config/ dpdk_mappings.yaml' file to emulate the same nic numbering after the nic has been bound to the DPDK driver. Partial-Bug: #1619330 Change-Id: I6b1e45003f851f1fcf5b8730890c75331e8d0f8f --- os_net_config/tests/test_utils.py | 151 ++++++++++++++++++++++++++++++++++++++ os_net_config/utils.py | 111 +++++++++++++++++++++++----- 2 files changed, 244 insertions(+), 18 deletions(-) diff --git a/os_net_config/tests/test_utils.py b/os_net_config/tests/test_utils.py index b6531a6..a2d5cc4 100644 --- a/os_net_config/tests/test_utils.py +++ b/os_net_config/tests/test_utils.py @@ -14,16 +14,43 @@ # License for the specific language governing permissions and limitations # under the License. +import os import os.path +import random import shutil import tempfile +import yaml from os_net_config.tests import base from os_net_config import utils +from oslo_concurrency import processutils + +_PCI_OUTPUT = '''driver: e1000e +version: 3.2.6-k +firmware-version: 0.13-3 +expansion-rom-version: +bus-info: 0000:00:19.0 +supports-statistics: yes +supports-test: yes +supports-eeprom-access: yes +supports-register-dump: yes +supports-priv-flags: no +''' + class TestUtils(base.TestCase): + def setUp(self): + super(TestUtils, self).setUp() + rand = str(int(random.random() * 100000)) + utils._DPDK_MAPPING_FILE = '/tmp/dpdk_mapping_' + rand + '.yaml' + + def tearDown(self): + super(TestUtils, self).tearDown() + if os.path.isfile(utils._DPDK_MAPPING_FILE): + os.remove(utils._DPDK_MAPPING_FILE) + def test_ordered_active_nics(self): tmpdir = tempfile.mkdtemp() @@ -49,3 +76,127 @@ class TestUtils(base.TestCase): self.assertEqual('z1', nics[7]) shutil.rmtree(tmpdir) + + def test_get_pci_address_success(self): + def test_execute(name, dummy1, dummy2=None, dummy3=None): + if 'ethtool' in name: + out = _PCI_OUTPUT + return out, None + self.stubs.Set(processutils, 'execute', test_execute) + pci = utils._get_pci_address('nic2', False) + self.assertEqual('0000:00:19.0', pci) + + def test_get_pci_address_exception(self): + def test_execute(name, dummy1, dummy2=None, dummy3=None): + if 'ethtool' in name: + raise processutils.ProcessExecutionError + self.stubs.Set(processutils, 'execute', test_execute) + pci = utils._get_pci_address('nic2', False) + self.assertEqual(None, pci) + + def test_get_pci_address_error(self): + def test_execute(name, dummy1, dummy2=None, dummy3=None): + if 'ethtool' in name: + return None, 'Error' + self.stubs.Set(processutils, 'execute', test_execute) + pci = utils._get_pci_address('nic2', False) + self.assertEqual(None, pci) + + def test_bind_dpdk_interfaces(self): + def test_execute(name, dummy1, dummy2=None, dummy3=None): + if 'ethtool' in name: + out = _PCI_OUTPUT + return out, None + if 'driverctl' in name: + return None, None + self.stubs.Set(processutils, 'execute', test_execute) + utils.bind_dpdk_interfaces('nic2', 'vfio-pci', False) + + def test_bind_dpdk_interfaces_fail(self): + def test_execute(name, dummy1, dummy2=None, dummy3=None): + if 'ethtool' in name: + out = _PCI_OUTPUT + return out, None + if 'driverctl' in name: + return None, 'Error' + self.stubs.Set(processutils, 'execute', test_execute) + self.assertRaises(utils.OvsDpdkBindException, + utils.bind_dpdk_interfaces, 'nic2', 'vfio-pci', + False) + + def test_update_dpdk_map_new(self): + utils._update_dpdk_map('eth1', '0000:03:00.0', 'vfio-pci') + try: + contents = utils.get_file_data(utils._DPDK_MAPPING_FILE) + except IOError: + pass + + dpdk_map = yaml.load(contents) if contents else [] + self.assertEqual(1, len(dpdk_map)) + dpdk_test = [{'name': 'eth1', 'pci_address': '0000:03:00.0', + 'driver': 'vfio-pci'}] + self.assertListEqual(dpdk_test, dpdk_map) + + def test_update_dpdk_map_exist(self): + dpdk_test = [{'name': 'eth1', 'pci_address': '0000:03:00.0', + 'driver': 'vfio-pci'}] + utils.write_yaml_config(utils._DPDK_MAPPING_FILE, dpdk_test) + + utils._update_dpdk_map('eth1', '0000:03:00.0', 'vfio-pci') + try: + contents = utils.get_file_data(utils._DPDK_MAPPING_FILE) + except IOError: + pass + + dpdk_map = yaml.load(contents) if contents else [] + self.assertEqual(1, len(dpdk_map)) + self.assertListEqual(dpdk_test, dpdk_map) + + def test_update_dpdk_map_value_change(self): + dpdk_test = [{'name': 'eth1', 'pci_address': '0000:03:00.0', + 'driver': 'vfio-pci'}] + utils.write_yaml_config(utils._DPDK_MAPPING_FILE, dpdk_test) + + dpdk_test = [{'name': 'eth1', 'pci_address': '0000:03:00.0', + 'driver': 'igb_uio'}] + utils._update_dpdk_map('eth1', '0000:03:00.0', 'igb_uio') + try: + contents = utils.get_file_data(utils._DPDK_MAPPING_FILE) + except IOError: + pass + + dpdk_map = yaml.load(contents) if contents else [] + self.assertEqual(1, len(dpdk_map)) + self.assertListEqual(dpdk_test, dpdk_map) + + def test_ordered_active_nics_with_dpdk_mapping(self): + + tmpdir = tempfile.mkdtemp() + self.stubs.Set(utils, '_SYS_CLASS_NET', tmpdir) + + def test_is_active_nic(interface_name): + return True + self.stubs.Set(utils, '_is_active_nic', test_is_active_nic) + + for nic in ['a1', 'em1', 'em2', 'eth2', 'z1', + 'enp8s0', 'enp10s0', 'enp1s0f0']: + with open(os.path.join(tmpdir, nic), 'w') as f: + f.write(nic) + + utils._update_dpdk_map('eth1', '0000:03:00.0', 'igb_uio') + utils._update_dpdk_map('p3p1', '0000:04:00.0', 'igb_uio') + + nics = utils.ordered_active_nics() + + self.assertEqual('em1', nics[0]) + self.assertEqual('em2', nics[1]) + self.assertEqual('eth1', nics[2]) # DPDK bound nic + self.assertEqual('eth2', nics[3]) + self.assertEqual('a1', nics[4]) + self.assertEqual('enp1s0f0', nics[5]) + self.assertEqual('enp8s0', nics[6]) + self.assertEqual('enp10s0', nics[7]) + self.assertEqual('p3p1', nics[8]) # DPDK bound nic + self.assertEqual('z1', nics[9]) + + shutil.rmtree(tmpdir) diff --git a/os_net_config/utils.py b/os_net_config/utils.py index 7b85d91..95325a3 100644 --- a/os_net_config/utils.py +++ b/os_net_config/utils.py @@ -18,12 +18,14 @@ import glob import logging import os import re +import yaml from oslo_concurrency import processutils logger = logging.getLogger(__name__) _SYS_CLASS_NET = '/sys/class/net' +_DPDK_MAPPING_FILE = '/var/lib/os-net-config/dpdk_mapping.yaml' class OvsDpdkBindException(ValueError): @@ -35,6 +37,18 @@ def write_config(filename, data): f.write(str(data)) +def write_yaml_config(filepath, data): + ensure_directory_presence(filepath) + with open(filepath, 'w') as f: + yaml.dump(data, f, default_flow_style=False) + + +def ensure_directory_presence(filepath): + dir_path = os.path.dirname(filepath) + if not os.path.exists(dir_path): + os.makedirs(dir_path) + + def get_file_data(filename): if not os.path.exists(filename): return '' @@ -93,6 +107,12 @@ def _natural_sort_key(s): for text in re.split(nsre, s)] +def _is_embedded_nic(nic): + if nic.startswith('em') or nic.startswith('eth') or nic.startswith('eno'): + return True + return False + + def ordered_active_nics(): embedded_nics = [] nics = [] @@ -100,8 +120,7 @@ def ordered_active_nics(): for name in glob.iglob(_SYS_CLASS_NET + '/*'): nic = name[(len(_SYS_CLASS_NET) + 1):] if _is_active_nic(nic): - if nic.startswith('em') or nic.startswith('eth') or \ - nic.startswith('eno'): + if _is_embedded_nic(nic): logger.debug("%s is an embedded active nic" % nic) embedded_nics.append(nic) else: @@ -109,6 +128,24 @@ def ordered_active_nics(): nics.append(nic) else: logger.debug("%s is not an active nic" % nic) + + # Adding nics which are bound to DPDK as it will not be found in '/sys' + # after it is bound to DPDK driver. + contents = get_file_data(_DPDK_MAPPING_FILE) + if contents: + dpdk_map = yaml.load(contents) + for item in dpdk_map: + nic = item['name'] + if _is_embedded_nic(nic): + logger.debug("%s is an embedded DPDK bound nic" % nic) + embedded_nics.append(nic) + else: + logger.debug("%s is an DPDK bound nic" % nic) + nics.append(nic) + else: + logger.debug("No DPDK mapping available in path (%s)" % + _DPDK_MAPPING_FILE) + # NOTE: we could just natural sort all active devices, # but this ensures em, eno, and eth are ordered first # (more backwards compatible) @@ -127,20 +164,30 @@ def diff(filename, data): def bind_dpdk_interfaces(ifname, driver, noop): - pci_addres = _get_pci_address(ifname, noop) + pci_address = _get_pci_address(ifname, noop) if not noop: - if pci_addres: + if pci_address: # modbprobe of the driver has to be done before binding. # for reboots, puppet will add the modprobe to /etc/rc.modules - processutils.execute('modprobe', 'vfio-pci') - - out, err = processutils.execute('driverctl', 'set-override', - pci_addres, driver) - if err: + if 'vfio-pci' in driver: + try: + processutils.execute('modprobe', 'vfio-pci') + except processutils.ProcessExecutionError: + msg = "Failed to modprobe vfio-pci module" + raise OvsDpdkBindException(msg) + + try: + out, err = processutils.execute('driverctl', 'set-override', + pci_address, driver) + if err: + msg = "Failed to bind dpdk interface err - %s" % err + raise OvsDpdkBindException(msg) + else: + _update_dpdk_map(ifname, pci_address, driver) + + except processutils.ProcessExecutionError: msg = "Failed to bind interface %s with dpdk" % ifname raise OvsDpdkBindException(msg) - else: - processutils.execute('driverctl', 'load-override', pci_addres) else: logger.info('Interface %(name)s bound to DPDK driver %(driver)s ' 'using driverctl command' % @@ -150,13 +197,41 @@ def bind_dpdk_interfaces(ifname, driver, noop): def _get_pci_address(ifname, noop): # TODO(skramaja): Validate if the given interface supports dpdk if not noop: - # If ifname is already bound, then ethtool will not be able to list the - # device, in which case, binding is already done, proceed with scripts - out, err = processutils.execute('ethtool', '-i', ifname) - if not err: - for item in out.split('\n'): - if 'bus-info' in item: - return item.split(' ')[1] + try: + out, err = processutils.execute('ethtool', '-i', ifname) + if not err: + for item in out.split('\n'): + if 'bus-info' in item: + return item.split(' ')[1] + except processutils.ProcessExecutionError: + # If ifname is already bound, then ethtool will not be able to + # list the device, in which case, binding is already done, proceed + # with scripts generation. + return + else: logger.info('Fetch the PCI address of the interface %s using ' 'ethtool' % ifname) + + +# Once the interface is bound to a DPDK driver, all the references to the +# interface including '/sys' and '/proc', will be removed. And there is no +# way to identify the nic name after it is bound. So, the DPDK bound nic info +# is stored persistently in a file and is used to for nic numbering on +# subsequent runs of os-net-config. +def _update_dpdk_map(ifname, pci_address, driver): + contents = get_file_data(_DPDK_MAPPING_FILE) + dpdk_map = yaml.load(contents) if contents else [] + for item in dpdk_map: + if item['pci_address'] == pci_address: + item['name'] = ifname + item['driver'] = driver + break + else: + new_item = {} + new_item['pci_address'] = pci_address + new_item['name'] = ifname + new_item['driver'] = driver + dpdk_map.append(new_item) + + write_yaml_config(_DPDK_MAPPING_FILE, dpdk_map) -- cgit 1.2.3-korg