aboutsummaryrefslogtreecommitdiffstats
path: root/os_net_config
diff options
context:
space:
mode:
authorSaravanan KR <skramaja@redhat.com>2016-08-31 17:10:56 +0530
committerSaravanan KR <skramaja@redhat.com>2016-09-20 11:25:11 +0530
commitc82cc955983065dba7b4d60858082150834ac356 (patch)
tree93fe7f741245fe8f26e2e8f15694f69ee1bda2e6 /os_net_config
parentf79d534c3e130bb539cf7ccb1d3f9586e922b9a7 (diff)
Fixed nic numbering issue of DPDK nics after the nic has bound
* os-net-config is called multiple times during the deploy. Once the interface is bound to a driver, it will not be listed for ethtool to get the pci address, which will through exception. Handled this exception. * Stored the DPDK bound nic configs at '/var/lib/os-net-config/ dpdk_mappings.yaml' file to emulate the same nic numbering after the nic has been bound to the DPDK driver. Partial-Bug: #1619330 Change-Id: I6b1e45003f851f1fcf5b8730890c75331e8d0f8f
Diffstat (limited to 'os_net_config')
-rw-r--r--os_net_config/tests/test_utils.py151
-rw-r--r--os_net_config/utils.py111
2 files changed, 244 insertions, 18 deletions
diff --git a/os_net_config/tests/test_utils.py b/os_net_config/tests/test_utils.py
index b6531a6..a2d5cc4 100644
--- a/os_net_config/tests/test_utils.py
+++ b/os_net_config/tests/test_utils.py
@@ -14,16 +14,43 @@
# License for the specific language governing permissions and limitations
# under the License.
+import os
import os.path
+import random
import shutil
import tempfile
+import yaml
from os_net_config.tests import base
from os_net_config import utils
+from oslo_concurrency import processutils
+
+_PCI_OUTPUT = '''driver: e1000e
+version: 3.2.6-k
+firmware-version: 0.13-3
+expansion-rom-version:
+bus-info: 0000:00:19.0
+supports-statistics: yes
+supports-test: yes
+supports-eeprom-access: yes
+supports-register-dump: yes
+supports-priv-flags: no
+'''
+
class TestUtils(base.TestCase):
+ def setUp(self):
+ super(TestUtils, self).setUp()
+ rand = str(int(random.random() * 100000))
+ utils._DPDK_MAPPING_FILE = '/tmp/dpdk_mapping_' + rand + '.yaml'
+
+ def tearDown(self):
+ super(TestUtils, self).tearDown()
+ if os.path.isfile(utils._DPDK_MAPPING_FILE):
+ os.remove(utils._DPDK_MAPPING_FILE)
+
def test_ordered_active_nics(self):
tmpdir = tempfile.mkdtemp()
@@ -49,3 +76,127 @@ class TestUtils(base.TestCase):
self.assertEqual('z1', nics[7])
shutil.rmtree(tmpdir)
+
+ def test_get_pci_address_success(self):
+ def test_execute(name, dummy1, dummy2=None, dummy3=None):
+ if 'ethtool' in name:
+ out = _PCI_OUTPUT
+ return out, None
+ self.stubs.Set(processutils, 'execute', test_execute)
+ pci = utils._get_pci_address('nic2', False)
+ self.assertEqual('0000:00:19.0', pci)
+
+ def test_get_pci_address_exception(self):
+ def test_execute(name, dummy1, dummy2=None, dummy3=None):
+ if 'ethtool' in name:
+ raise processutils.ProcessExecutionError
+ self.stubs.Set(processutils, 'execute', test_execute)
+ pci = utils._get_pci_address('nic2', False)
+ self.assertEqual(None, pci)
+
+ def test_get_pci_address_error(self):
+ def test_execute(name, dummy1, dummy2=None, dummy3=None):
+ if 'ethtool' in name:
+ return None, 'Error'
+ self.stubs.Set(processutils, 'execute', test_execute)
+ pci = utils._get_pci_address('nic2', False)
+ self.assertEqual(None, pci)
+
+ def test_bind_dpdk_interfaces(self):
+ def test_execute(name, dummy1, dummy2=None, dummy3=None):
+ if 'ethtool' in name:
+ out = _PCI_OUTPUT
+ return out, None
+ if 'driverctl' in name:
+ return None, None
+ self.stubs.Set(processutils, 'execute', test_execute)
+ utils.bind_dpdk_interfaces('nic2', 'vfio-pci', False)
+
+ def test_bind_dpdk_interfaces_fail(self):
+ def test_execute(name, dummy1, dummy2=None, dummy3=None):
+ if 'ethtool' in name:
+ out = _PCI_OUTPUT
+ return out, None
+ if 'driverctl' in name:
+ return None, 'Error'
+ self.stubs.Set(processutils, 'execute', test_execute)
+ self.assertRaises(utils.OvsDpdkBindException,
+ utils.bind_dpdk_interfaces, 'nic2', 'vfio-pci',
+ False)
+
+ def test_update_dpdk_map_new(self):
+ utils._update_dpdk_map('eth1', '0000:03:00.0', 'vfio-pci')
+ try:
+ contents = utils.get_file_data(utils._DPDK_MAPPING_FILE)
+ except IOError:
+ pass
+
+ dpdk_map = yaml.load(contents) if contents else []
+ self.assertEqual(1, len(dpdk_map))
+ dpdk_test = [{'name': 'eth1', 'pci_address': '0000:03:00.0',
+ 'driver': 'vfio-pci'}]
+ self.assertListEqual(dpdk_test, dpdk_map)
+
+ def test_update_dpdk_map_exist(self):
+ dpdk_test = [{'name': 'eth1', 'pci_address': '0000:03:00.0',
+ 'driver': 'vfio-pci'}]
+ utils.write_yaml_config(utils._DPDK_MAPPING_FILE, dpdk_test)
+
+ utils._update_dpdk_map('eth1', '0000:03:00.0', 'vfio-pci')
+ try:
+ contents = utils.get_file_data(utils._DPDK_MAPPING_FILE)
+ except IOError:
+ pass
+
+ dpdk_map = yaml.load(contents) if contents else []
+ self.assertEqual(1, len(dpdk_map))
+ self.assertListEqual(dpdk_test, dpdk_map)
+
+ def test_update_dpdk_map_value_change(self):
+ dpdk_test = [{'name': 'eth1', 'pci_address': '0000:03:00.0',
+ 'driver': 'vfio-pci'}]
+ utils.write_yaml_config(utils._DPDK_MAPPING_FILE, dpdk_test)
+
+ dpdk_test = [{'name': 'eth1', 'pci_address': '0000:03:00.0',
+ 'driver': 'igb_uio'}]
+ utils._update_dpdk_map('eth1', '0000:03:00.0', 'igb_uio')
+ try:
+ contents = utils.get_file_data(utils._DPDK_MAPPING_FILE)
+ except IOError:
+ pass
+
+ dpdk_map = yaml.load(contents) if contents else []
+ self.assertEqual(1, len(dpdk_map))
+ self.assertListEqual(dpdk_test, dpdk_map)
+
+ def test_ordered_active_nics_with_dpdk_mapping(self):
+
+ tmpdir = tempfile.mkdtemp()
+ self.stubs.Set(utils, '_SYS_CLASS_NET', tmpdir)
+
+ def test_is_active_nic(interface_name):
+ return True
+ self.stubs.Set(utils, '_is_active_nic', test_is_active_nic)
+
+ for nic in ['a1', 'em1', 'em2', 'eth2', 'z1',
+ 'enp8s0', 'enp10s0', 'enp1s0f0']:
+ with open(os.path.join(tmpdir, nic), 'w') as f:
+ f.write(nic)
+
+ utils._update_dpdk_map('eth1', '0000:03:00.0', 'igb_uio')
+ utils._update_dpdk_map('p3p1', '0000:04:00.0', 'igb_uio')
+
+ nics = utils.ordered_active_nics()
+
+ self.assertEqual('em1', nics[0])
+ self.assertEqual('em2', nics[1])
+ self.assertEqual('eth1', nics[2]) # DPDK bound nic
+ self.assertEqual('eth2', nics[3])
+ self.assertEqual('a1', nics[4])
+ self.assertEqual('enp1s0f0', nics[5])
+ self.assertEqual('enp8s0', nics[6])
+ self.assertEqual('enp10s0', nics[7])
+ self.assertEqual('p3p1', nics[8]) # DPDK bound nic
+ self.assertEqual('z1', nics[9])
+
+ shutil.rmtree(tmpdir)
diff --git a/os_net_config/utils.py b/os_net_config/utils.py
index 7b85d91..95325a3 100644
--- a/os_net_config/utils.py
+++ b/os_net_config/utils.py
@@ -18,12 +18,14 @@ import glob
import logging
import os
import re
+import yaml
from oslo_concurrency import processutils
logger = logging.getLogger(__name__)
_SYS_CLASS_NET = '/sys/class/net'
+_DPDK_MAPPING_FILE = '/var/lib/os-net-config/dpdk_mapping.yaml'
class OvsDpdkBindException(ValueError):
@@ -35,6 +37,18 @@ def write_config(filename, data):
f.write(str(data))
+def write_yaml_config(filepath, data):
+ ensure_directory_presence(filepath)
+ with open(filepath, 'w') as f:
+ yaml.dump(data, f, default_flow_style=False)
+
+
+def ensure_directory_presence(filepath):
+ dir_path = os.path.dirname(filepath)
+ if not os.path.exists(dir_path):
+ os.makedirs(dir_path)
+
+
def get_file_data(filename):
if not os.path.exists(filename):
return ''
@@ -93,6 +107,12 @@ def _natural_sort_key(s):
for text in re.split(nsre, s)]
+def _is_embedded_nic(nic):
+ if nic.startswith('em') or nic.startswith('eth') or nic.startswith('eno'):
+ return True
+ return False
+
+
def ordered_active_nics():
embedded_nics = []
nics = []
@@ -100,8 +120,7 @@ def ordered_active_nics():
for name in glob.iglob(_SYS_CLASS_NET + '/*'):
nic = name[(len(_SYS_CLASS_NET) + 1):]
if _is_active_nic(nic):
- if nic.startswith('em') or nic.startswith('eth') or \
- nic.startswith('eno'):
+ if _is_embedded_nic(nic):
logger.debug("%s is an embedded active nic" % nic)
embedded_nics.append(nic)
else:
@@ -109,6 +128,24 @@ def ordered_active_nics():
nics.append(nic)
else:
logger.debug("%s is not an active nic" % nic)
+
+ # Adding nics which are bound to DPDK as it will not be found in '/sys'
+ # after it is bound to DPDK driver.
+ contents = get_file_data(_DPDK_MAPPING_FILE)
+ if contents:
+ dpdk_map = yaml.load(contents)
+ for item in dpdk_map:
+ nic = item['name']
+ if _is_embedded_nic(nic):
+ logger.debug("%s is an embedded DPDK bound nic" % nic)
+ embedded_nics.append(nic)
+ else:
+ logger.debug("%s is an DPDK bound nic" % nic)
+ nics.append(nic)
+ else:
+ logger.debug("No DPDK mapping available in path (%s)" %
+ _DPDK_MAPPING_FILE)
+
# NOTE: we could just natural sort all active devices,
# but this ensures em, eno, and eth are ordered first
# (more backwards compatible)
@@ -127,20 +164,30 @@ def diff(filename, data):
def bind_dpdk_interfaces(ifname, driver, noop):
- pci_addres = _get_pci_address(ifname, noop)
+ pci_address = _get_pci_address(ifname, noop)
if not noop:
- if pci_addres:
+ if pci_address:
# modbprobe of the driver has to be done before binding.
# for reboots, puppet will add the modprobe to /etc/rc.modules
- processutils.execute('modprobe', 'vfio-pci')
-
- out, err = processutils.execute('driverctl', 'set-override',
- pci_addres, driver)
- if err:
+ if 'vfio-pci' in driver:
+ try:
+ processutils.execute('modprobe', 'vfio-pci')
+ except processutils.ProcessExecutionError:
+ msg = "Failed to modprobe vfio-pci module"
+ raise OvsDpdkBindException(msg)
+
+ try:
+ out, err = processutils.execute('driverctl', 'set-override',
+ pci_address, driver)
+ if err:
+ msg = "Failed to bind dpdk interface err - %s" % err
+ raise OvsDpdkBindException(msg)
+ else:
+ _update_dpdk_map(ifname, pci_address, driver)
+
+ except processutils.ProcessExecutionError:
msg = "Failed to bind interface %s with dpdk" % ifname
raise OvsDpdkBindException(msg)
- else:
- processutils.execute('driverctl', 'load-override', pci_addres)
else:
logger.info('Interface %(name)s bound to DPDK driver %(driver)s '
'using driverctl command' %
@@ -150,13 +197,41 @@ def bind_dpdk_interfaces(ifname, driver, noop):
def _get_pci_address(ifname, noop):
# TODO(skramaja): Validate if the given interface supports dpdk
if not noop:
- # If ifname is already bound, then ethtool will not be able to list the
- # device, in which case, binding is already done, proceed with scripts
- out, err = processutils.execute('ethtool', '-i', ifname)
- if not err:
- for item in out.split('\n'):
- if 'bus-info' in item:
- return item.split(' ')[1]
+ try:
+ out, err = processutils.execute('ethtool', '-i', ifname)
+ if not err:
+ for item in out.split('\n'):
+ if 'bus-info' in item:
+ return item.split(' ')[1]
+ except processutils.ProcessExecutionError:
+ # If ifname is already bound, then ethtool will not be able to
+ # list the device, in which case, binding is already done, proceed
+ # with scripts generation.
+ return
+
else:
logger.info('Fetch the PCI address of the interface %s using '
'ethtool' % ifname)
+
+
+# Once the interface is bound to a DPDK driver, all the references to the
+# interface including '/sys' and '/proc', will be removed. And there is no
+# way to identify the nic name after it is bound. So, the DPDK bound nic info
+# is stored persistently in a file and is used to for nic numbering on
+# subsequent runs of os-net-config.
+def _update_dpdk_map(ifname, pci_address, driver):
+ contents = get_file_data(_DPDK_MAPPING_FILE)
+ dpdk_map = yaml.load(contents) if contents else []
+ for item in dpdk_map:
+ if item['pci_address'] == pci_address:
+ item['name'] = ifname
+ item['driver'] = driver
+ break
+ else:
+ new_item = {}
+ new_item['pci_address'] = pci_address
+ new_item['name'] = ifname
+ new_item['driver'] = driver
+ dpdk_map.append(new_item)
+
+ write_yaml_config(_DPDK_MAPPING_FILE, dpdk_map)