summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMaryam Tahhan <maryam.tahhan@intel.com>2016-03-02 13:01:41 +0000
committerMaryam Tahhan <maryam.tahhan@intel.com>2016-03-08 13:58:14 +0000
commitf7366919b9d985b2b3d267750ea16667b636cdac (patch)
treea66e93d86a33556edfd266929448fb47fa924641
parent86bc27fe76137a777979f17556fa999a74027428 (diff)
dpdk: enable vfio_pci support
Enable vfio_pci support for DPDK in the case where the end user doesn't want to use igb_uio. Changes: * Use generic functions to strip the path and .ko extension from the module to be inserted/removed. * Removed debug line in Modules_Manager. * Removed commented code. * Fixed all pylint issues. JIRA: VSPERF-249 Change-Id: I69c9077735879bcbf7ce5c970c1ec53c219e9f90 Signed-off-by: Maryam Tahhan <maryam.tahhan@intel.com> Reviewed-by: Martin Klozik <martinx.klozik@intel.com> Reviewed-by: Al Morton <acmorton@att.com> Reviewed-by: Christian Trautman <ctrautma@redhat.com>
-rw-r--r--conf/02_vswitch.conf8
-rwxr-xr-xdocs/userguide/testusage.rst37
-rw-r--r--src/dpdk/dpdk.py169
-rw-r--r--tools/module_manager.py81
4 files changed, 171 insertions, 124 deletions
diff --git a/conf/02_vswitch.conf b/conf/02_vswitch.conf
index b9862c92..f0475313 100644
--- a/conf/02_vswitch.conf
+++ b/conf/02_vswitch.conf
@@ -1,4 +1,4 @@
-# Copyright 2015 Intel Corporation.
+# Copyright 2015-2016 Intel Corporation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -41,6 +41,10 @@ BLACKLIST_NICS = ['0000:09:00.0', '0000:09:00.1', '0000:09:00.2',
'0000:09:00.3']
# for DPDK_MODULES the path is in reference to the build directory
+# To use vfio set
+# DPDK_MODULES = [
+# ('vfio-pci'),
+# ]
DPDK_MODULES = [
('kmod', 'igb_uio'),
]
@@ -50,6 +54,8 @@ VHOST_MODULE = [
]
# list of modules that will be inserted using 'modprobe' on system init
+# To use vfio set
+# SYS_MODULES = ['cuse']
SYS_MODULES = ['uio', 'cuse']
# vhost character device file used by dpdkvhostport QemuWrap cases
diff --git a/docs/userguide/testusage.rst b/docs/userguide/testusage.rst
index c1f02843..098a9d95 100755
--- a/docs/userguide/testusage.rst
+++ b/docs/userguide/testusage.rst
@@ -309,6 +309,43 @@ To run tests using Vanilla OVS:
$ ./vsperf --conf-file<path_to_custom_conf>/10_custom.conf
+Using vfio_pci with DPDK
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To use vfio with DPDK instead of igb_uio edit 'conf/02_vswitch.conf'
+with the following parameters:
+
+.. code-block:: console
+
+ DPDK_MODULES = [
+ ('vfio-pci'),
+ ]
+ SYS_MODULES = ['cuse']
+
+**NOTE:** Please ensure that Intel VT-d is enabled in BIOS.
+
+**NOTE:** Please ensure your boot/grub parameters include
+the following:
+
+.. code-block:: console
+
+ iommu=pt intel_iommu=on
+
+To check that IOMMU is enabled on your platform:
+
+.. code-block:: console
+
+ $ dmesg | grep IOMMU
+ [ 0.000000] Intel-IOMMU: enabled
+ [ 0.139882] dmar: IOMMU 0: reg_base_addr fbffe000 ver 1:0 cap d2078c106f0466 ecap f020de
+ [ 0.139888] dmar: IOMMU 1: reg_base_addr ebffc000 ver 1:0 cap d2078c106f0466 ecap f020de
+ [ 0.139893] IOAPIC id 2 under DRHD base 0xfbffe000 IOMMU 0
+ [ 0.139894] IOAPIC id 0 under DRHD base 0xebffc000 IOMMU 1
+ [ 0.139895] IOAPIC id 1 under DRHD base 0xebffc000 IOMMU 1
+ [ 3.335744] IOMMU: dmar0 using Queued invalidation
+ [ 3.335746] IOMMU: dmar1 using Queued invalidation
+ ....
+
Selection of loopback application for PVP and PVVP tests
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/src/dpdk/dpdk.py b/src/dpdk/dpdk.py
index f2f6ec9f..127ecaf2 100644
--- a/src/dpdk/dpdk.py
+++ b/src/dpdk/dpdk.py
@@ -34,7 +34,6 @@ RTE_PCI_TOOL = os.path.join(
settings.getValue('RTE_SDK'), 'tools', 'dpdk_nic_bind.py')
_DPDK_MODULE_MANAGER = ModuleManager()
-
#
# system management
#
@@ -46,7 +45,6 @@ def init():
if not _is_linux():
_LOGGER.error('Not running on a compatible Linux version. Exiting...')
return
-
_insert_modules()
_remove_vhost_net()
_bind_nics()
@@ -63,31 +61,10 @@ def cleanup():
_remove_modules()
_vhost_user_cleanup()
-
-#
-# vhost specific modules management
-#
-
-
-def insert_vhost_modules():
- """Inserts VHOST related kernel modules
- """
- mod_path_prefix = os.path.join(settings.getValue('RTE_SDK'),
- 'lib',
- 'librte_vhost')
- _insert_module_group('VHOST_MODULE', mod_path_prefix)
-
-
-def remove_vhost_modules():
- """Removes all VHOST related kernel modules
- """
- _remove_module_group('VHOST_MODULE')
-
#
# basic compatibility test
#
-
def _is_linux():
"""Check if running on Linux.
@@ -102,20 +79,6 @@ def _is_linux():
# module management
#
-
-def _is_module_inserted(module):
- """Check if a module is inserted on system.
- """
- with open('/proc/modules') as mod_file:
- loaded_mods = mod_file.readlines()
-
- # first check if module is loaded
- for line in loaded_mods:
- if line.startswith(module):
- return True
- return False
-
-
def _insert_modules():
"""Ensure required modules are inserted on system.
"""
@@ -123,74 +86,48 @@ def _insert_modules():
_DPDK_MODULE_MANAGER.insert_modules(settings.getValue('SYS_MODULES'))
mod_path_prefix = settings.getValue('OVS_DIR')
- _insert_module_group('OVS_MODULES', mod_path_prefix)
- mod_path_prefix = os.path.join(settings.getValue('RTE_SDK'),
- settings.getValue('RTE_TARGET'))
- _insert_module_group('DPDK_MODULES', mod_path_prefix)
-
-
-def _insert_module_group(module_group, group_path_prefix):
- """Ensure all modules in a group are inserted into the system.
-
- :param module_group: A name of configuration item containing a list
- of module names
- """
- for module in settings.getValue(module_group):
- # first check if module is loaded
- if _is_module_inserted(module[1]):
- continue
-
- try:
- mod_path = os.path.join(group_path_prefix, module[0],
- '%s.ko' % module[1])
- tasks.run_task(['sudo', 'insmod', mod_path], _LOGGER,
- 'Inserting module \'%s\'...' % module[1], True)
- except subprocess.CalledProcessError:
- _LOGGER.error('Unable to insert module \'%s\'.', module[1])
- raise # fail catastrophically
-
+ _DPDK_MODULE_MANAGER.insert_module_group(settings.getValue('OVS_MODULES'),
+ mod_path_prefix)
+ if 'vfio-pci' not in settings.getValue('DPDK_MODULES'):
+ mod_path_prefix = os.path.join(settings.getValue('RTE_SDK'),
+ settings.getValue('RTE_TARGET'))
+ _DPDK_MODULE_MANAGER.insert_module_group(settings.getValue('DPDK_MODULES'),
+ mod_path_prefix)
+ else:
+ _DPDK_MODULE_MANAGER.insert_modules(settings.getValue('DPDK_MODULES'))
def _remove_modules():
"""Ensure required modules are removed from system.
"""
- _remove_module_group('OVS_MODULES')
- _remove_module_group('DPDK_MODULES')
-
_DPDK_MODULE_MANAGER.remove_modules()
-def _remove_module_group(module_group):
- """Ensure all modules in a group are removed from the system.
+#
+# vhost specific modules management
+#
- :param module_group: A name of configuration item containing a list
- of module names
+def insert_vhost_modules():
+ """Inserts VHOST related kernel modules
"""
- for module in settings.getValue(module_group):
- # first check if module is loaded
- if not _is_module_inserted(module[1]):
- continue
+ mod_path_prefix = os.path.join(settings.getValue('RTE_SDK'),
+ 'lib',
+ 'librte_vhost')
+ _DPDK_MODULE_MANAGER.insert_module_group('VHOST_MODULE', mod_path_prefix)
- try:
- tasks.run_task(['sudo', 'rmmod', module[1]], _LOGGER,
- 'Removing module \'%s\'...' % module[1], True)
- except subprocess.CalledProcessError:
- _LOGGER.error('Unable to remove module \'%s\'.', module[1])
- continue
+
+def remove_vhost_modules():
+ """Removes all VHOST related kernel modules
+ """
+ _DPDK_MODULE_MANAGER.remove_module_group(settings.getValue('VHOST_MODULE'))
#
-# 'vhost-net' module management
+# 'vhost-net' module cleanup
#
def _remove_vhost_net():
"""Remove vhost-net driver and file.
"""
- if _is_module_inserted('vhost_net'):
- try:
- tasks.run_task(['sudo', 'rmmod', 'vhost_net'], _LOGGER,
- 'Removing \'/dev/vhost-net\' directory...', True)
- except subprocess.CalledProcessError:
- _LOGGER.error('Unable to remove module \'vhost_net\'.')
-
+ _DPDK_MODULE_MANAGER.remove_module('vhost-net')
try:
tasks.run_task(['sudo', 'rm', '-f', '/dev/vhost-net'], _LOGGER,
'Removing \'/dev/vhost-net\' directory...', True)
@@ -198,6 +135,26 @@ def _remove_vhost_net():
_LOGGER.error('Unable to remove directory \'/dev/vhost-net\'.')
#
+# Vhost-user cleanup
+#
+
+def _vhost_user_cleanup():
+ """Remove files created by vhost-user tests.
+ """
+ for sock in settings.getValue('VHOST_USER_SOCKS'):
+ if os.path.exists(sock):
+ try:
+ tasks.run_task(['sudo', 'rm', sock],
+ _LOGGER,
+ 'Deleting vhost-user socket \'%s\'...' %
+ sock,
+ True)
+
+ except subprocess.CalledProcessError:
+ _LOGGER.error('Unable to delete vhost-user socket \'%s\'.',
+ sock)
+ continue
+#
# NIC management
#
@@ -206,7 +163,17 @@ def _bind_nics():
"""Bind NICs using the Intel DPDK ``dpdk_nic_bind.py`` tool.
"""
try:
- tasks.run_task(['sudo', RTE_PCI_TOOL, '--bind', 'igb_uio'] +
+ _driver = 'igb_uio'
+ if 'vfio-pci' in settings.getValue('DPDK_MODULES'):
+ _driver = 'vfio-pci'
+ tasks.run_task(['sudo', 'chmod', 'a+x', '/dev/vfio'],
+ _LOGGER, 'Setting VFIO permissions .. a+x',
+ True)
+ tasks.run_task(['sudo', 'chmod', '-R', '666', '/dev/vfio/'],
+ _LOGGER, 'Setting VFIO permissions .. 0666',
+ True)
+
+ tasks.run_task(['sudo', RTE_PCI_TOOL, '--bind=' + _driver] +
settings.getValue('WHITELIST_NICS'), _LOGGER,
'Binding NICs %s...' %
settings.getValue('WHITELIST_NICS'),
@@ -256,30 +223,6 @@ def _unbind_nics():
str(settings.getValue('WHITELIST_NICS')),
nic_drivers)
-
-
-#
-# Vhost-user cleanup
-#
-
-def _vhost_user_cleanup():
- """Remove files created by vhost-user tests.
- """
- for sock in settings.getValue('VHOST_USER_SOCKS'):
- if os.path.exists(sock):
- try:
- tasks.run_task(['sudo', 'rm', sock],
- _LOGGER,
- 'Deleting vhost-user socket \'%s\'...' %
- sock,
- True)
-
- except subprocess.CalledProcessError:
- _LOGGER.error('Unable to delete vhost-user socket \'%s\'.',
- sock)
- continue
-
-
class Dpdk(object):
"""A context manager for the system init/cleanup.
"""
diff --git a/tools/module_manager.py b/tools/module_manager.py
index 39ad5cf4..565bac51 100644
--- a/tools/module_manager.py
+++ b/tools/module_manager.py
@@ -1,4 +1,4 @@
-# Copyright 2015 Intel Corporation.
+# Copyright 2015-2016 Intel Corporation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -14,11 +14,13 @@
"""Simple kernel module manager implementation.
"""
-
+import os
import subprocess
import logging
+
from tools import tasks
+_LOGGER = logging.getLogger(__name__)
class ModuleManager(object):
"""Simple module manager which acts as system wrapper for Kernel Modules.
"""
@@ -28,7 +30,7 @@ class ModuleManager(object):
def __init__(self):
"""Initializes data
"""
- self._modules = None
+ self._modules = []
def insert_modules(self, modules):
"""Method inserts list of modules. In case that module name ends
@@ -37,9 +39,9 @@ class ModuleManager(object):
:returns: None
"""
- self._modules = modules
+
for module in modules:
- if ModuleManager.is_module_inserted(module):
+ if self.is_module_inserted(module):
continue
try:
@@ -49,17 +51,39 @@ class ModuleManager(object):
else:
tasks.run_task(['sudo', 'modprobe', module], self._logger,
'Modprobe module \'%s\'...' % module, True)
-
+ _LOGGER.info("Inserted Module %s", module)
+ self._modules.append(module)
except subprocess.CalledProcessError:
self._logger.error('Unable to insert module \'%s\'.', module)
raise # fail catastrophically
+ def insert_module_group(self, module_group, group_path_prefix):
+ """Ensure all modules in a group are inserted into the system.
+
+ :param module_group: A name of configuration item containing a list
+ of module names
+ """
+ for module in module_group:
+ # first check if module is loaded
+ if self.is_module_inserted(module[1]):
+ continue
+
+ try:
+ mod_path = os.path.join(group_path_prefix, module[0],
+ '%s.ko' % module[1])
+ tasks.run_task(['sudo', 'insmod', mod_path], _LOGGER,
+ 'Inserting module \'%s\'...' % module[1], True)
+ self._modules.append(module)
+ except subprocess.CalledProcessError:
+ _LOGGER.error('Unable to insert module \'%s\'.', module[1])
+ raise # fail catastrophically
+
def remove_modules(self):
- """Removes all modules that have been previously instereted.
+ """Removes all modules that have been previously inserted.
"""
for module in self._modules:
# first check if module is loaded
- if not ModuleManager.is_module_inserted(module):
+ if not self.is_module_inserted(module):
continue
try:
@@ -67,16 +91,19 @@ class ModuleManager(object):
# with .ko suffix
tasks.run_task(['sudo', 'rmmod', module], self._logger,
'Removing module \'%s\'...' % module, True)
+ self._modules.remove(module)
except subprocess.CalledProcessError:
self._logger.error('Unable to remove module \'%s\'.', module)
continue
-
@staticmethod
def is_module_inserted(module):
"""Check if a module is inserted on system.
"""
+ if module.endswith('.ko'):
# get module base name, i.e strip path and .ko suffix if possible
- module_base_name = module.split('.')[0].split('/').pop()
+ module_base_name = os.path.basename(os.path.splitext(module)[0])
+ else:
+ module_base_name = module
# get list of modules from kernel
with open('/proc/modules') as mod_file:
@@ -87,3 +114,37 @@ class ModuleManager(object):
if line.startswith(module_base_name):
return True
return False
+
+ def remove_module(self, module):
+ """Removes a single module.
+ """
+ if self.is_module_inserted(module):
+ # get module base name, i.e strip path and .ko suffix if possible
+ module_base_name = os.path.basename(os.path.splitext(module)[0])
+
+ try:
+ # rmmod supports both simple module name and full module path
+ # with .ko suffix
+ tasks.run_task(['sudo', 'rmmod', module_base_name], self._logger,
+ 'Removing module \'%s\'...' % module, True)
+ self._modules.remove(module)
+ except subprocess.CalledProcessError:
+ self._logger.error('Unable to remove module \'%s\'.', module_base_name)
+
+ def remove_module_group(self, module_group):
+ """Removes all modules in the modules group.
+ """
+ for module in module_group:
+ if not self.is_module_inserted(module[1]):
+ continue
+ # get module base name, i.e strip path and .ko suffix if possible
+ module_base_name = os.path.basename(os.path.splitext(module)[0])
+
+ try:
+ # rmmod supports both simple module name and full module path
+ # with .ko suffix
+ tasks.run_task(['sudo', 'rmmod', module_base_name], self._logger,
+ 'Removing module \'%s\'...' % module, True)
+ self._modules.remove(module)
+ except subprocess.CalledProcessError:
+ self._logger.error('Unable to remove module \'%s\'.', module_base_name)