diff options
| -rw-r--r-- | conf/02_vswitch.conf | 8 | ||||
| -rwxr-xr-x | docs/userguide/testusage.rst | 37 | ||||
| -rw-r--r-- | src/dpdk/dpdk.py | 169 | ||||
| -rw-r--r-- | tools/module_manager.py | 81 |
4 files changed, 171 insertions, 124 deletions
diff --git a/conf/02_vswitch.conf b/conf/02_vswitch.conf index b9862c92..f0475313 100644 --- a/conf/02_vswitch.conf +++ b/conf/02_vswitch.conf @@ -1,4 +1,4 @@ -# Copyright 2015 Intel Corporation. +# Copyright 2015-2016 Intel Corporation. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -41,6 +41,10 @@ BLACKLIST_NICS = ['0000:09:00.0', '0000:09:00.1', '0000:09:00.2', '0000:09:00.3'] # for DPDK_MODULES the path is in reference to the build directory +# To use vfio set +# DPDK_MODULES = [ +# ('vfio-pci'), +# ] DPDK_MODULES = [ ('kmod', 'igb_uio'), ] @@ -50,6 +54,8 @@ VHOST_MODULE = [ ] # list of modules that will be inserted using 'modprobe' on system init +# To use vfio set +# SYS_MODULES = ['cuse'] SYS_MODULES = ['uio', 'cuse'] # vhost character device file used by dpdkvhostport QemuWrap cases diff --git a/docs/userguide/testusage.rst b/docs/userguide/testusage.rst index c1f02843..098a9d95 100755 --- a/docs/userguide/testusage.rst +++ b/docs/userguide/testusage.rst @@ -309,6 +309,43 @@ To run tests using Vanilla OVS: $ ./vsperf --conf-file<path_to_custom_conf>/10_custom.conf +Using vfio_pci with DPDK +^^^^^^^^^^^^^^^^^^^^^^^^^ + +To use vfio with DPDK instead of igb_uio edit 'conf/02_vswitch.conf' +with the following parameters: + +.. code-block:: console + + DPDK_MODULES = [ + ('vfio-pci'), + ] + SYS_MODULES = ['cuse'] + +**NOTE:** Please ensure that Intel VT-d is enabled in BIOS. + +**NOTE:** Please ensure your boot/grub parameters include +the following: + +.. code-block:: console + + iommu=pt intel_iommu=on + +To check that IOMMU is enabled on your platform: + +.. code-block:: console + + $ dmesg | grep IOMMU + [ 0.000000] Intel-IOMMU: enabled + [ 0.139882] dmar: IOMMU 0: reg_base_addr fbffe000 ver 1:0 cap d2078c106f0466 ecap f020de + [ 0.139888] dmar: IOMMU 1: reg_base_addr ebffc000 ver 1:0 cap d2078c106f0466 ecap f020de + [ 0.139893] IOAPIC id 2 under DRHD base 0xfbffe000 IOMMU 0 + [ 0.139894] IOAPIC id 0 under DRHD base 0xebffc000 IOMMU 1 + [ 0.139895] IOAPIC id 1 under DRHD base 0xebffc000 IOMMU 1 + [ 3.335744] IOMMU: dmar0 using Queued invalidation + [ 3.335746] IOMMU: dmar1 using Queued invalidation + .... + Selection of loopback application for PVP and PVVP tests ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/src/dpdk/dpdk.py b/src/dpdk/dpdk.py index f2f6ec9f..127ecaf2 100644 --- a/src/dpdk/dpdk.py +++ b/src/dpdk/dpdk.py @@ -34,7 +34,6 @@ RTE_PCI_TOOL = os.path.join( settings.getValue('RTE_SDK'), 'tools', 'dpdk_nic_bind.py') _DPDK_MODULE_MANAGER = ModuleManager() - # # system management # @@ -46,7 +45,6 @@ def init(): if not _is_linux(): _LOGGER.error('Not running on a compatible Linux version. Exiting...') return - _insert_modules() _remove_vhost_net() _bind_nics() @@ -63,31 +61,10 @@ def cleanup(): _remove_modules() _vhost_user_cleanup() - -# -# vhost specific modules management -# - - -def insert_vhost_modules(): - """Inserts VHOST related kernel modules - """ - mod_path_prefix = os.path.join(settings.getValue('RTE_SDK'), - 'lib', - 'librte_vhost') - _insert_module_group('VHOST_MODULE', mod_path_prefix) - - -def remove_vhost_modules(): - """Removes all VHOST related kernel modules - """ - _remove_module_group('VHOST_MODULE') - # # basic compatibility test # - def _is_linux(): """Check if running on Linux. @@ -102,20 +79,6 @@ def _is_linux(): # module management # - -def _is_module_inserted(module): - """Check if a module is inserted on system. - """ - with open('/proc/modules') as mod_file: - loaded_mods = mod_file.readlines() - - # first check if module is loaded - for line in loaded_mods: - if line.startswith(module): - return True - return False - - def _insert_modules(): """Ensure required modules are inserted on system. """ @@ -123,74 +86,48 @@ def _insert_modules(): _DPDK_MODULE_MANAGER.insert_modules(settings.getValue('SYS_MODULES')) mod_path_prefix = settings.getValue('OVS_DIR') - _insert_module_group('OVS_MODULES', mod_path_prefix) - mod_path_prefix = os.path.join(settings.getValue('RTE_SDK'), - settings.getValue('RTE_TARGET')) - _insert_module_group('DPDK_MODULES', mod_path_prefix) - - -def _insert_module_group(module_group, group_path_prefix): - """Ensure all modules in a group are inserted into the system. - - :param module_group: A name of configuration item containing a list - of module names - """ - for module in settings.getValue(module_group): - # first check if module is loaded - if _is_module_inserted(module[1]): - continue - - try: - mod_path = os.path.join(group_path_prefix, module[0], - '%s.ko' % module[1]) - tasks.run_task(['sudo', 'insmod', mod_path], _LOGGER, - 'Inserting module \'%s\'...' % module[1], True) - except subprocess.CalledProcessError: - _LOGGER.error('Unable to insert module \'%s\'.', module[1]) - raise # fail catastrophically - + _DPDK_MODULE_MANAGER.insert_module_group(settings.getValue('OVS_MODULES'), + mod_path_prefix) + if 'vfio-pci' not in settings.getValue('DPDK_MODULES'): + mod_path_prefix = os.path.join(settings.getValue('RTE_SDK'), + settings.getValue('RTE_TARGET')) + _DPDK_MODULE_MANAGER.insert_module_group(settings.getValue('DPDK_MODULES'), + mod_path_prefix) + else: + _DPDK_MODULE_MANAGER.insert_modules(settings.getValue('DPDK_MODULES')) def _remove_modules(): """Ensure required modules are removed from system. """ - _remove_module_group('OVS_MODULES') - _remove_module_group('DPDK_MODULES') - _DPDK_MODULE_MANAGER.remove_modules() -def _remove_module_group(module_group): - """Ensure all modules in a group are removed from the system. +# +# vhost specific modules management +# - :param module_group: A name of configuration item containing a list - of module names +def insert_vhost_modules(): + """Inserts VHOST related kernel modules """ - for module in settings.getValue(module_group): - # first check if module is loaded - if not _is_module_inserted(module[1]): - continue + mod_path_prefix = os.path.join(settings.getValue('RTE_SDK'), + 'lib', + 'librte_vhost') + _DPDK_MODULE_MANAGER.insert_module_group('VHOST_MODULE', mod_path_prefix) - try: - tasks.run_task(['sudo', 'rmmod', module[1]], _LOGGER, - 'Removing module \'%s\'...' % module[1], True) - except subprocess.CalledProcessError: - _LOGGER.error('Unable to remove module \'%s\'.', module[1]) - continue + +def remove_vhost_modules(): + """Removes all VHOST related kernel modules + """ + _DPDK_MODULE_MANAGER.remove_module_group(settings.getValue('VHOST_MODULE')) # -# 'vhost-net' module management +# 'vhost-net' module cleanup # def _remove_vhost_net(): """Remove vhost-net driver and file. """ - if _is_module_inserted('vhost_net'): - try: - tasks.run_task(['sudo', 'rmmod', 'vhost_net'], _LOGGER, - 'Removing \'/dev/vhost-net\' directory...', True) - except subprocess.CalledProcessError: - _LOGGER.error('Unable to remove module \'vhost_net\'.') - + _DPDK_MODULE_MANAGER.remove_module('vhost-net') try: tasks.run_task(['sudo', 'rm', '-f', '/dev/vhost-net'], _LOGGER, 'Removing \'/dev/vhost-net\' directory...', True) @@ -198,6 +135,26 @@ def _remove_vhost_net(): _LOGGER.error('Unable to remove directory \'/dev/vhost-net\'.') # +# Vhost-user cleanup +# + +def _vhost_user_cleanup(): + """Remove files created by vhost-user tests. + """ + for sock in settings.getValue('VHOST_USER_SOCKS'): + if os.path.exists(sock): + try: + tasks.run_task(['sudo', 'rm', sock], + _LOGGER, + 'Deleting vhost-user socket \'%s\'...' % + sock, + True) + + except subprocess.CalledProcessError: + _LOGGER.error('Unable to delete vhost-user socket \'%s\'.', + sock) + continue +# # NIC management # @@ -206,7 +163,17 @@ def _bind_nics(): """Bind NICs using the Intel DPDK ``dpdk_nic_bind.py`` tool. """ try: - tasks.run_task(['sudo', RTE_PCI_TOOL, '--bind', 'igb_uio'] + + _driver = 'igb_uio' + if 'vfio-pci' in settings.getValue('DPDK_MODULES'): + _driver = 'vfio-pci' + tasks.run_task(['sudo', 'chmod', 'a+x', '/dev/vfio'], + _LOGGER, 'Setting VFIO permissions .. a+x', + True) + tasks.run_task(['sudo', 'chmod', '-R', '666', '/dev/vfio/'], + _LOGGER, 'Setting VFIO permissions .. 0666', + True) + + tasks.run_task(['sudo', RTE_PCI_TOOL, '--bind=' + _driver] + settings.getValue('WHITELIST_NICS'), _LOGGER, 'Binding NICs %s...' % settings.getValue('WHITELIST_NICS'), @@ -256,30 +223,6 @@ def _unbind_nics(): str(settings.getValue('WHITELIST_NICS')), nic_drivers) - - -# -# Vhost-user cleanup -# - -def _vhost_user_cleanup(): - """Remove files created by vhost-user tests. - """ - for sock in settings.getValue('VHOST_USER_SOCKS'): - if os.path.exists(sock): - try: - tasks.run_task(['sudo', 'rm', sock], - _LOGGER, - 'Deleting vhost-user socket \'%s\'...' % - sock, - True) - - except subprocess.CalledProcessError: - _LOGGER.error('Unable to delete vhost-user socket \'%s\'.', - sock) - continue - - class Dpdk(object): """A context manager for the system init/cleanup. """ diff --git a/tools/module_manager.py b/tools/module_manager.py index 39ad5cf4..565bac51 100644 --- a/tools/module_manager.py +++ b/tools/module_manager.py @@ -1,4 +1,4 @@ -# Copyright 2015 Intel Corporation. +# Copyright 2015-2016 Intel Corporation. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,11 +14,13 @@ """Simple kernel module manager implementation. """ - +import os import subprocess import logging + from tools import tasks +_LOGGER = logging.getLogger(__name__) class ModuleManager(object): """Simple module manager which acts as system wrapper for Kernel Modules. """ @@ -28,7 +30,7 @@ class ModuleManager(object): def __init__(self): """Initializes data """ - self._modules = None + self._modules = [] def insert_modules(self, modules): """Method inserts list of modules. In case that module name ends @@ -37,9 +39,9 @@ class ModuleManager(object): :returns: None """ - self._modules = modules + for module in modules: - if ModuleManager.is_module_inserted(module): + if self.is_module_inserted(module): continue try: @@ -49,17 +51,39 @@ class ModuleManager(object): else: tasks.run_task(['sudo', 'modprobe', module], self._logger, 'Modprobe module \'%s\'...' % module, True) - + _LOGGER.info("Inserted Module %s", module) + self._modules.append(module) except subprocess.CalledProcessError: self._logger.error('Unable to insert module \'%s\'.', module) raise # fail catastrophically + def insert_module_group(self, module_group, group_path_prefix): + """Ensure all modules in a group are inserted into the system. + + :param module_group: A name of configuration item containing a list + of module names + """ + for module in module_group: + # first check if module is loaded + if self.is_module_inserted(module[1]): + continue + + try: + mod_path = os.path.join(group_path_prefix, module[0], + '%s.ko' % module[1]) + tasks.run_task(['sudo', 'insmod', mod_path], _LOGGER, + 'Inserting module \'%s\'...' % module[1], True) + self._modules.append(module) + except subprocess.CalledProcessError: + _LOGGER.error('Unable to insert module \'%s\'.', module[1]) + raise # fail catastrophically + def remove_modules(self): - """Removes all modules that have been previously instereted. + """Removes all modules that have been previously inserted. """ for module in self._modules: # first check if module is loaded - if not ModuleManager.is_module_inserted(module): + if not self.is_module_inserted(module): continue try: @@ -67,16 +91,19 @@ class ModuleManager(object): # with .ko suffix tasks.run_task(['sudo', 'rmmod', module], self._logger, 'Removing module \'%s\'...' % module, True) + self._modules.remove(module) except subprocess.CalledProcessError: self._logger.error('Unable to remove module \'%s\'.', module) continue - @staticmethod def is_module_inserted(module): """Check if a module is inserted on system. """ + if module.endswith('.ko'): # get module base name, i.e strip path and .ko suffix if possible - module_base_name = module.split('.')[0].split('/').pop() + module_base_name = os.path.basename(os.path.splitext(module)[0]) + else: + module_base_name = module # get list of modules from kernel with open('/proc/modules') as mod_file: @@ -87,3 +114,37 @@ class ModuleManager(object): if line.startswith(module_base_name): return True return False + + def remove_module(self, module): + """Removes a single module. + """ + if self.is_module_inserted(module): + # get module base name, i.e strip path and .ko suffix if possible + module_base_name = os.path.basename(os.path.splitext(module)[0]) + + try: + # rmmod supports both simple module name and full module path + # with .ko suffix + tasks.run_task(['sudo', 'rmmod', module_base_name], self._logger, + 'Removing module \'%s\'...' % module, True) + self._modules.remove(module) + except subprocess.CalledProcessError: + self._logger.error('Unable to remove module \'%s\'.', module_base_name) + + def remove_module_group(self, module_group): + """Removes all modules in the modules group. + """ + for module in module_group: + if not self.is_module_inserted(module[1]): + continue + # get module base name, i.e strip path and .ko suffix if possible + module_base_name = os.path.basename(os.path.splitext(module)[0]) + + try: + # rmmod supports both simple module name and full module path + # with .ko suffix + tasks.run_task(['sudo', 'rmmod', module_base_name], self._logger, + 'Removing module \'%s\'...' % module, True) + self._modules.remove(module) + except subprocess.CalledProcessError: + self._logger.error('Unable to remove module \'%s\'.', module_base_name) |