dpdk: enable vfio_pci support

Enable vfio_pci support for DPDK in the case where the end user
doesn't want to use igb_uio.
Changes:
* Use generic functions to strip the path and .ko extension from the
  module to be inserted/removed.
* Removed debug line in Modules_Manager.
* Removed commented code.
* Fixed all pylint issues.
JIRA: VSPERF-249

Change-Id: I69c9077735879bcbf7ce5c970c1ec53c219e9f90
Signed-off-by: Maryam Tahhan <maryam.tahhan@intel.com>
Reviewed-by: Martin Klozik <martinx.klozik@intel.com>
Reviewed-by: Al Morton <acmorton@att.com>
Reviewed-by: Christian Trautman <ctrautma@redhat.com>

4 files changed, 171 insertions, 124 deletions

diff --git a/conf/02_vswitch.conf b/conf/02_vswitch.conf
index b9862c92..f0475313 100644
--- a/conf/02_vswitch.conf
+++ b/conf/02_vswitch.conf
@@ -1,4 +1,4 @@
-# Copyright 2015 Intel Corporation.
+# Copyright 2015-2016 Intel Corporation.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -41,6 +41,10 @@ BLACKLIST_NICS = ['0000:09:00.0', '0000:09:00.1', '0000:09:00.2',
                   '0000:09:00.3']
 
 # for DPDK_MODULES the path is in reference to the build directory
+# To use vfio set
+# DPDK_MODULES = [
+#     ('vfio-pci'),
+# ]
 DPDK_MODULES = [
     ('kmod', 'igb_uio'),
 ]
@@ -50,6 +54,8 @@ VHOST_MODULE = [
 ]
 
 # list of modules that will be inserted using 'modprobe' on system init
+# To use vfio set
+# SYS_MODULES = ['cuse']
 SYS_MODULES = ['uio', 'cuse']
 
 # vhost character device file used by dpdkvhostport QemuWrap cases
diff --git a/docs/userguide/testusage.rst b/docs/userguide/testusage.rst
index c1f02843..098a9d95 100755
--- a/docs/userguide/testusage.rst
+++ b/docs/userguide/testusage.rst
@@ -309,6 +309,43 @@ To run tests using Vanilla OVS:
 
      $ ./vsperf --conf-file<path_to_custom_conf>/10_custom.conf
 
+Using vfio_pci with DPDK
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To use vfio with DPDK instead of igb_uio edit 'conf/02_vswitch.conf'
+with the following parameters:
+
+.. code-block:: console
+
+    DPDK_MODULES = [
+     ('vfio-pci'),
+    ]
+    SYS_MODULES = ['cuse']
+
+**NOTE:** Please ensure that Intel VT-d is enabled in BIOS.
+
+**NOTE:** Please ensure your boot/grub parameters include
+the following:
+
+.. code-block:: console
+
+    iommu=pt intel_iommu=on
+
+To check that IOMMU is enabled on your platform:
+
+.. code-block:: console
+
+    $ dmesg | grep IOMMU
+    [    0.000000] Intel-IOMMU: enabled
+    [    0.139882] dmar: IOMMU 0: reg_base_addr fbffe000 ver 1:0 cap d2078c106f0466 ecap f020de
+    [    0.139888] dmar: IOMMU 1: reg_base_addr ebffc000 ver 1:0 cap d2078c106f0466 ecap f020de
+    [    0.139893] IOAPIC id 2 under DRHD base  0xfbffe000 IOMMU 0
+    [    0.139894] IOAPIC id 0 under DRHD base  0xebffc000 IOMMU 1
+    [    0.139895] IOAPIC id 1 under DRHD base  0xebffc000 IOMMU 1
+    [    3.335744] IOMMU: dmar0 using Queued invalidation
+    [    3.335746] IOMMU: dmar1 using Queued invalidation
+    ....
+
 Selection of loopback application for PVP and PVVP tests
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/src/dpdk/dpdk.py b/src/dpdk/dpdk.py
index f2f6ec9f..127ecaf2 100644
--- a/src/dpdk/dpdk.py
+++ b/src/dpdk/dpdk.py
@@ -34,7 +34,6 @@ RTE_PCI_TOOL = os.path.join(
     settings.getValue('RTE_SDK'), 'tools', 'dpdk_nic_bind.py')
 
 _DPDK_MODULE_MANAGER = ModuleManager()
-
 #
 # system management
 #
@@ -46,7 +45,6 @@ def init():
     if not _is_linux():
         _LOGGER.error('Not running on a compatible Linux version. Exiting...')
         return
-
     _insert_modules()
     _remove_vhost_net()
     _bind_nics()
@@ -63,31 +61,10 @@ def cleanup():
     _remove_modules()
     _vhost_user_cleanup()
 
-
-#
-# vhost specific modules management
-#
-
-
-def insert_vhost_modules():
-    """Inserts VHOST related kernel modules
-    """
-    mod_path_prefix = os.path.join(settings.getValue('RTE_SDK'),
-                                   'lib',
-                                   'librte_vhost')
-    _insert_module_group('VHOST_MODULE', mod_path_prefix)
-
-
-def remove_vhost_modules():
-    """Removes all VHOST related kernel modules
-    """
-    _remove_module_group('VHOST_MODULE')
-
 #
 # basic compatibility test
 #
 
-
 def _is_linux():
     """Check if running on Linux.
 
@@ -102,20 +79,6 @@ def _is_linux():
 # module management
 #
 
-
-def _is_module_inserted(module):
-    """Check if a module is inserted on system.
-    """
-    with open('/proc/modules') as mod_file:
-        loaded_mods = mod_file.readlines()
-
-    # first check if module is loaded
-    for line in loaded_mods:
-        if line.startswith(module):
-            return True
-    return False
-
-
 def _insert_modules():
     """Ensure required modules are inserted on system.
     """
@@ -123,74 +86,48 @@ def _insert_modules():
     _DPDK_MODULE_MANAGER.insert_modules(settings.getValue('SYS_MODULES'))
 
     mod_path_prefix = settings.getValue('OVS_DIR')
-    _insert_module_group('OVS_MODULES', mod_path_prefix)
-    mod_path_prefix = os.path.join(settings.getValue('RTE_SDK'),
-                                   settings.getValue('RTE_TARGET'))
-    _insert_module_group('DPDK_MODULES', mod_path_prefix)
-
-
-def _insert_module_group(module_group, group_path_prefix):
-    """Ensure all modules in a group are inserted into the system.
-
-    :param module_group: A name of configuration item containing a list
-    of module names
-    """
-    for module in settings.getValue(module_group):
-        # first check if module is loaded
-        if _is_module_inserted(module[1]):
-            continue
-
-        try:
-            mod_path = os.path.join(group_path_prefix, module[0],
-                                    '%s.ko' % module[1])
-            tasks.run_task(['sudo', 'insmod', mod_path], _LOGGER,
-                           'Inserting module \'%s\'...' % module[1], True)
-        except subprocess.CalledProcessError:
-            _LOGGER.error('Unable to insert module \'%s\'.', module[1])
-            raise  # fail catastrophically
-
+    _DPDK_MODULE_MANAGER.insert_module_group(settings.getValue('OVS_MODULES'),
+                                             mod_path_prefix)
+    if 'vfio-pci' not in settings.getValue('DPDK_MODULES'):
+        mod_path_prefix = os.path.join(settings.getValue('RTE_SDK'),
+                                       settings.getValue('RTE_TARGET'))
+        _DPDK_MODULE_MANAGER.insert_module_group(settings.getValue('DPDK_MODULES'),
+                                                 mod_path_prefix)
+    else:
+        _DPDK_MODULE_MANAGER.insert_modules(settings.getValue('DPDK_MODULES'))
 
 def _remove_modules():
     """Ensure required modules are removed from system.
     """
-    _remove_module_group('OVS_MODULES')
-    _remove_module_group('DPDK_MODULES')
-
     _DPDK_MODULE_MANAGER.remove_modules()
 
-def _remove_module_group(module_group):
-    """Ensure all modules in a group are removed from the system.
+#
+# vhost specific modules management
+#
 
-    :param module_group: A name of configuration item containing a list
-    of module names
+def insert_vhost_modules():
+    """Inserts VHOST related kernel modules
     """
-    for module in settings.getValue(module_group):
-        # first check if module is loaded
-        if not _is_module_inserted(module[1]):
-            continue
+    mod_path_prefix = os.path.join(settings.getValue('RTE_SDK'),
+                                   'lib',
+                                   'librte_vhost')
+    _DPDK_MODULE_MANAGER.insert_module_group('VHOST_MODULE', mod_path_prefix)
 
-        try:
-            tasks.run_task(['sudo', 'rmmod', module[1]], _LOGGER,
-                           'Removing module \'%s\'...' % module[1], True)
-        except subprocess.CalledProcessError:
-            _LOGGER.error('Unable to remove module \'%s\'.', module[1])
-            continue
+
+def remove_vhost_modules():
+    """Removes all VHOST related kernel modules
+    """
+    _DPDK_MODULE_MANAGER.remove_module_group(settings.getValue('VHOST_MODULE'))
 
 
 #
-# 'vhost-net' module management
+# 'vhost-net' module cleanup
 #
 
 def _remove_vhost_net():
     """Remove vhost-net driver and file.
     """
-    if _is_module_inserted('vhost_net'):
-        try:
-            tasks.run_task(['sudo', 'rmmod', 'vhost_net'], _LOGGER,
-                           'Removing \'/dev/vhost-net\' directory...', True)
-        except subprocess.CalledProcessError:
-            _LOGGER.error('Unable to remove module \'vhost_net\'.')
-
+    _DPDK_MODULE_MANAGER.remove_module('vhost-net')
     try:
         tasks.run_task(['sudo', 'rm', '-f', '/dev/vhost-net'], _LOGGER,
                        'Removing \'/dev/vhost-net\' directory...', True)
@@ -198,6 +135,26 @@ def _remove_vhost_net():
         _LOGGER.error('Unable to remove directory \'/dev/vhost-net\'.')
 
 #
+# Vhost-user cleanup
+#
+
+def _vhost_user_cleanup():
+    """Remove files created by vhost-user tests.
+    """
+    for sock in settings.getValue('VHOST_USER_SOCKS'):
+        if os.path.exists(sock):
+            try:
+                tasks.run_task(['sudo', 'rm', sock],
+                               _LOGGER,
+                               'Deleting vhost-user socket \'%s\'...' %
+                               sock,
+                               True)
+
+            except subprocess.CalledProcessError:
+                _LOGGER.error('Unable to delete vhost-user socket \'%s\'.',
+                              sock)
+                continue
+#
 # NIC management
 #
 
@@ -206,7 +163,17 @@ def _bind_nics():
     """Bind NICs using the Intel DPDK ``dpdk_nic_bind.py`` tool.
     """
     try:
-        tasks.run_task(['sudo', RTE_PCI_TOOL, '--bind', 'igb_uio'] +
+        _driver = 'igb_uio'
+        if 'vfio-pci' in settings.getValue('DPDK_MODULES'):
+            _driver = 'vfio-pci'
+            tasks.run_task(['sudo', 'chmod', 'a+x', '/dev/vfio'],
+                           _LOGGER, 'Setting VFIO permissions .. a+x',
+                           True)
+            tasks.run_task(['sudo', 'chmod', '-R', '666', '/dev/vfio/'],
+                           _LOGGER, 'Setting VFIO permissions .. 0666',
+                           True)
+
+        tasks.run_task(['sudo', RTE_PCI_TOOL, '--bind=' + _driver] +
                        settings.getValue('WHITELIST_NICS'), _LOGGER,
                        'Binding NICs %s...' %
                        settings.getValue('WHITELIST_NICS'),
@@ -256,30 +223,6 @@ def _unbind_nics():
                           str(settings.getValue('WHITELIST_NICS')),
                           nic_drivers)
 
-
-
-#
-# Vhost-user cleanup
-#
-
-def _vhost_user_cleanup():
-    """Remove files created by vhost-user tests.
-    """
-    for sock in settings.getValue('VHOST_USER_SOCKS'):
-        if os.path.exists(sock):
-            try:
-                tasks.run_task(['sudo', 'rm', sock],
-                               _LOGGER,
-                               'Deleting vhost-user socket \'%s\'...' %
-                               sock,
-                               True)
-
-            except subprocess.CalledProcessError:
-                _LOGGER.error('Unable to delete vhost-user socket \'%s\'.',
-                              sock)
-                continue
-
-
 class Dpdk(object):
     """A context manager for the system init/cleanup.
     """
diff --git a/tools/module_manager.py b/tools/module_manager.py
index 39ad5cf4..565bac51 100644
--- a/tools/module_manager.py
+++ b/tools/module_manager.py
@@ -1,4 +1,4 @@
-# Copyright 2015 Intel Corporation.
+# Copyright 2015-2016 Intel Corporation.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,11 +14,13 @@
 
 """Simple kernel module manager implementation.
 """
-
+import os
 import subprocess
 import logging
+
 from tools import tasks
 
+_LOGGER = logging.getLogger(__name__)
 class ModuleManager(object):
     """Simple module manager which acts as system wrapper for Kernel Modules.
     """
@@ -28,7 +30,7 @@ class ModuleManager(object):
     def __init__(self):
         """Initializes data
         """
-        self._modules = None
+        self._modules = []
 
     def insert_modules(self, modules):
         """Method inserts list of modules. In case that module name ends
@@ -37,9 +39,9 @@ class ModuleManager(object):
 
         :returns: None
         """
-        self._modules = modules
+
         for module in modules:
-            if ModuleManager.is_module_inserted(module):
+            if self.is_module_inserted(module):
                 continue
 
             try:
@@ -49,17 +51,39 @@ class ModuleManager(object):
                 else:
                     tasks.run_task(['sudo', 'modprobe', module], self._logger,
                                    'Modprobe module \'%s\'...' % module, True)
-
+                _LOGGER.info("Inserted Module %s", module)
+                self._modules.append(module)
             except subprocess.CalledProcessError:
                 self._logger.error('Unable to insert module \'%s\'.', module)
                 raise  # fail catastrophically
 
+    def insert_module_group(self, module_group, group_path_prefix):
+        """Ensure all modules in a group are inserted into the system.
+
+        :param module_group: A name of configuration item containing a list
+        of module names
+        """
+        for module in module_group:
+            # first check if module is loaded
+            if self.is_module_inserted(module[1]):
+                continue
+
+            try:
+                mod_path = os.path.join(group_path_prefix, module[0],
+                                        '%s.ko' % module[1])
+                tasks.run_task(['sudo', 'insmod', mod_path], _LOGGER,
+                               'Inserting module \'%s\'...' % module[1], True)
+                self._modules.append(module)
+            except subprocess.CalledProcessError:
+                _LOGGER.error('Unable to insert module \'%s\'.', module[1])
+                raise  # fail catastrophically
+
     def remove_modules(self):
-        """Removes all modules that have been previously instereted.
+        """Removes all modules that have been previously inserted.
         """
         for module in self._modules:
             # first check if module is loaded
-            if not ModuleManager.is_module_inserted(module):
+            if not self.is_module_inserted(module):
                 continue
 
             try:
@@ -67,16 +91,19 @@ class ModuleManager(object):
                 # with .ko suffix
                 tasks.run_task(['sudo', 'rmmod', module], self._logger,
                                'Removing module \'%s\'...' % module, True)
+                self._modules.remove(module)
             except subprocess.CalledProcessError:
                 self._logger.error('Unable to remove module \'%s\'.', module)
                 continue
-
     @staticmethod
     def is_module_inserted(module):
         """Check if a module is inserted on system.
         """
+        if module.endswith('.ko'):
         # get module base name, i.e strip path and .ko suffix if possible
-        module_base_name = module.split('.')[0].split('/').pop()
+            module_base_name = os.path.basename(os.path.splitext(module)[0])
+        else:
+            module_base_name = module
 
         # get list of modules from kernel
         with open('/proc/modules') as mod_file:
@@ -87,3 +114,37 @@ class ModuleManager(object):
             if line.startswith(module_base_name):
                 return True
         return False
+
+    def remove_module(self, module):
+        """Removes a single module.
+        """
+        if self.is_module_inserted(module):
+            # get module base name, i.e strip path and .ko suffix if possible
+            module_base_name = os.path.basename(os.path.splitext(module)[0])
+
+            try:
+                # rmmod supports both simple module name and full module path
+                # with .ko suffix
+                tasks.run_task(['sudo', 'rmmod', module_base_name], self._logger,
+                               'Removing module \'%s\'...' % module, True)
+                self._modules.remove(module)
+            except subprocess.CalledProcessError:
+                self._logger.error('Unable to remove module \'%s\'.', module_base_name)
+
+    def remove_module_group(self, module_group):
+        """Removes all modules in the modules group.
+        """
+        for module in module_group:
+            if not self.is_module_inserted(module[1]):
+                continue
+            # get module base name, i.e strip path and .ko suffix if possible
+            module_base_name = os.path.basename(os.path.splitext(module)[0])
+
+            try:
+                # rmmod supports both simple module name and full module path
+                # with .ko suffix
+                tasks.run_task(['sudo', 'rmmod', module_base_name], self._logger,
+                               'Removing module \'%s\'...' % module, True)
+                self._modules.remove(module)
+            except subprocess.CalledProcessError:
+                self._logger.error('Unable to remove module \'%s\'.', module_base_name)