sriov: Support of SRIOV and Qemu PCI passthrough

Generic support of SRIOV has been added. Virtual interfaces can be
used in multiplei scenarios instead of physical NICs. Virtual
functions can be directly accessed from VM by PCI passthrough method.
Another option is to use VFs with vSwtich to evaluate impact
on performance.
Additonal modifications:
* Automatic detection of NIC details has been added to simplify
configuration.
* Obsoleted configuration options have been removed.
* Logging usage within vsperf script was fixed.
* Vsperf main was refactored and final cleanup function added.
* Configurable forwarding mode of TestPMD executed inside VM.

JIRA: VSPERF-198
Change-Id: I4a0d5d262b245d433b12419de79399fb5825a623
Signed-off-by: Martin Klozik <martinx.klozik@intel.com>
Reviewed-by: Maryam Tahhan <maryam.tahhan@intel.com>
Reviewed-by: Al Morton <acmorton@att.com>
Reviewed-by: Christian Trautman <ctrautma@redhat.com>
Reviewed-by: Brian Castelli <brian.castelli@spirent.com>

2 files changed, 107 insertions, 2 deletions

diff --git a/vnfs/qemu/qemu.py b/vnfs/qemu/qemu.py
index ec728c5c..9cb23ac6 100644
--- a/vnfs/qemu/qemu.py
+++ b/vnfs/qemu/qemu.py
@@ -66,6 +66,14 @@ class IVnfQemu(IVnf):
         # cli option take precedence to config file values
         self._guest_loopback = S.getValue('GUEST_LOOPBACK')[self._number]
 
+        self._testpmd_fwd_mode = S.getValue('GUEST_TESTPMD_FWD_MODE')
+        # in case of SRIOV we must ensure, that MAC addresses are not swapped
+        if S.getValue('SRIOV_ENABLED') and self._testpmd_fwd_mode.startswith('mac') and \
+           not S.getValue('VNF').endswith('PciPassthrough'):
+
+            self._logger.info("SRIOV detected, forwarding mode of testpmd was changed from '%s' to '%s'",
+                              self._testpmd_fwd_mode, 'io')
+            self._testpmd_fwd_mode = 'io'
 
         name = 'Client%d' % self._number
         vnc = ':%d' % self._number
@@ -306,6 +314,11 @@ class IVnfQemu(IVnf):
         # modify makefile if needed
         self._modify_dpdk_makefile()
 
+        # disable network interfaces, so DPDK can take care of them
+        self.execute_and_wait('ifdown ' + self._net1)
+        self.execute_and_wait('ifdown ' + self._net2)
+
+        # build and insert igb_uio and rebind interfaces to it
         self.execute_and_wait('make RTE_OUTPUT=$RTE_SDK/$RTE_TARGET -C '
                               '$RTE_SDK/lib/librte_eal/linuxapp/igb_uio')
         self.execute_and_wait('modprobe uio')
@@ -313,9 +326,14 @@ class IVnfQemu(IVnf):
                               S.getValue('RTE_TARGET'))
         self.execute_and_wait('./tools/dpdk_nic_bind.py --status')
         self.execute_and_wait(
+            './tools/dpdk_nic_bind.py -u' ' ' +
+            S.getValue('GUEST_NET1_PCI_ADDRESS')[self._number] + ' ' +
+            S.getValue('GUEST_NET2_PCI_ADDRESS')[self._number])
+        self.execute_and_wait(
             './tools/dpdk_nic_bind.py -b igb_uio' ' ' +
             S.getValue('GUEST_NET1_PCI_ADDRESS')[self._number] + ' ' +
             S.getValue('GUEST_NET2_PCI_ADDRESS')[self._number])
+        self.execute_and_wait('./tools/dpdk_nic_bind.py --status')
 
         # build and run 'test-pmd'
         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
@@ -325,9 +343,9 @@ class IVnfQemu(IVnf):
         self.execute_and_wait('./testpmd -c 0x3 -n 4 --socket-mem 512 --'
                               ' --burst=64 -i --txqflags=0xf00 ' +
                               '--disable-hw-vlan', 60, "Done")
-        self.execute('set fwd mac_retry', 1)
+        self.execute('set fwd ' + self._testpmd_fwd_mode, 1)
         self.execute_and_wait('start', 20,
-                              'TX RS bit threshold=0 - TXQ flags=0xf00')
+                              'TX RS bit threshold=.+ - TXQ flags=0xf00')
 
     def _configure_l2fwd(self):
         """
diff --git a/vnfs/qemu/qemu_pci_passthrough.py b/vnfs/qemu/qemu_pci_passthrough.py
new file mode 100644
index 00000000..1b55fdf2
--- /dev/null
+++ b/vnfs/qemu/qemu_pci_passthrough.py
@@ -0,0 +1,87 @@
+# Copyright 2015 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Automation of QEMU hypervisor with direct access to host NICs via
+   PCI passthrough.
+"""
+
+import logging
+import subprocess
+import os
+
+from conf import settings as S
+from vnfs.qemu.qemu import IVnfQemu
+from tools import tasks
+from tools.module_manager import ModuleManager
+
+_MODULE_MANAGER = ModuleManager()
+_RTE_PCI_TOOL = os.path.join(S.getValue('RTE_SDK'), 'tools', 'dpdk_nic_bind.py')
+
+class QemuPciPassthrough(IVnfQemu):
+    """
+    Control an instance of QEMU with direct access to the host network devices
+    """
+    def __init__(self):
+        """
+        Initialization function.
+        """
+        super(QemuPciPassthrough, self).__init__()
+        self._logger = logging.getLogger(__name__)
+        self._nics = S.getValue('NICS')
+
+        # in case of SRIOV and PCI passthrough we must ensure, that MAC addresses are swapped
+        if S.getValue('SRIOV_ENABLED') and not self._testpmd_fwd_mode.startswith('mac'):
+            self._logger.info("SRIOV detected, forwarding mode of testpmd was changed from '%s' to '%s'",
+                              self._testpmd_fwd_mode, 'mac_retry')
+            self._testpmd_fwd_mode = 'mac_retry'
+
+        for nic in self._nics:
+            self._cmd += ['-device', 'vfio-pci,host=' + nic['pci']]
+
+    def start(self):
+        """
+        Start QEMU instance, bind host NICs to vfio-pci driver
+        """
+        # load vfio-pci
+        _MODULE_MANAGER.insert_modules(['vfio-pci'])
+
+        # bind every interface to vfio-pci driver
+        try:
+            nics_list = list(tmp_nic['pci'] for tmp_nic in self._nics)
+            tasks.run_task(['sudo', _RTE_PCI_TOOL, '--bind=vfio-pci'] + nics_list,
+                           self._logger, 'Binding NICs %s...' % nics_list, True)
+
+        except subprocess.CalledProcessError:
+            self._logger.error('Unable to bind NICs %s', self._nics)
+
+        super(QemuPciPassthrough, self).start()
+
+    def stop(self):
+        """
+        Stop QEMU instance, bind host NICs to the original driver
+        """
+        super(QemuPciPassthrough, self).stop()
+
+        # bind original driver to every interface
+        for nic in self._nics:
+            if nic['driver']:
+                try:
+                    tasks.run_task(['sudo', _RTE_PCI_TOOL, '--bind=' + nic['driver'], nic['pci']],
+                                   self._logger, 'Binding NIC %s...' % nic['pci'], True)
+
+                except subprocess.CalledProcessError:
+                    self._logger.error('Unable to bind NIC %s to driver %s', nic['pci'], nic['driver'])
+
+        # unload vfio-pci driver
+        _MODULE_MANAGER.remove_modules()