vnfs: Enable PVP using vhost-cuse

Enable PVP testing using vhost-cuse as guest access method

Recent changes:
* Move NEWS.md changes to NEWS.rst
* Update comment to vhost-cuse
* Restore config back to checkout state after make
* Merge OPNFV 1092 updates

* Add PVP-vhost-cuse to NEWS.md
* Add comment/example to GUEST_CORE_BINDING
* Move hardcoded values to conf/04_vnfs.conf
* Set default VNF to QemuDpdkVhostCuse
* Compile eventfd_link if VHOST_USER=n
* Use MAC and PCI address from conf
* Use vhost method in conf to properly set interface type

JIRA: VSPERF-59

Change-Id: Ib1159e216f3e25b9971c0935969680582683916b
Signed-off-by: Madarang, Dino Simeon <dino.simeonx.madarang@intel.com>
Reviewed-by: Maryam Tahhan <maryam.tahhan@intel.com>

6 files changed, 425 insertions, 3 deletions

diff --git a/conf/02_vswitch.conf b/conf/02_vswitch.conf
index 48bf5964..851b6d62 100644
--- a/conf/02_vswitch.conf
+++ b/conf/02_vswitch.conf
@@ -80,3 +80,4 @@ LOG_FILE_OVS = 'ovs.log'
 VSWITCH_DIR = os.path.join(ROOT_DIR, 'vswitches')
 VSWITCH = "OvsDpdkVhost"
 
+VHOST_METHOD = 'cuse'
diff --git a/conf/04_vnf.conf b/conf/04_vnf.conf
index 7d1399d1..1c3712ed 100644
--- a/conf/04_vnf.conf
+++ b/conf/04_vnf.conf
@@ -74,3 +74,17 @@ GUEST_NET2_MAC = '00:00:00:00:00:02'
 GUEST_NET1_PCI_ADDRESS = '00:04.0'
 GUEST_NET2_PCI_ADDRESS = '00:05.0'
 
+GUEST_MEMORY = '3072'
+
+# test-pmd requires 2 VM cores
+GUEST_SMP = '2'
+
+# Host cores to use to affinitize the SMP cores of a QEMU instance
+# For 2 VNFs you may use [(4,5), (6, 7)]
+GUEST_CORE_BINDING = [(4, 5)]
+
+# Starte Qemu on cores 3, 4,5 (0x038)
+QEMU_CORE = '38'
+
+GUEST_OVS_DPDK_DIR = '/root/ovs_dpdk'
+OVS_DPDK_SHARE = '/mnt/ovs_dpdk_share'
diff --git a/docs/NEWS.rst b/docs/NEWS.rst
index 8c7ecaaa..223d3c8a 100644
--- a/docs/NEWS.rst
+++ b/docs/NEWS.rst
@@ -3,6 +3,7 @@ August 2015
 New
 ---
 - Backport and enhancement of reporting
+- PVP deployment scenario testing using vhost-cuse as guest access method
 
 July 2015
 =========
diff --git a/src/dpdk/Makefile b/src/dpdk/Makefile
index 8335ed4e..71839e26 100755
--- a/src/dpdk/Makefile
+++ b/src/dpdk/Makefile
@@ -52,6 +52,9 @@ force_make: $(TAG_DONE_FLAG)
 	$(AT)sed -i -e 's/CONFIG_RTE_LIBRTE_KNI=./CONFIG_RTE_LIBRTE_KNI=n/g' $(CONFIG_FILE)
 	$(AT)cd $(WORK_DIR); make install T=$(DPDK_TARGET) -j
 	$(AT)cd `dirname $(CONFIG_FILE)` && git checkout `basename $(CONFIG_FILE)` && cd -
+ifeq ($(VHOST_USER),n)
+	$(AT)cd $(WORK_DIR)/lib/librte_vhost/eventfd_link; make
+endif
 	@echo "Make done"
 
 install: $(INSTALL_TARGET)
diff --git a/vnfs/qemu/qemu_dpdk_vhost_cuse.py b/vnfs/qemu/qemu_dpdk_vhost_cuse.py
new file mode 100644
index 00000000..43b732fc
--- /dev/null
+++ b/vnfs/qemu/qemu_dpdk_vhost_cuse.py
@@ -0,0 +1,388 @@
+# Copyright 2015 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Automation of QEMU hypervisor for launching vhost-cuse enabled guests.
+"""
+
+import os
+import time
+import logging
+import locale
+import re
+import subprocess
+
+from tools import tasks
+from conf import settings
+from vnfs.vnf.vnf import IVnf
+
+_QEMU_BIN = settings.getValue('QEMU_BIN')
+_RTE_TARGET = settings.getValue('RTE_TARGET')
+
+_GUEST_MEMORY = settings.getValue('GUEST_MEMORY')
+_GUEST_SMP = settings.getValue('GUEST_SMP')
+_GUEST_CORE_BINDING = settings.getValue('GUEST_CORE_BINDING')
+_QEMU_CORE = settings.getValue('QEMU_CORE')
+
+_GUEST_IMAGE = settings.getValue('GUEST_IMAGE')
+_GUEST_SHARE_DIR = settings.getValue('GUEST_SHARE_DIR')
+
+_GUEST_USERNAME = settings.getValue('GUEST_USERNAME')
+_GUEST_PASSWORD = settings.getValue('GUEST_PASSWORD')
+
+_GUEST_PROMPT_LOGIN = settings.getValue('GUEST_PROMPT_LOGIN')
+_GUEST_PROMPT_PASSWORD = settings.getValue('GUEST_PROMPT_PASSWORD')
+_GUEST_PROMPT = settings.getValue('GUEST_PROMPT')
+
+_QEMU_GUEST_DPDK_PROMPT = settings.getValue('QEMU_GUEST_DPDK_PROMPT')
+_QEMU_GUEST_TEST_PMD_PROMPT = settings.getValue('QEMU_GUEST_TEST_PMD_PROMPT')
+_HUGEPAGE_DIR = settings.getValue('HUGEPAGE_DIR')
+
+_GUEST_OVS_DPDK_DIR = settings.getValue('GUEST_OVS_DPDK_DIR')
+_OVS_DPDK_SHARE = settings.getValue('OVS_DPDK_SHARE')
+
+_LOG_FILE_QEMU = os.path.join(
+    settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_QEMU'))
+_LOG_FILE_GUEST_CMDS = os.path.join(
+    settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_GUEST_CMDS'))
+
+_OVS_VAR_DIR = settings.getValue('OVS_VAR_DIR')
+_GUEST_NET1_MAC = settings.getValue('GUEST_NET1_MAC')
+_GUEST_NET2_MAC = settings.getValue('GUEST_NET2_MAC')
+_GUEST_NET1_PCI_ADDRESS = settings.getValue('GUEST_NET1_PCI_ADDRESS')
+_GUEST_NET2_PCI_ADDRESS = settings.getValue('GUEST_NET2_PCI_ADDRESS')
+
+class QemuDpdkVhostCuse(tasks.Process, IVnf):
+    """
+    Control an instance of QEMU with vHost user guest communication.
+    """
+    _bin = _QEMU_BIN
+    _logfile = _LOG_FILE_QEMU
+    _cmd = None
+    _expect = _GUEST_PROMPT_LOGIN
+    _proc_name = 'qemu'
+    _number_vnfs = 0
+
+    class GuestCommandFilter(logging.Filter):
+        """
+        Filter out strings beginning with 'guestcmd :'.
+        """
+        def filter(self, record):
+            return record.getMessage().startswith(self.prefix)
+
+    def __init__(self, memory=_GUEST_MEMORY, cpus=_GUEST_SMP,
+                 monitor_path='/tmp', shared_path_host=_GUEST_SHARE_DIR,
+                 args='', timeout=120, deployment="PVP"):
+        """
+        Initialisation function.
+
+        :param timeout: Time to wait for login prompt. If set to
+            0 do not wait.
+        :param number: Number of QEMU instance, used when multiple QEMU
+            instances are started at once.
+        :param args: Arguments to pass to QEMU.
+
+        :returns: None
+        """
+        self._logger = logging.getLogger(__name__)
+        self._number = self._number_vnfs
+        self._number_vnfs = self._number_vnfs + 1
+        self._logfile = self._logfile + str(self._number)
+        self._log_prefix = 'guest_%d_cmd : ' % self._number
+        self._timeout = timeout
+        self._monitor = '%s/vm%dmonitor' % (monitor_path, self._number)
+
+        name = 'Client%d' % self._number
+        vnc = ':%d' % self._number
+        self._cmd = ['sudo', '-E', 'taskset ' + str(_QEMU_CORE), self._bin,
+                     '-m', str(memory), '-smp', str(cpus), '-cpu', 'host',
+                     '-drive', 'if=scsi,file='+_GUEST_IMAGE,
+                     '-drive',
+                     'if=scsi,file=fat:rw:%s,snapshot=off' % shared_path_host,
+                     '-boot', 'c', '--enable-kvm',
+                     '-monitor', 'unix:%s,server,nowait' % self._monitor,
+                     '-object',
+                     'memory-backend-file,id=mem,size=' + str(memory) + 'M,' +
+                     'mem-path=' + _HUGEPAGE_DIR + ',share=on',
+                     '-numa', 'node,memdev=mem -mem-prealloc',
+                     '-net', 'none', '-no-reboot',
+                     '-netdev',
+                     'type=tap,id=net1,script=no,downscript=no,' +
+                     'ifname=dpdkvhostcuse0,vhost=on',
+                     '-device',
+                     'virtio-net-pci,netdev=net1,mac=' + _GUEST_NET1_MAC,
+                     '-netdev',
+                     'type=tap,id=net2,script=no,downscript=no,' +
+                     'ifname=dpdkvhostcuse1,vhost=on',
+                     '-device',
+                     'virtio-net-pci,netdev=net2,mac=' + _GUEST_NET2_MAC,
+                     '-nographic', '-vnc', str(vnc), '-name', name,
+                     '-snapshot',
+                    ]
+        self._cmd.extend(args)
+        self._configure_logging()
+
+    def _configure_logging(self):
+        """
+        Configure logging.
+        """
+        self.GuestCommandFilter.prefix = self._log_prefix
+
+        logger = logging.getLogger()
+        cmd_logger = logging.FileHandler(
+            filename=_LOG_FILE_GUEST_CMDS + str(self._number))
+        cmd_logger.setLevel(logging.DEBUG)
+        cmd_logger.addFilter(self.GuestCommandFilter())
+        logger.addHandler(cmd_logger)
+
+    # startup/Shutdown
+
+    def start(self):
+        """
+        Start QEMU instance, login and prepare for commands.
+        """
+        super(QemuDpdkVhostCuse, self).start()
+        self._affinitize()
+
+        if self._timeout:
+            self._login()
+            self._config_guest_loopback()
+
+    def stop(self):
+        """
+        Kill QEMU instance if it is alive.
+        """
+        self._logger.info('Killing QEMU...')
+
+        super(QemuDpdkVhostCuse, self).kill()
+
+    # helper functions
+
+    def _login(self, timeout=120):
+        """
+        Login to QEMU instance.
+
+        This can be used immediately after booting the machine, provided a
+        sufficiently long ``timeout`` is given.
+
+        :param timeout: Timeout to wait for login to complete.
+
+        :returns: None
+        """
+        # if no timeout was set, we likely started QEMU without waiting for it
+        # to boot. This being the case, we best check that it has finished
+        # first.
+        if not self._timeout:
+            self._expect_process(timeout=timeout)
+
+        self._child.sendline(_GUEST_USERNAME)
+        self._child.expect(_GUEST_PROMPT_PASSWORD, timeout=5)
+        self._child.sendline(_GUEST_PASSWORD)
+
+        self._expect_process(_GUEST_PROMPT, timeout=5)
+
+    def execute(self, cmd, delay=0):
+        """
+        Send ``cmd`` with no wait.
+
+        Useful for asynchronous commands.
+
+        :param cmd: Command to send to guest.
+        :param timeout: Delay to wait after sending command before returning.
+
+        :returns: None
+        """
+        self._logger.debug('%s%s', self._log_prefix, cmd)
+        self._child.sendline(cmd)
+        time.sleep(delay)
+
+    def wait(self, msg=_GUEST_PROMPT, timeout=30):
+        """
+        Wait for ``msg``.
+
+        :param msg: Message to wait for from guest.
+        :param timeout: Time to wait for message.
+
+        :returns: None
+        """
+        self._child.expect(msg, timeout=timeout)
+
+    def execute_and_wait(self, cmd, timeout=30, prompt=_GUEST_PROMPT):
+        """
+        Send ``cmd`` and wait ``timeout`` seconds for prompt.
+
+        :param cmd: Command to send to guest.
+        :param timeout: Time to wait for prompt.
+
+        :returns: None
+        """
+        self.execute(cmd)
+        self.wait(prompt, timeout=timeout)
+
+    def send_and_pass(self, cmd, timeout=30):
+        """
+        Send ``cmd`` and wait ``timeout`` seconds for it to pass.
+
+        :param cmd: Command to send to guest.
+        :param timeout: Time to wait for prompt before checking return code.
+
+        :returns: None
+        """
+        self.execute(cmd)
+        self.wait(_GUEST_PROMPT, timeout=timeout)
+        self.execute('echo $?')
+        self._child.expect('^0$', timeout=1)  # expect a 0
+        self.wait(_GUEST_PROMPT, timeout=timeout)
+
+    def _affinitize(self):
+        """
+        Affinitize the SMP cores of a QEMU instance.
+
+        This is a bit of a hack. The 'socat' utility is used to
+        interact with the QEMU HMP. This is necessary due to the lack
+        of QMP in older versions of QEMU, like v1.6.2. In future
+        releases, this should be replaced with calls to libvirt or
+        another Python-QEMU wrapper library.
+
+        :returns: None
+        """
+        thread_id = (r'.* CPU #%d: .* thread_id=(\d+)')
+
+        self._logger.info('Affinitizing guest...')
+
+        cur_locale = locale.getlocale()[1]
+        proc = subprocess.Popen(
+            ('echo', 'info cpus'), stdout=subprocess.PIPE)
+        output = subprocess.check_output(
+            ('sudo', 'socat', '-', 'UNIX-CONNECT:%s' % self._monitor),
+            stdin=proc.stdout)
+        proc.wait()
+
+        for cpu in range(0, int(_GUEST_SMP)):
+            match = None
+            for line in output.decode(cur_locale).split('\n'):
+                match = re.search(thread_id % cpu, line)
+                if match:
+                    self._affinitize_pid(
+                        _GUEST_CORE_BINDING[self._number - 1][cpu],
+                        match.group(1))
+                    break
+
+            if not match:
+                self._logger.error('Failed to affinitize guest core #%d. Could'
+                                   ' not parse tid.', cpu)
+
+    def _config_guest_loopback(self):
+        """
+        Configure VM to run testpmd
+
+        Configure performs the following:
+        * Mount hugepages
+        * mount shared directory for copying DPDK
+        * Disable firewall
+        * Compile DPDK
+        * DPDK NIC bind
+        * Run testpmd
+        """
+
+        # Guest images _should_ have 1024 hugepages by default,
+        # but just in case:'''
+        self.execute_and_wait('sysctl vm.nr_hugepages=1024')
+
+        # Mount hugepages
+        self.execute_and_wait('mkdir -p /dev/hugepages')
+        self.execute_and_wait(
+            'mount -t hugetlbfs hugetlbfs /dev/hugepages')
+
+        # mount shared directory
+        self.execute_and_wait('umount ' + _OVS_DPDK_SHARE)
+        self.execute_and_wait('rm -rf ' + _GUEST_OVS_DPDK_DIR)
+        self.execute_and_wait('mkdir -p ' + _OVS_DPDK_SHARE)
+        self.execute_and_wait('mount -o iocharset=utf8 /dev/sdb1 ' +
+                              _OVS_DPDK_SHARE)
+        self.execute_and_wait('mkdir -p ' + _GUEST_OVS_DPDK_DIR)
+        self.execute_and_wait('cp -a ' + _OVS_DPDK_SHARE + '/* ' + _GUEST_OVS_DPDK_DIR)
+
+        # Disable services (F16)
+        self.execute_and_wait('systemctl status iptables.service')
+        self.execute_and_wait('systemctl stop iptables.service')
+
+        # build and configure system for dpdk
+        self.execute_and_wait('cd ' + _GUEST_OVS_DPDK_DIR + '/DPDK',
+                              prompt=_QEMU_GUEST_DPDK_PROMPT)
+
+        self.execute_and_wait('export CC=gcc', prompt=_QEMU_GUEST_DPDK_PROMPT)
+        self.execute_and_wait('export RTE_SDK=' + _GUEST_OVS_DPDK_DIR + '/DPDK',
+                              prompt=_QEMU_GUEST_DPDK_PROMPT)
+        self.execute_and_wait('export RTE_TARGET=%s' % _RTE_TARGET,
+                              prompt=_QEMU_GUEST_DPDK_PROMPT)
+
+        self.execute_and_wait("sed -i -e 's/CONFIG_RTE_LIBRTE_VHOST_USER=n/" +
+                              "CONFIG_RTE_LIBRTE_VHOST_USER=y/g' config/common_linuxapp",
+                              prompt=_QEMU_GUEST_DPDK_PROMPT)
+
+        self.execute_and_wait('make uninstall', prompt=_QEMU_GUEST_DPDK_PROMPT)
+        self.execute_and_wait('make install T=%s -j 2' % _RTE_TARGET,
+                              timeout=300, prompt=_QEMU_GUEST_DPDK_PROMPT)
+
+        self.execute_and_wait('modprobe uio', prompt=_QEMU_GUEST_DPDK_PROMPT)
+        self.execute_and_wait('insmod %s/kmod/igb_uio.ko' % _RTE_TARGET,
+                              prompt=_QEMU_GUEST_DPDK_PROMPT)
+        self.execute_and_wait('./tools/dpdk_nic_bind.py --status',
+                              prompt=_QEMU_GUEST_DPDK_PROMPT)
+        self.execute_and_wait('./tools/dpdk_nic_bind.py -b igb_uio'
+                              ' ' + _GUEST_NET1_PCI_ADDRESS + ' '
+                              + _GUEST_NET2_PCI_ADDRESS,
+                              prompt=_QEMU_GUEST_DPDK_PROMPT)
+
+        self.execute_and_wait('./tools/dpdk_nic_bind.py --status',
+                              prompt=_QEMU_GUEST_DPDK_PROMPT)
+
+        self.execute_and_wait('cd ' +  _RTE_TARGET + '/build/app/test-pmd',
+                              prompt=_QEMU_GUEST_TEST_PMD_PROMPT)
+
+        self.execute_and_wait('./testpmd -c 0x3 -n 4 --socket-mem 512 --'
+                              ' --burst=64 -i --txqflags=0xf00 ' +
+                              '--disable-hw-vlan', 20, "Done")
+        self.execute('set fwd mac_retry', 1)
+        self.execute_and_wait('start', 20,
+                              'TX RS bit threshold=0 - TXQ flags=0xf00')
+
+
+if __name__ == '__main__':
+    import sys
+
+    with QemuDpdkVhostCuse() as vm1:
+        print(
+            '\n\n************************\n'
+            'Basic command line suitable for ls, cd, grep and cat.\n If you'
+            ' try to run Vim from here you\'re going to have a bad time.\n'
+            'For more complex tasks please use \'vncviewer :1\' to connect to'
+            ' this VM\nUsername: %s Password: %s\nPress ctrl-C to quit\n'
+            '************************\n' % (_GUEST_USERNAME, _GUEST_PASSWORD))
+
+        if sys.argv[1]:
+            with open(sys.argv[1], 'r') as file_:
+                for logline in file_:
+                    # lines are of format:
+                    #   guest_N_cmd : <command>
+                    # and we only want the <command> piece
+                    cmdline = logline.split(':')[1].strip()
+
+                    # use a no timeout since we don't know how long we
+                    # should wait
+                    vm1.send_and_wait(cmdline, timeout=-1)
+
+        while True:
+            USER_INPUT = input()
+            vm1.send_and_wait(USER_INPUT, timeout=5)
diff --git a/vswitches/ovs_dpdk_vhost.py b/vswitches/ovs_dpdk_vhost.py
index 3ff41260..ee0939a4 100644
--- a/vswitches/ovs_dpdk_vhost.py
+++ b/vswitches/ovs_dpdk_vhost.py
@@ -15,6 +15,7 @@
 """VSPERF VSwitch implementation using DPDK and vhost ports
 """
 
+import logging
 from conf import settings
 from vswitches.vswitch import IVSwitch
 from src.ovs import VSwitchd, OFBridge
@@ -33,11 +34,17 @@ class OvsDpdkVhost(IVSwitch):
     implementation. For generic information of the nature of the methods,
     see the interface.
     """
+
+    _logger = logging.getLogger()
+
     def __init__(self):
         vswitchd_args = ['--dpdk']
         vswitchd_args += settings.getValue('VSWITCHD_DPDK_ARGS')
         vswitchd_args += _VSWITCHD_CONST_ARGS
 
+        self._logger.info("Inserting VHOST modules into kernel...")
+        dpdk.insert_vhost_modules()
+
         self._vswitchd = VSwitchd(vswitchd_args=vswitchd_args,
                                   expected_cmd=
                                   r'EAL: Master l*core \d+ is ready')
@@ -58,6 +65,7 @@ class OvsDpdkVhost(IVSwitch):
         """
         self._vswitchd.kill()
         dpdk.cleanup()
+        dpdk.remove_vhost_modules()
 
     def add_switch(self, switch_name):
         """See IVswitch for general description
@@ -100,9 +108,16 @@ class OvsDpdkVhost(IVSwitch):
         """
         bridge = self._bridges[switch_name]
         # Changed dpdkvhost to dpdkvhostuser to be able to run in Qemu 2.2
-        vhost_count = self._get_port_count(bridge, 'type=dpdkvhostuser')
-        port_name = 'dpdkvhostuser' + str(vhost_count)
-        params = ['--', 'set', 'Interface', port_name, 'type=dpdkvhostuser']
+        vhost_method = settings.getValue('VHOST_METHOD')
+        if vhost_method == "cuse":
+            vhost_count = self._get_port_count(bridge, 'type=dpdkvhostcuse')
+            port_name = 'dpdkvhostcuse' + str(vhost_count)
+            params = ['--', 'set', 'Interface', port_name, 'type=dpdkvhostcuse']
+        else:
+            vhost_count = self._get_port_count(bridge, 'type=dpdkvhostuser')
+            port_name = 'dpdkvhostuser' + str(vhost_count)
+            params = ['--', 'set', 'Interface', port_name, 'type=dpdkvhostuser']
+
         of_port = bridge.add_port(port_name, params)
 
         return (port_name, of_port)