dpdk: Support new way of DPDK configuration in ovs-vswitchd

Configuration of DPDK options in ovs-vswitchd has changed.
Previously used option --dpdk was deprecated and all DPDK
related options have to be configured through ovsdb via ovs-vsctl
calls. VSPERF was modified to detect and use correct version
of DPDK configuration. New configuration options should
be put into VSWITCHD_DPDK_CONFIG dictionary.
VSPERF classes specific to OVS were refactored.

Change-Id: Ia3fad5906221439f477638f1f9734289dbf737bb
JIRA: VSPERF-291
Signed-off-by: Martin Klozik <martinx.klozik@intel.com>
Reviewed-by: Maryam Tahhan <maryam.tahhan@intel.com>
Reviewed-by: Al Morton <acmorton@att.com>
Reviewed-by: Christian Trautman <ctrautma@redhat.com>
Reviewed-by: Brian Castelli <brian.castelli@spirent.com>

9 files changed, 214 insertions, 184 deletions

diff --git a/conf/02_vswitch.conf b/conf/02_vswitch.conf
index 67f96991..7f9daf1c 100644
--- a/conf/02_vswitch.conf
+++ b/conf/02_vswitch.conf
@@ -69,8 +69,22 @@ VHOST_USER_SOCKS = ['/tmp/dpdkvhostuser0', '/tmp/dpdkvhostuser1',
 # ############################
 # These are DPDK EAL parameters and they may need to be changed depending on
 # hardware configuration, like cpu numbering and NUMA.
+#
+# parameters used for legacy DPDK configuration through '--dpdk' option of ovs-vswitchd
+# e.g. ovs-vswitchd --dpdk --socket-mem 1024,0
 VSWITCHD_DPDK_ARGS = ['-c', '0x4', '-n', '4', '--socket-mem 1024,0']
 
+# options used for new type of OVS configuration via calls to ovs-vsctl
+# e.g. ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-socket-mem="1024,0"
+VSWITCHD_DPDK_CONFIG = {
+    'dpdk-init' : 'true',
+    'dpdk-lcore-mask' : '0x4',
+    'dpdk-socket-mem' : '1024,0',
+}
+# Note: VSPERF will automatically detect, which type of DPDK configuration should
+# be used.
+
+# parameters passed to ovs-vswitchd in case that OvsVanilla is selected
 VSWITCHD_VANILLA_ARGS = []
 
 # use full module path to load module matching OVS version built from the source
diff --git a/docs/configguide/installation.rst b/docs/configguide/installation.rst
index d6161b9a..4ec14197 100755
--- a/docs/configguide/installation.rst
+++ b/docs/configguide/installation.rst
@@ -141,6 +141,15 @@ your configuration in the ``02_vswitch.conf`` file.
 .. code:: bash
 
     VSWITCHD_DPDK_ARGS = ['-c', '0x4', '-n', '4', '--socket-mem 1024,1024']
+    VSWITCHD_DPDK_CONFIG = {
+        'dpdk-init' : 'true',
+        'dpdk-lcore-mask' : '0x4',
+        'dpdk-socket-mem' : '1024,1024',
+    }
+
+Note: Option VSWITCHD_DPDK_ARGS is used for vswitchd, which supports --dpdk
+parameter. In recent vswitchd versions, option VSWITCHD_DPDK_CONFIG will be
+used to configure vswitchd via ovs-vsctl calls.
 
 With the --socket-mem argument set to use 1 hugepage on the specified sockets as
 seen above, the configuration will need 9 hugepages total to run all tests
diff --git a/docs/userguide/testusage.rst b/docs/userguide/testusage.rst
index 233f7015..104723e3 100755
--- a/docs/userguide/testusage.rst
+++ b/docs/userguide/testusage.rst
@@ -569,6 +569,16 @@ an appropriate amount of memory:
 .. code-block:: console
 
     VSWITCHD_DPDK_ARGS = ['-c', '0x4', '-n', '4', '--socket-mem 1024,0']
+    VSWITCHD_DPDK_CONFIG = {
+        'dpdk-init' : 'true',
+        'dpdk-lcore-mask' : '0x4',
+        'dpdk-socket-mem' : '1024,0',
+    }
+
+Note: Option VSWITCHD_DPDK_ARGS is used for vswitchd, which supports --dpdk
+parameter. In recent vswitchd versions, option VSWITCHD_DPDK_CONFIG will be
+used to configure vswitchd via ovs-vsctl calls.
+
 
 More information
 ^^^^^^^^^^^^^^^^
diff --git a/src/ovs/__init__.py b/src/ovs/__init__.py
index 8c157006..77592ea3 100644
--- a/src/ovs/__init__.py
+++ b/src/ovs/__init__.py
@@ -21,6 +21,5 @@ and external setup of vswitchd-external process, kernel modules etc.
 
 """
 
-from src.ovs.daemon import *
 from src.ovs.ofctl import *
 from src.ovs.dpctl import *
diff --git a/src/ovs/daemon.py b/src/ovs/daemon.py
deleted file mode 100644
index f9b037b2..00000000
--- a/src/ovs/daemon.py
+++ /dev/null
@@ -1,163 +0,0 @@
-# Copyright 2015 Intel Corporation.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Class wrapper for controlling an OVS instance.
-
-Wraps a pair of ``ovs-vswitchd`` and ``ovsdb-server`` processes.
-"""
-
-import os
-import logging
-import pexpect
-
-from conf import settings
-from tools import tasks
-
-_OVS_VAR_DIR = settings.getValue('OVS_VAR_DIR')
-_OVS_ETC_DIR = settings.getValue('OVS_ETC_DIR')
-
-_LOG_FILE_VSWITCHD = os.path.join(
-    settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_VSWITCHD'))
-
-class VSwitchd(tasks.Process):
-    """Class wrapper for controlling an OVS instance.
-
-    Wraps a pair of ``ovs-vswitchd`` and ``ovsdb-server`` processes.
-    """
-    _ovsdb_pid = None
-    _logfile = _LOG_FILE_VSWITCHD
-    _ovsdb_pidfile_path = os.path.join(settings.getValue('LOG_DIR'), "ovsdb_pidfile.pid")
-    _vswitchd_pidfile_path = os.path.join(settings.getValue('LOG_DIR'), "vswitchd_pidfile.pid")
-    _proc_name = 'ovs-vswitchd'
-
-    def __init__(self, timeout=30, vswitchd_args=None, expected_cmd=None):
-        """Initialise the wrapper with a specific start timeout and extra
-        parameters.
-
-        :param timeout: Timeout to wait for application to start.
-        :param vswitchd_args: Command line parameters for vswitchd.
-
-        :returns: None
-        """
-        self._logger = logging.getLogger(__name__)
-        self._timeout = timeout
-        self._expect = expected_cmd
-        vswitchd_args = vswitchd_args or []
-        ovs_vswitchd_bin = os.path.join(
-            settings.getValue('OVS_DIR'), 'vswitchd', 'ovs-vswitchd')
-        sep = ['--'] if '--dpdk' in vswitchd_args else []
-        self._cmd = ['sudo', '-E', ovs_vswitchd_bin] + vswitchd_args + sep + \
-                    ['--pidfile=' + self._vswitchd_pidfile_path, '--overwrite-pidfile',
-                     '--log-file=' + self._logfile]
-
-    # startup/shutdown
-
-    def start(self):
-        """ Start ``ovsdb-server`` and ``ovs-vswitchd`` instance.
-
-        :returns: None
-        :raises: pexpect.EOF, pexpect.TIMEOUT
-        """
-
-        self._reset_ovsdb()
-        self._start_ovsdb()  # this has to be started first
-
-        try:
-            super(VSwitchd, self).start()
-            self.relinquish()
-        except (pexpect.EOF, pexpect.TIMEOUT) as exc:
-            logging.error("Exception during VSwitch start.")
-            self._kill_ovsdb()
-            raise exc
-
-    def kill(self, signal='-15', sleep=10):
-        """Kill ``ovs-vswitchd`` instance if it is alive.
-
-        :returns: None
-        """
-        self._logger.info('Killing ovs-vswitchd...')
-        with open(self._vswitchd_pidfile_path, "r") as pidfile:
-            vswitchd_pid = pidfile.read().strip()
-            tasks.terminate_task(vswitchd_pid, logger=self._logger)
-
-        self._kill_ovsdb()  # ovsdb must be killed after vswitchd
-
-        # just for case, that sudo envelope has not terminated
-        super(VSwitchd, self).kill(signal, sleep)
-
-    # helper functions
-
-    def _reset_ovsdb(self):
-        """Reset system for 'ovsdb'.
-
-        :returns: None
-        """
-        self._logger.info('Resetting system after last run...')
-
-        tasks.run_task(['sudo', 'rm', '-rf', _OVS_VAR_DIR], self._logger)
-        tasks.run_task(['sudo', 'mkdir', '-p', _OVS_VAR_DIR], self._logger)
-        tasks.run_task(['sudo', 'rm', '-rf', _OVS_ETC_DIR], self._logger)
-        tasks.run_task(['sudo', 'mkdir', '-p', _OVS_ETC_DIR], self._logger)
-
-        tasks.run_task(['sudo', 'rm', '-f',
-                        os.path.join(_OVS_ETC_DIR, 'conf.db')],
-                       self._logger)
-
-        self._logger.info('System reset after last run.')
-
-    def _start_ovsdb(self):
-        """Start ``ovsdb-server`` instance.
-
-        :returns: None
-        """
-        ovsdb_tool_bin = os.path.join(
-            settings.getValue('OVS_DIR'), 'ovsdb', 'ovsdb-tool')
-        tasks.run_task(['sudo', ovsdb_tool_bin, 'create',
-                        os.path.join(_OVS_ETC_DIR, 'conf.db'),
-                        os.path.join(settings.getValue('OVS_DIR'), 'vswitchd',
-                                     'vswitch.ovsschema')],
-                       self._logger,
-                       'Creating ovsdb configuration database...')
-
-        ovsdb_server_bin = os.path.join(
-            settings.getValue('OVS_DIR'), 'ovsdb', 'ovsdb-server')
-
-        tasks.run_background_task(
-            ['sudo', ovsdb_server_bin,
-             '--remote=punix:%s' % os.path.join(_OVS_VAR_DIR, 'db.sock'),
-             '--remote=db:Open_vSwitch,Open_vSwitch,manager_options',
-             '--pidfile=' + self._ovsdb_pidfile_path, '--overwrite-pidfile'],
-            self._logger,
-            'Starting ovsdb-server...')
-
-    def _kill_ovsdb(self):
-        """Kill ``ovsdb-server`` instance.
-
-        :returns: None
-        """
-        with open(self._ovsdb_pidfile_path, "r") as pidfile:
-            ovsdb_pid = pidfile.read().strip()
-
-        self._logger.info("Killing ovsdb with pid: " + ovsdb_pid)
-
-        if ovsdb_pid:
-            tasks.terminate_task(ovsdb_pid, logger=self._logger)
-
-    @staticmethod
-    def get_db_sock_path():
-        """Method returns location of db.sock file
-
-        :returns: path to db.sock file.
-        """
-        return os.path.join(_OVS_VAR_DIR, 'db.sock')
diff --git a/src/ovs/ofctl.py b/src/ovs/ofctl.py
index 93894889..1ee48133 100644
--- a/src/ovs/ofctl.py
+++ b/src/ovs/ofctl.py
@@ -57,12 +57,18 @@ class OFBase(object):
     def run_vsctl(self, args, check_error=False):
         """Run ``ovs-vsctl`` with supplied arguments.
 
+        In case that timeout is set to -1, then ovs-vsctl
+        will be called with --no-wait option.
+
         :param args: Arguments to pass to ``ovs-vsctl``
         :param check_error: Throw exception on error
 
         :return: None
         """
-        cmd = ['sudo', _OVS_VSCTL_BIN, '--timeout', str(self.timeout)] + args
+        if self.timeout == -1:
+            cmd = ['sudo', _OVS_VSCTL_BIN, '--no-wait'] + args
+        else:
+            cmd = ['sudo', _OVS_VSCTL_BIN, '--timeout', str(self.timeout)] + args
         return tasks.run_task(
             cmd, self.logger, 'Running ovs-vsctl...', check_error)
 
diff --git a/vswitches/ovs.py b/vswitches/ovs.py
index dd49a1fc..a8184320 100644
--- a/vswitches/ovs.py
+++ b/vswitches/ovs.py
@@ -17,40 +17,77 @@
 
 import logging
 import re
+import os
+import pexpect
 from conf import settings
 from vswitches.vswitch import IVSwitch
 from src.ovs import OFBridge, flow_key, flow_match
+from tools import tasks
 
-_VSWITCHD_CONST_ARGS = []
+_OVS_VAR_DIR = settings.getValue('OVS_VAR_DIR')
+_OVS_ETC_DIR = settings.getValue('OVS_ETC_DIR')
 
-class IVSwitchOvs(IVSwitch):
+class IVSwitchOvs(IVSwitch, tasks.Process):
     """Open vSwitch base class implementation
 
     The method docstrings document only considerations specific to this
     implementation. For generic information of the nature of the methods,
     see the interface.
     """
+    _logfile = os.path.join(settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_VSWITCHD'))
+    _ovsdb_pidfile_path = os.path.join(settings.getValue('LOG_DIR'), "ovsdb_pidfile.pid")
+    _vswitchd_pidfile_path = os.path.join(settings.getValue('LOG_DIR'), "vswitchd_pidfile.pid")
+    _proc_name = 'ovs-vswitchd'
 
     def __init__(self):
         """See IVswitch for general description
         """
-        self._vswitchd = None
         self._logger = logging.getLogger(__name__)
+        self._expect = None
+        self._timeout = 30
         self._bridges = {}
-        self._vswitchd_args = _VSWITCHD_CONST_ARGS
+        self._vswitchd_args = ['--pidfile=' + self._vswitchd_pidfile_path,
+                               '--overwrite-pidfile', '--log-file=' + self._logfile]
+        self._cmd = []
+        self._cmd_template = ['sudo', '-E', os.path.join(settings.getValue('OVS_DIR'),
+                                                         'vswitchd', 'ovs-vswitchd')]
 
     def start(self):
-        """See IVswitch for general description
+        """ Start ``ovsdb-server`` and ``ovs-vswitchd`` instance.
+
+        :raises: pexpect.EOF, pexpect.TIMEOUT
         """
         self._logger.info("Starting vswitchd...")
-        self._vswitchd.start()
+
+        self._cmd = self._cmd_template + self._vswitchd_args
+
+        # DB must be started before vswitchd
+        self._reset_ovsdb()
+        self._start_ovsdb()
+
+        # DB must be up before vswitchd config is altered
+        self.configure()
+
+        try:
+            tasks.Process.start(self)
+            self.relinquish()
+        except (pexpect.EOF, pexpect.TIMEOUT) as exc:
+            logging.error("Exception during VSwitch start.")
+            self._kill_ovsdb()
+            raise exc
+
         self._logger.info("Vswitchd...Started.")
 
+    def configure(self):
+        """ Configure vswitchd through ovsdb if needed
+        """
+        pass
+
     def stop(self):
         """See IVswitch for general description
         """
         self._logger.info("Terminating vswitchd...")
-        self._vswitchd.kill()
+        self.kill()
         self._logger.info("Vswitchd...Terminated.")
 
     def add_switch(self, switch_name, params=None):
@@ -153,6 +190,93 @@ class IVSwitchOvs(IVSwitch):
             cnt = 0
         return cnt
 
+    def kill(self, signal='-15', sleep=10):
+        """Kill ``ovs-vswitchd`` and ``ovs-ovsdb`` instances if they are alive.
+
+        :returns: None
+        """
+        if os.path.isfile(self._vswitchd_pidfile_path):
+            self._logger.info('Killing ovs-vswitchd...')
+            with open(self._vswitchd_pidfile_path, "r") as pidfile:
+                vswitchd_pid = pidfile.read().strip()
+                tasks.terminate_task(vswitchd_pid, logger=self._logger)
+
+        self._kill_ovsdb()  # ovsdb must be killed after vswitchd
+
+        # just for case, that sudo envelope has not been terminated yet
+        tasks.Process.kill(self, signal, sleep)
+
+    # helper functions
+
+    def _reset_ovsdb(self):
+        """Reset system for 'ovsdb'.
+
+        :returns: None
+        """
+        self._logger.info('Resetting system after last run...')
+
+        tasks.run_task(['sudo', 'rm', '-rf', _OVS_VAR_DIR], self._logger)
+        tasks.run_task(['sudo', 'mkdir', '-p', _OVS_VAR_DIR], self._logger)
+        tasks.run_task(['sudo', 'rm', '-rf', _OVS_ETC_DIR], self._logger)
+        tasks.run_task(['sudo', 'mkdir', '-p', _OVS_ETC_DIR], self._logger)
+
+        tasks.run_task(['sudo', 'rm', '-f',
+                        os.path.join(_OVS_ETC_DIR, 'conf.db')],
+                       self._logger)
+
+        self._logger.info('System reset after last run.')
+
+    def _start_ovsdb(self):
+        """Start ``ovsdb-server`` instance.
+
+        :returns: None
+        """
+        ovsdb_tool_bin = os.path.join(
+            settings.getValue('OVS_DIR'), 'ovsdb', 'ovsdb-tool')
+        tasks.run_task(['sudo', ovsdb_tool_bin, 'create',
+                        os.path.join(_OVS_ETC_DIR, 'conf.db'),
+                        os.path.join(settings.getValue('OVS_DIR'), 'vswitchd',
+                                     'vswitch.ovsschema')],
+                       self._logger,
+                       'Creating ovsdb configuration database...')
+
+        ovsdb_server_bin = os.path.join(
+            settings.getValue('OVS_DIR'), 'ovsdb', 'ovsdb-server')
+
+        tasks.run_background_task(
+            ['sudo', ovsdb_server_bin,
+             '--remote=punix:%s' % os.path.join(_OVS_VAR_DIR, 'db.sock'),
+             '--remote=db:Open_vSwitch,Open_vSwitch,manager_options',
+             '--pidfile=' + self._ovsdb_pidfile_path, '--overwrite-pidfile'],
+            self._logger,
+            'Starting ovsdb-server...')
+
+    def _kill_ovsdb(self):
+        """Kill ``ovsdb-server`` instance.
+
+        :returns: None
+        """
+        if os.path.isfile(self._ovsdb_pidfile_path):
+            with open(self._ovsdb_pidfile_path, "r") as pidfile:
+                ovsdb_pid = pidfile.read().strip()
+
+            self._logger.info("Killing ovsdb with pid: " + ovsdb_pid)
+
+            if ovsdb_pid:
+                tasks.terminate_task(ovsdb_pid, logger=self._logger)
+
+    @staticmethod
+    def get_db_sock_path():
+        """Method returns location of db.sock file
+
+        :returns: path to db.sock file.
+        """
+        return os.path.join(_OVS_VAR_DIR, 'db.sock')
+
+    #
+    # validate methods required for integration testcases
+    #
+
     def validate_add_switch(self, result, switch_name, params=None):
         """Validate - Create a new logical switch with no ports
         """
diff --git a/vswitches/ovs_dpdk_vhost.py b/vswitches/ovs_dpdk_vhost.py
index 9d29c9d1..82b952de 100644
--- a/vswitches/ovs_dpdk_vhost.py
+++ b/vswitches/ovs_dpdk_vhost.py
@@ -16,10 +16,13 @@
 """
 
 import logging
+import subprocess
+import os
+
+from src.ovs import OFBridge
+from src.dpdk import dpdk
 from conf import settings
 from vswitches.ovs import IVSwitchOvs
-from src.ovs import VSwitchd
-from src.dpdk import dpdk
 
 class OvsDpdkVhost(IVSwitchOvs):
     """ Open vSwitch with DPDK support
@@ -36,16 +39,32 @@ class OvsDpdkVhost(IVSwitchOvs):
     def __init__(self):
         super(OvsDpdkVhost, self).__init__()
         self._logger = logging.getLogger(__name__)
+        self._expect = r'EAL: Master l*core \d+ is ready'
+
+        vswitchd_args = []
+
+        # legacy DPDK configuration through --dpdk option of vswitchd
+        if self.old_dpdk_config():
+            vswitchd_args = ['--dpdk'] + settings.getValue('VSWITCHD_DPDK_ARGS')
+            if self._vswitchd_args:
+                self._vswitchd_args = vswitchd_args + ['--'] + self._vswitchd_args
+            else:
+                self._vswitchd_args = vswitchd_args
 
-        self._vswitchd_args = ['--dpdk']
-        self._vswitchd_args += settings.getValue('VSWITCHD_DPDK_ARGS')
         if settings.getValue('VNF').endswith('Cuse'):
             self._logger.info("Inserting VHOST Cuse modules into kernel...")
             dpdk.insert_vhost_modules()
 
-        self._vswitchd = VSwitchd(vswitchd_args=self._vswitchd_args,
-                                  expected_cmd=
-                                  r'EAL: Master l*core \d+ is ready')
+    def configure(self):
+        """ Configure vswitchd DPDK options through ovsdb if needed
+        """
+        dpdk_config = settings.getValue('VSWITCHD_DPDK_CONFIG')
+        if dpdk_config and not self.old_dpdk_config():
+            # enforce calls to ovs-vsctl with --no-wait
+            tmp_br = OFBridge(timeout=-1)
+            for option in dpdk_config:
+                tmp_br.set_db_attribute('Open_vSwitch', '.',
+                                        'other_config:' + option, dpdk_config[option])
 
     def start(self):
         """See IVswitch for general description
@@ -115,3 +134,17 @@ class OvsDpdkVhost(IVSwitchOvs):
         of_port = bridge.add_port(port_name, params)
 
         return (port_name, of_port)
+
+    @staticmethod
+    def old_dpdk_config():
+        """Checks if ovs-vswitchd uses legacy dpdk configuration via --dpdk option
+
+        :returns: True if legacy --dpdk option is supported, otherwise it returns False
+        """
+
+        ovs_vswitchd_bin = os.path.join(settings.getValue('OVS_DIR'), 'vswitchd', 'ovs-vswitchd')
+        try:
+            subprocess.check_output(ovs_vswitchd_bin + r' --help | grep "\-\-dpdk"', shell=True)
+            return True
+        except subprocess.CalledProcessError:
+            return False
diff --git a/vswitches/ovs_vanilla.py b/vswitches/ovs_vanilla.py
index f5fecc2a..a6b720ae 100644
--- a/vswitches/ovs_vanilla.py
+++ b/vswitches/ovs_vanilla.py
@@ -18,7 +18,7 @@
 import logging
 from conf import settings
 from vswitches.ovs import IVSwitchOvs
-from src.ovs import VSwitchd, DPCtl
+from src.ovs import DPCtl
 from tools.module_manager import ModuleManager
 from tools import tasks
 
@@ -39,11 +39,9 @@ class OvsVanilla(IVSwitchOvs):
         super(OvsVanilla, self).__init__()
         self._ports = list(nic['device'] for nic in settings.getValue('NICS'))
         self._logger = logging.getLogger(__name__)
-        self._vswitchd_args = ["unix:%s" % VSwitchd.get_db_sock_path()]
+        self._vswitchd_args += ["unix:%s" % self.get_db_sock_path()]
         self._vswitchd_args += settings.getValue('VSWITCHD_VANILLA_ARGS')
-        self._vswitchd = VSwitchd(vswitchd_args=self._vswitchd_args,
-                                  expected_cmd="db.sock: connected")
-        self._bridges = {}
+        self._expect = "db.sock: connected"
         self._module_manager = ModuleManager()
 
     def start(self):