Merge "paths: Support binary packages"

5 files changed, 227 insertions, 58 deletions

diff --git a/tools/functions.py b/tools/functions.py
index 60ed0802..3bd8cc4d 100644
--- a/tools/functions.py
+++ b/tools/functions.py
@@ -15,20 +15,126 @@
 """Various helper functions
 """
 
-from conf import settings
+import os
+import logging
+import glob
+import shutil
+from conf import settings as S
 
 #
 # Support functions
 #
 
 def settings_update_paths():
-    """ Configure paths to OVS and DPDK based on VSWITCH and VNF values
+    """ Configure paths to OVS, DPDK and QEMU sources and binaries based on
+        selected vswitch type and src/binary switch. Data are taken from
+        PATHS dictionary and after their processing they are stored inside TOOLS.
+        PATHS dictionary has specific section for 'vswitch', 'qemu' and 'dpdk'
+        Following processing is done for every item:
+            item 'type' - string, which defines the type of paths ('src' or 'bin') to be selected
+                  for a given section:
+                      'src' means, that VSPERF will use OVS, DPDK or QEMU built from sources
+                          e.g. by execution of systems/build_base_machine.sh script during VSPERF
+                          installation
+                      'bin' means, that VSPERF will use OVS, DPDK or QEMU binaries installed
+                          in the OS, e.g. via OS specific packaging system
+            item 'path' - string with valid path; Its content is checked for existence, prefixed
+                  with section name and stored into TOOLS for later use
+                  e.g. TOOLS['dpdk_src'] or TOOLS['vswitch_src']
+            item 'modules' - list of strings; Every value from given list is checked for '.ko'
+                  suffix. In case it matches and it is not an absolute path to the module, then
+                  module name is prefixed with 'path' defined for the same section
+                  e.g. TOOLS['vswitch_modules'] = [
+                      '/tmp/vsperf/src_vanilla/ovs/ovs/datapath/linux/openvswitch.ko']
+            all other items - string - if given string is a relative path and item 'path'
+                  is defined for a given section, then item content will be prefixed with
+                  content of the 'path'. Otherwise tool name will be searched within
+                  standard system directories. Also any OS filename wildcards will be
+                  expanded to the real path. At the end of processing, every absolute
+                  path is checked for its existence. In case that temporary path (i.e. path
+                  with '_tmp' suffix) doesn't exist, then log will be written and vsperf will
+                  continue. If any other path will not exist, then vsperf execution will
+                  be terminated with runtime error.
+
+        Note: In case that 'bin' type is set for DPDK, then TOOLS['dpdk_src'] will be set to
+        the value of PATHS['dpdk']['src']['path']. The reason is, that VSPERF uses downloaded
+        DPDK sources to copy DPDK and testpmd into the GUEST, where testpmd is built. In case,
+        that DPDK sources are not available, then vsperf will continue with test execution,
+        but testpmd can't be used as a guest loopback. This is useful in case, that other guest
+        loopback applications (e.g. buildin) are used by CI jobs, etc.
     """
     # set dpdk and ovs paths accorfing to VNF and VSWITCH
-    if settings.getValue('VSWITCH').endswith('Vanilla'):
-        # settings paths for Vanilla
-        settings.setValue('OVS_DIR', (settings.getValue('OVS_DIR_VANILLA')))
-    else:
-        # default - set to VHOST USER but can be changed during enhancement
-        settings.setValue('RTE_SDK', (settings.getValue('RTE_SDK_USER')))
-        settings.setValue('OVS_DIR', (settings.getValue('OVS_DIR_USER')))
+    paths = {}
+    vswitch_type = S.getValue('PATHS')['vswitch'][S.getValue('VSWITCH')]['type']
+    paths['vswitch'] = S.getValue('PATHS')['vswitch'][S.getValue('VSWITCH')][vswitch_type]
+    paths['dpdk'] = S.getValue('PATHS')['dpdk'][S.getValue('PATHS')['dpdk']['type']]
+    paths['qemu'] = S.getValue('PATHS')['qemu'][S.getValue('PATHS')['qemu']['type']]
+    paths['paths'] = {}
+    paths['paths']['ovs_var_tmp'] = S.getValue('PATHS')['vswitch']['ovs_var_tmp']
+    paths['paths']['ovs_etc_tmp'] = S.getValue('PATHS')['vswitch']['ovs_etc_tmp']
+
+    tools = {}
+    for path_class in paths:
+        for tool in paths[path_class]:
+            tmp_tool = paths[path_class][tool]
+
+            # store valid path of given class into tools dict
+            if tool == 'path':
+                if os.path.isdir(tmp_tool):
+                    tools['{}_src'.format(path_class)] = tmp_tool
+                    continue
+                else:
+                    raise RuntimeError('Path {} does not exist.'.format(tmp_tool))
+
+            # store list of modules of given class into tools dict
+            if tool == 'modules':
+                tmp_modules = []
+                for module in tmp_tool:
+                    # add path to the .ko modules and check it for existence
+                    if module.endswith('.ko') and not os.path.isabs(module):
+                        module = os.path.join(paths[path_class]['path'], module)
+                        if not os.path.exists(module):
+                            raise RuntimeError('Cannot locate modlue {}'.format(module))
+
+                    tmp_modules.append(module)
+
+                tools['{}_modules'.format(path_class)] = tmp_modules
+                continue
+
+            # if path to the tool is relative, then 'path' will be prefixed
+            # in case that 'path' is not defined, then tool will be searched
+            # within standard system paths
+            if not os.path.isabs(tmp_tool):
+                if 'path' in paths[path_class]:
+                    tmp_tool = os.path.join(paths[path_class]['path'], tmp_tool)
+                elif shutil.which(tmp_tool):
+                    tmp_tool = shutil.which(tmp_tool)
+                else:
+                    raise RuntimeError('Cannot locate tool {}'.format(tmp_tool))
+
+            # expand OS wildcards in paths if needed
+            if glob.has_magic(tmp_tool):
+                tmp_glob = glob.glob(tmp_tool)
+                if len(tmp_glob) == 0:
+                    raise RuntimeError('Path to the {} is not valid: {}.'.format(tool, tmp_tool))
+                elif len(tmp_glob) > 1:
+                    raise RuntimeError('Path to the {} is ambiguous {}'.format(tool, tmp_glob))
+                elif len(tmp_glob) == 1:
+                    tmp_tool = tmp_glob[0]
+            elif not os.path.exists(tmp_tool):
+                if tool.endswith('_tmp'):
+                    logging.getLogger().debug('Temporary path to the {} does not '
+                                              'exist: {}.'.format(tool, tmp_tool))
+                else:
+                    raise RuntimeError('Path to the {} is not valid: {}'.format(tool, tmp_tool))
+
+            tools[tool] = tmp_tool
+
+    # ensure, that dpkg_src for bin will be set to downloaded DPDK sources, so it can
+    # be copied to the guest share dir and used by GUEST to build and run testpmd
+    # Validity of the path is not checked by purpose, so user can use VSPERF without
+    # downloading DPDK sources. In that case guest loopback can't be set to 'testpmd'
+    if S.getValue('PATHS')['dpdk']['type'] == 'bin':
+        tools['dpdk_src'] = S.getValue('PATHS')['dpdk']['src']['path']
+
+    S.setValue('TOOLS', tools)
diff --git a/tools/module_manager.py b/tools/module_manager.py
index 2eb4c63d..911f7252 100644
--- a/tools/module_manager.py
+++ b/tools/module_manager.py
@@ -31,30 +31,40 @@ class ModuleManager(object):
         """
         self._modules = []
 
-    def insert_module(self, module):
+    def insert_module(self, module, auto_remove=True):
         """Method inserts given module.
 
         In case that module name ends with .ko suffix then insmod will
         be used for its insertion. Otherwise modprobe will be called.
 
         :param module: a name of kernel module
+        :param auto_remove: if True (by default), then module will be
+            automatically removed by remove_modules() method
         """
         module_base_name = os.path.basename(os.path.splitext(module)[0])
 
         if self.is_module_inserted(module):
             self._logger.info('Module already loaded \'%s\'.', module_base_name)
             # add it to internal list, so we can try to remove it at the end
-            self._modules.append(module)
+            if auto_remove:
+                self._modules.append(module)
             return
 
         try:
             if module.endswith('.ko'):
+                # load module dependecies first, but suppress automatic
+                # module removal at the end; Just for case, that module
+                # depends on generic module
+                for depmod in self.get_module_dependecies(module):
+                    self.insert_module(depmod, auto_remove=False)
+
                 tasks.run_task(['sudo', 'insmod', module], self._logger,
                                'Insmod module \'%s\'...' % module_base_name, True)
             else:
                 tasks.run_task(['sudo', 'modprobe', module], self._logger,
                                'Modprobe module \'%s\'...' % module_base_name, True)
-            self._modules.append(module)
+            if auto_remove:
+                self._modules.append(module)
         except subprocess.CalledProcessError:
             # in case of error, show full module name
             self._logger.error('Unable to insert module \'%s\'.', module)
@@ -68,17 +78,6 @@ class ModuleManager(object):
         for module in modules:
             self.insert_module(module)
 
-    def insert_module_group(self, module_group, path_prefix):
-        """Ensure all modules in a group are inserted into the system.
-
-        :param module_group: A name of configuration item containing a list
-            of module names
-        :param path_prefix: A name of directory which contains given
-            group of modules
-        """
-        for (path_suffix, module) in module_group:
-            self.insert_module(os.path.join(path_prefix, path_suffix, '%s.ko' % module))
-
     def remove_module(self, module):
         """Removes a single module.
 
@@ -143,3 +142,26 @@ class ModuleManager(object):
                 return line
 
         return None
+
+    @staticmethod
+    def get_module_dependecies(module):
+        """Return list of modules, which must be loaded before module itself
+
+        :param module: a name of kernel module
+        :returns: In case that module has any dependencies, then list of module
+            names will be returned. Otherwise it returns empty list, i.e. [].
+        """
+        deps = ''
+        try:
+            # get list of module dependecies from kernel
+            deps = subprocess.check_output('modinfo -F depends {}'.format(module),
+                                           shell=True).decode().rstrip('\n')
+        except subprocess.CalledProcessError:
+            # in case of error, show full module name...
+            self._logger.info('Unable to get list of dependecies for module \'%s\'.', module)
+            # ...and try to continue, just for case that dependecies are already loaded
+
+        if len(deps):
+            return deps.split(',')
+        else:
+            return []
diff --git a/tools/networkcard.py b/tools/networkcard.py
index 8d704fd5..945534be 100644
--- a/tools/networkcard.py
+++ b/tools/networkcard.py
@@ -249,7 +249,7 @@ def reinit_vfs(pf_pci_handle):
 
     :param pf_pci_handle: PCI slot identifier of PF with domain part.
     """
-    rte_pci_tool = glob.glob(os.path.join(settings.getValue('RTE_SDK'), 'tools', 'dpdk*bind.py'))[0]
+    rte_pci_tool = settings.getValue('TOOLS')['bind-tool']
 
     for vf_nic in get_sriov_vfs_list(pf_pci_handle):
         nic_driver = get_driver(vf_nic)
diff --git a/tools/report/report.py b/tools/report/report.py
index 7d991011..1115f052 100644
--- a/tools/report/report.py
+++ b/tools/report/report.py
@@ -20,8 +20,8 @@ Generate reports in format defined by X.
 
 import sys
 import os
-import jinja2
 import logging
+import jinja2
 
 from core.results.results_constants import ResultsConstants
 from conf import settings as S
@@ -64,7 +64,8 @@ def _get_env(result):
     if result[ResultsConstants.DEPLOYMENT].count('v'):
         env.update({'vnf': systeminfo.get_version(S.getValue('VNF')),
                     'guest_image': S.getValue('GUEST_IMAGE'),
-                    'loopback_app': list(map(systeminfo.get_version, S.getValue('GUEST_LOOPBACK'))),
+                    'loopback_app': list(map(systeminfo.get_loopback_version,
+                                             S.getValue('GUEST_LOOPBACK'))),
                    })
 
     return env
diff --git a/tools/systeminfo.py b/tools/systeminfo.py
index 50dc17e0..fb1616d4 100644
--- a/tools/systeminfo.py
+++ b/tools/systeminfo.py
@@ -19,6 +19,7 @@ import os
 import platform
 import subprocess
 import locale
+import re
 
 from conf import settings as S
 
@@ -176,6 +177,40 @@ def pid_isalive(pid):
     """
     return os.path.isdir('/proc/' + str(pid))
 
+def get_bin_version(binary, regex):
+    """ get version of given binary selected by given regex
+
+    :returns: version string or None
+    """
+    try:
+        output = subprocess.check_output(binary, shell=True).decode().rstrip('\n')
+    except subprocess.CalledProcessError:
+        return None
+
+    versions = re.findall(regex, output)
+    if len(versions):
+        return versions[0]
+    else:
+        return None
+
+def get_git_tag(path):
+    """ get tag of recent commit from repository located at 'path'
+
+    :returns: git tag in form of string with commit hash or None if there
+        isn't any git repository at given path
+    """
+    try:
+        if os.path.isdir(path):
+            return subprocess.check_output('cd {}; git rev-parse HEAD'.format(path), shell=True,
+                                           stderr=subprocess.DEVNULL).decode().rstrip('\n')
+        elif os.path.isfile(path):
+            return subprocess.check_output('cd $(dirname {}); git log -1 --pretty="%H" {}'.format(path, path),
+                                           shell=True, stderr=subprocess.DEVNULL).decode().rstrip('\n')
+        else:
+            return None
+    except subprocess.CalledProcessError:
+        return None
+
 # This function uses long switch per purpose, so let us suppress pylint warning too-many-branches
 # pylint: disable=R0912
 def get_version(app_name):
@@ -186,45 +221,38 @@ def get_version(app_name):
 
     """
     app_version_file = {
-        'ovs' : os.path.join(S.getValue('OVS_DIR'), 'include/openvswitch/version.h'),
-        'dpdk' : os.path.join(S.getValue('RTE_SDK'), 'lib/librte_eal/common/include/rte_version.h'),
-        'qemu' : os.path.join(S.getValue('QEMU_DIR'), 'VERSION'),
-        'l2fwd' : os.path.join(S.getValue('ROOT_DIR'), 'src/l2fwd/l2fwd.c'),
+        'ovs' : r'Open vSwitch\) ([0-9.]+)',
+        'testpmd' : r'RTE Version: \'\S+ ([0-9.]+)',
+        'qemu' : r'QEMU emulator version ([0-9.]+)',
+        'loopback_l2fwd' : os.path.join(S.getValue('ROOT_DIR'), 'src/l2fwd/l2fwd.c'),
+        'loopback_testpmd' : os.path.join(S.getValue('TOOLS')['dpdk_src'],
+                                          'lib/librte_eal/common/include/rte_version.h'),
         'ixnet' : os.path.join(S.getValue('TRAFFICGEN_IXNET_LIB_PATH'), 'pkgIndex.tcl'),
     }
 
 
-    def get_git_tag(path):
-        """ get tag of recent commit from repository located at 'path'
-
-        :returns: git tag in form of string with commit hash or None if there
-            isn't any git repository at given path
-        """
-        try:
-            if os.path.isdir(path):
-                return subprocess.check_output('cd {}; git rev-parse HEAD'.format(path), shell=True,
-                                               stderr=subprocess.DEVNULL).decode().rstrip('\n')
-            elif os.path.isfile(path):
-                return subprocess.check_output('cd $(dirname {}); git log -1 --pretty="%H" {}'.format(path, path),
-                                               shell=True, stderr=subprocess.DEVNULL).decode().rstrip('\n')
-            else:
-                return None
-        except subprocess.CalledProcessError:
-            return None
-
 
     app_version = None
     app_git_tag = None
 
     if app_name.lower().startswith('ovs'):
-        app_version = match_line(app_version_file['ovs'], '#define OVS_PACKAGE_VERSION')
-        if app_version:
-            app_version = app_version.split('"')[1]
-        app_git_tag = get_git_tag(S.getValue('OVS_DIR'))
+        app_version = get_bin_version('{} --version'.format(S.getValue('TOOLS')['ovs-vswitchd']),
+                                      app_version_file['ovs'])
+        if 'vswitch_src' in S.getValue('TOOLS'):
+            app_git_tag = get_git_tag(S.getValue('TOOLS')['vswitch_src'])
     elif app_name.lower() in ['dpdk', 'testpmd']:
+        app_version = get_bin_version('{} -v -h'.format(S.getValue('TOOLS')['testpmd']),
+                                      app_version_file['testpmd'])
+        # we have to consult PATHS settings to be sure, that dpdk/testpmd
+        # were build from the sources
+        if S.getValue('PATHS')[app_name.lower()]['type'] == 'src':
+            app_git_tag = get_git_tag(S.getValue('TOOLS')['dpdk_src'])
+    elif app_name.lower() == 'loopback_testpmd':
+        # testpmd inside the guest is compiled from downloaded sources
+        # stored at TOOS['dpdk_src'] directory
         tmp_ver = ['', '', '']
         dpdk_16 = False
-        with open(app_version_file['dpdk']) as file_:
+        with open(app_version_file['loopback_testpmd']) as file_:
             for line in file_:
                 if not line.strip():
                     continue
@@ -263,10 +291,12 @@ def get_version(app_name):
 
         if len(tmp_ver[0]):
             app_version = '.'.join(tmp_ver)
-        app_git_tag = get_git_tag(S.getValue('RTE_SDK'))
+        app_git_tag = get_git_tag(S.getValue('TOOLS')['dpdk_src'])
     elif app_name.lower().startswith('qemu'):
-        app_version = match_line(app_version_file['qemu'], '')
-        app_git_tag = get_git_tag(S.getValue('QEMU_DIR'))
+        app_version = get_bin_version('{} --version'.format(S.getValue('TOOLS')['qemu-system']),
+                                      app_version_file['qemu'])
+        if 'qemu_src' in S.getValue('TOOLS'):
+            app_git_tag = get_git_tag(S.getValue('TOOLS')['qemu_src'])
     elif app_name.lower() == 'ixnet':
         app_version = match_line(app_version_file['ixnet'], 'package provide IxTclNetwork')
         if app_version:
@@ -283,13 +313,23 @@ def get_version(app_name):
     elif app_name.lower() == 'vswitchperf':
         app_git_tag = get_git_tag(S.getValue('ROOT_DIR'))
     elif app_name.lower() == 'l2fwd':
-        app_version = match_line(app_version_file[app_name], 'MODULE_VERSION')
+        app_version = match_line(app_version_file['loopback_l2fwd'], 'MODULE_VERSION')
         if app_version:
             app_version = app_version.split('"')[1]
-        app_git_tag = get_git_tag(app_version_file[app_name])
+        app_git_tag = get_git_tag(app_version_file['loopback_l2fwd'])
     elif app_name.lower() in ['linux_bridge', 'buildin']:
         # without login into running VM, it is not possible to check bridge_utils version
         app_version = 'NA'
         app_git_tag = 'NA'
 
     return {'name' : app_name, 'version' : app_version, 'git_tag' : app_git_tag}
+
+def get_loopback_version(loopback_app_name):
+    """ Get version of given guest loopback application and its git tag
+
+    :returns: dictionary {'name' : app_name, 'version' : app_version, 'git_tag' : app_git_tag) in case that
+        version or git tag are not known or not applicaple, than None is returned for any unknown value
+    """
+    version = get_version("loopback_{}".format(loopback_app_name))
+    version['name'] = loopback_app_name
+    return version