integration: Test vHost User numa awareness

Open vSwitch with DPDK can optimize memory usage
in case of NUMA architecture to avoid unnecessary
memory access across NUMA slots. In a nutshell,
PMD threads serving virtual NICs are co-located at
the same NUMA slot as QEMU instance, which is using
these NICs.
This patch adds new (functional) integration testcase,
which verifies OVS vHost User numa awareness feature.
Step driven test objects were updated to allow a call
of OS utilies and evaluation of conditions. Also the
documentation was updated with the list of supported
test objects and their methods.

JIRA: VSPERF-377

Change-Id: I184e71e066d27b5b9fc9e6a9f7e240e2d1b5a0fa
Signed-off-by: Martin Klozik <martinx.klozik@intel.com>
Reviewed-by: Maryam Tahhan <maryam.tahhan@intel.com>
Reviewed-by: Ciara Loftus <ciara.loftus@intel.com>
Reviewed-by: Al Morton <acmorton@att.com>
Reviewed-by: Christian Trautman <ctrautma@redhat.com>
Reviewed-by: Bill Michalowski <bmichalo@redhat.com>
Reviewed-by: Antonio Fischetti <antonio.fischetti@intel.com>

7 files changed, 387 insertions, 14 deletions

diff --git a/conf/__init__.py b/conf/__init__.py
index 88e8cec6..4d6f57fe 100644
--- a/conf/__init__.py
+++ b/conf/__init__.py
@@ -170,6 +170,7 @@ class Settings(object):
                 for macro, args, param, _, step in re.findall(_PARSE_PATTERN, value):
                     multi = int(step) if len(step) and int(step) else 1
                     if macro == '#EVAL':
+                        # pylint: disable=eval-used
                         tmp_result = str(eval(param))
                     elif macro == '#MAC':
                         mac_value = netaddr.EUI(param).value
@@ -204,6 +205,20 @@ class Settings(object):
         """
         return pprint.pformat(self.__dict__)
 
+    #
+    # validation methods used by step driven testcases
+    #
+    def validate_getValue(self, result, attr):
+        """Verifies, that correct value was returned
+        """
+        assert result == self.__dict__[attr]
+        return True
+
+    def validate_setValue(self, dummy_result, name, value):
+        """Verifies, that value was correctly set
+        """
+        assert value == self.__dict__[name]
+        return True
 
 settings = Settings()
 
diff --git a/conf/integration/01_testcases.conf b/conf/integration/01_testcases.conf
index a67702f8..084323b2 100644
--- a/conf/integration/01_testcases.conf
+++ b/conf/integration/01_testcases.conf
@@ -649,6 +649,79 @@ INTEGRATION_TESTS = [
         ] + STEP_VSWITCH_FLOWS_FINIT +
         STEP_VSWITCH_2PHY_6VM_FINIT
     },
+    {
+        # Testcase for verification of vHost User NUMA awareness feature
+        # introduced in DPDK v2.2. Test case will execute two VNFs, each
+        # pinned to different NUMA slot. After that it will verify that
+        # QEMU and PMD threads serving its interfaces are co-located
+        # at the same NUMA slot.
+        #
+        # Prerequisites:
+        #     * architecture with at least 2 NUMA slots
+        #     * OVS with DPDK support and DPDK v2.2 and newer
+        #     * OVS configuration utilizing both NUMA slots
+        #
+        #     Example of OVS configuration valid for DPDK v16.04 and cores
+        #     split between NUMA slots as follows:
+        #           node 0 cpus: 0 1 2 3 4 5 6 7 8 9
+        #           node 1 cpus: 10 11 12 13 14 15 16 17 18 19
+        #
+        #     VSWITCH_PMD_CPU_MASK = '1010'
+        #     VSWITCHD_DPDK_CONFIG = {
+        #         'dpdk-init' : 'true',
+        #         'dpdk-lcore-mask' : '0x4004',
+        #         'pmd-cpu-mask' : 'FF0FF',
+        #         'dpdk-socket-mem' : '1024,1024',
+        #     }
+        #
+        "Name": "vhost_numa_awareness",
+        "Deployment": "clean",
+        "Description": "vSwitch DPDK - verify that PMD threads are served "
+                       "by the same NUMA slot as QEMU instances",
+        "vSwitch" : "OvsDpdkVhost",
+        "TestSteps": STEP_VSWITCH_PVVP_INIT +                                       # STEP 0-6
+            [
+                # check that at least 2 numa slots are available
+                ['tools', 'exec', 'numactl -H', 'available: ([0-9]+)'],             # STEP 7
+                ['tools', 'assert', '#STEP[-1][0]>1'],                              # STEP 8
+                # store last 2 cores from numa slot 0
+                ['tools', 'exec', 'numactl -H', 'node 0 cpus:.*\s+(\\d+) (\\d+)$'], # STEP 9
+                # store last 2 cores from numa slot 1
+                ['tools', 'exec', 'numactl -H', 'node 1 cpus:.*\s+(\\d+) (\\d+)$'], # STEP 10
+                # pin VNF1 to 1st NUMA slot and VNF2 to 2nd NUMA slot
+                ['settings', 'setValue', 'GUEST_CORE_BINDING',                      # STEP 11
+                    [("#STEP[-2][0][0]", "#STEP[-2][0][1]"),
+                     ("#STEP[-1][0][0]", "#STEP[-1][0][1]")]
+                ],
+                # start 2 VNFs
+                ['vnf1', 'start'],                                                  # STEP 12
+                ['vnf2', 'start'],                                                  # STEP 13
+                # read paths to ovs utilities
+                ['settings', 'getValue', 'TOOLS'],                                  # STEP 14
+                # check that PMD thread serving VNF1 runs at NUMA slot 0
+                ## i.e. get numa slot ID serving dpdhvhostuser0...
+                ['tools', 'exec', "sudo #STEP[-1]['ovs-appctl'] "                   # STEP 15
+                    "dpif-netdev/pmd-rxq-show | "
+                    "sed -e '/dpdkvhostuser0/,$d' | tac",
+                    'pmd thread numa_id ([0-9])+'
+                ],
+                ## ...and check that it is NUMA slot 0
+                ['tools', 'assert', '#STEP[-1][0]==0'],                             # STEP 16
+                # check that PMD thread serving VNF2 runs at NUMA slot 1
+                ## i.e. get numa slot ID serving dpdhvhostuser2...
+                ['tools', 'exec', "sudo #STEP[-3]['ovs-appctl'] "                   # STEP 17
+                    "dpif-netdev/pmd-rxq-show | "
+                    "sed -e '/dpdkvhostuser2/,$d' | tac",
+                    'pmd thread numa_id ([0-9])+'
+                ],
+                ## ...and check that it is NUMA slot 1
+                ['tools', 'assert', '#STEP[-1][0]==1'],                             # STEP 18
+                # clean up
+                ['vnf2', 'stop'],                                                   # STEP 19
+                ['vnf1', 'stop'],                                                   # STEP 20
+            ] +
+            STEP_VSWITCH_PVVP_FINIT                                                 # STEP 21...
+    },
 ]
 
 # Example of TC definition with exact vSwitch, VNF and TRAFFICGEN values.
diff --git a/docs/userguide/integration.rst b/docs/userguide/integration.rst
index b0926d89..ceb11635 100755
--- a/docs/userguide/integration.rst
+++ b/docs/userguide/integration.rst
@@ -49,6 +49,173 @@ test will fail and terminate. The test will continue until a failure is detected
 or all steps pass. A csv report file is generated after a test completes with an
 OK or FAIL result.
 
+Test objects and their functions
+--------------------------------
+
+Every test step can call a function of one of the supported test objects. The list
+of supported objects and their most common functions follows:
+
+    * ``vswitch`` - provides functions for vSwitch configuration
+
+      List of supported functions:
+
+        * ``add_switch br_name`` - creates a new switch (bridge) with given ``br_name``
+        * ``del_switch br_name`` - deletes switch (bridge) with given ``br_name``
+        * ``add_phy_port br_name`` - adds a physical port into bridge specified by ``br_name``
+        * ``add_vport br_name`` - adds a virtual port into bridge specified by ``br_name``
+        * ``del_port br_name port_name`` - removes physical or virtual port specified by
+          ``port_name`` from bridge ``br_name``
+        * ``add_flow br_name flow`` - adds flow specified by ``flow`` dictionary into
+          the bridge ``br_name``; Content of flow dictionary will be passed to the vSwitch.
+          In case of Open vSwitch it will be passed to the ``ovs-ofctl add-flow`` command.
+          Please see Open vSwitch documentation for the list of supported flow parameters.
+        * ``del_flow br_name [flow]`` - deletes flow specified by ``flow`` dictionary from
+          bridge ``br_name``; In case that optional parameter ``flow`` is not specified
+          or set to an empty dictionary ``{}``, then all flows from bridge ``br_name``
+          will be deleted.
+        * ``dump_flows br_name`` - dumps all flows from bridge specified by ``br_name``
+        * ``enable_stp br_name`` - enables Spanning Tree Protocol for bridge ``br_name``
+        * ``disable_stp br_name`` - disables Spanning Tree Protocol for bridge ``br_name``
+        * ``enable_rstp br_name`` - enables Rapid Spanning Tree Protocol for bridge ``br_name``
+        * ``disable_rstp br_name`` - disables Rapid Spanning Tree Protocol for bridge ``br_name``
+
+        Examples:
+
+        .. code-block:: python
+
+            ['vswitch', 'add_switch', 'int_br0']
+
+            ['vswitch', 'del_switch', 'int_br0']
+
+            ['vswitch', 'add_phy_port', 'int_br0']
+
+            ['vswitch', 'del_port', 'int_br0', '#STEP[2][0]']
+
+            ['vswitch', 'add_flow', 'int_br0', {'in_port': '1', 'actions': ['output:2'],
+             'idle_timeout': '0'}],
+
+            ['vswitch', 'enable_rstp', 'int_br0']
+
+    * ``vnf[ID]`` - provides functions for deployment and termination of VNFs; Optional
+      alfanumerical ``ID`` is used for VNF identification in case that testcase
+      deploys multiple VNFs.
+
+      List of supported functions:
+
+        * ``start`` - starts a VNF based on VSPERF configuration
+        * ``stop`` - gracefully terminates given VNF
+
+        Examples:
+
+        .. code-block:: python
+
+            ['vnf1', 'start']
+            ['vnf2', 'start']
+            ['vnf2', 'stop']
+            ['vnf1', 'stop']
+
+    * ``trafficgen`` - triggers traffic generation
+
+      List of supported functions:
+
+        * ``send_traffic traffic`` - starts a traffic based on the vsperf configuration
+          and given ``traffic`` dictionary. More details about ``traffic`` dictionary
+          and its possible values are available at `Traffic Generator Integration Guide
+          <http://artifacts.opnfv.org/vswitchperf/docs/design/trafficgen_integration_guide.html#step-5-supported-traffic-types>`__
+
+        Examples:
+
+        .. code-block:: python
+
+            ['trafficgen', 'send_traffic', {'traffic_type' : 'throughput'}]
+
+            ['trafficgen', 'send_traffic', {'traffic_type' : 'back2back', 'bidir' : 'True'}]
+
+    * ``settings`` - reads or modifies VSPERF configuration
+
+      List of supported functions:
+
+        * ``getValue param`` - returns value of given ``param``
+        * ``setValue param value`` - sets value of ``param`` to given ``value``
+
+        Examples:
+
+        .. code-block:: python
+
+            ['settings', 'getValue', 'TOOLS']
+
+            ['settings', 'setValue', 'GUEST_USERNAME', ['root']]
+
+    * ``namespace`` - creates or modifies network namespaces
+
+      List of supported functions:
+
+        * ``create_namespace name`` - creates new namespace with given ``name``
+        * ``delete_namespace name`` - deletes namespace specified by its ``name``
+        * ``assign_port_to_namespace port name [port_up]`` - assigns NIC specified by ``port``
+          into given namespace ``name``; If optional parameter ``port_up`` is set to ``True``,
+          then port will be brought up.
+        * ``add_ip_to_namespace_eth port name addr cidr`` - assigns an IP address ``addr``/``cidr``
+          to the NIC specified by ``port`` within namespace ``name``
+        * ``reset_port_to_root port name`` - returns given ``port`` from namespace ``name`` back
+          to the root namespace
+
+        Examples:
+
+        .. code-block:: python
+
+            ['namespace', 'create_namespace', 'testns']
+
+            ['namespace', 'assign_port_to_namespace', 'eth0', 'testns']
+
+    * ``veth`` - manipulates with eth and veth devices
+
+      List of supported functions:
+
+        * ``add_veth_port port peer_port`` - adds a pair of veth ports named ``port`` and
+          ``peer_port``
+        * ``del_veth_port port peer_port`` - deletes a veth port pair specified by ``port``
+          and ``peer_port``
+        * ``bring_up_eth_port eth_port [namespace]`` - brings up ``eth_port`` in (optional)
+          ``namespace``
+
+        Examples:
+
+        .. code-block:: python
+
+            ['veth', 'add_veth_port', 'veth', 'veth1']
+
+            ['veth', 'bring_up_eth_port', 'eth1']
+
+    * ``tools`` - provides a set of helper functions
+
+      List of supported functions:
+
+        * ``Assert condition`` - evaluates given ``condition`` and raises ``AssertionError``
+          in case that condition is not ``True``
+        * ``Eval expression`` - evaluates given expression as a python code and returns
+          its result
+        * ``Exec command [regex]`` - executes a shell command and filters its output by
+          (optional) regular expression
+
+        Examples:
+
+        .. code-block:: python
+
+            ['tools', 'exec', 'numactl -H', 'available: ([0-9]+)']
+            ['tools', 'assert', '#STEP[-1][0]>1']
+
+    * ``wait`` - is used for test case interruption. This object doesn't have
+      any functions. Once reached, vsperf will pause test execution and waits
+      for press of ``Enter key``. It can be used during testcase design
+      for debugging purposes.
+
+      Examples:
+
+      .. code-block:: python
+
+        ['wait']
+
 Test Macros
 -----------
 
@@ -75,6 +242,16 @@ This test profile uses the vswitch add_vport method which returns a string
 value of the port added. This is later called by the del_port method using the
 name from step 1.
 
+It is also possible to use negative indexes in step macros. In that case
+``#STEP[-1]`` will refer to the result from previous step, ``#STEP[-2]``
+will refer to result of step called before previous step, etc. It means,
+that you could change ``STEP 2`` from previous example to achieve the same
+functionality:
+
+.. code-block:: python
+
+                ['vswitch', 'del_port', 'int_br0', '#STEP[-1][0]'],  # STEP 2
+
 Also commonly used steps can be created as a separate profile.
 
 .. code-block:: python
diff --git a/src/dpdk/Makefile b/src/dpdk/Makefile
index 1cd85e3e..29967029 100755
--- a/src/dpdk/Makefile
+++ b/src/dpdk/Makefile
@@ -72,6 +72,7 @@ ifdef CONFIG_FILE_BASE
 	$(AT)sed -i -e 's/CONFIG_RTE_LIBRTE_VHOST_USER=.\+/CONFIG_RTE_LIBRTE_VHOST_USER=$(VHOST_USER)/g' $(CONFIG_FILE_BASE)
 	$(AT)sed -i -e 's/CONFIG_RTE_LIBRTE_VHOST=./CONFIG_RTE_LIBRTE_VHOST=y/g' $(CONFIG_FILE_BASE)
 	$(AT)sed -i -e 's/CONFIG_RTE_LIBRTE_KNI=./CONFIG_RTE_LIBRTE_KNI=n/g' $(CONFIG_FILE_BASE)
+	$(AT)sed -i -e 's/CONFIG_RTE_LIBRTE_VHOST_NUMA=./CONFIG_RTE_LIBRTE_VHOST_NUMA=y/g' $(CONFIG_FILE_BASE)
 else
 	$(AT)sed -i -e 's/CONFIG_RTE_LIBRTE_VHOST_USER=.\+/CONFIG_RTE_LIBRTE_VHOST_USER=$(VHOST_USER)/g' $(CONFIG_FILE_LINUXAPP)
 	$(AT)sed -i -e 's/CONFIG_RTE_BUILD_COMBINE_LIBS=./CONFIG_RTE_BUILD_COMBINE_LIBS=y/g' $(CONFIG_FILE_LINUXAPP)
diff --git a/testcases/integration.py b/testcases/integration.py
index 88a6f12c..bec38624 100644
--- a/testcases/integration.py
+++ b/testcases/integration.py
@@ -18,12 +18,14 @@ import os
 import time
 import logging
 import copy
+import re
 
+from collections import OrderedDict
 from testcases import TestCase
 from conf import settings as S
-from collections import OrderedDict
 from tools import namespace
 from tools import veth
+from tools.teststepstools import TestStepsTools
 from core.loader import Loader
 
 CHECK_PREFIX = 'validate_'
@@ -61,17 +63,18 @@ class IntegrationTestCase(TestCase):
             """ Evaluates referrences to results from previous steps
             """
             def eval_param(param, STEP):
+                # pylint: disable=invalid-name
                 """ Helper function
                 """
                 if isinstance(param, str):
-                    tmp_param = ''
                     # evaluate every #STEP reference inside parameter itself
-                    for chunk in param.split('#'):
-                        if chunk.startswith('STEP['):
-                            tmp_param = tmp_param + str(eval(chunk))
-                        else:
-                            tmp_param = tmp_param + chunk
-                    return tmp_param
+                    macros = re.findall(r'#STEP\[[\w\[\]\-\'\"]+\]', param)
+                    if macros:
+                        for macro in macros:
+                            # pylint: disable=eval-used
+                            tmp_val = str(eval(macro[1:]))
+                            param = param.replace(macro, tmp_val)
+                    return param
                 elif isinstance(param, list) or isinstance(param, tuple):
                     tmp_list = []
                     for item in param:
@@ -136,6 +139,11 @@ class IntegrationTestCase(TestCase):
                                             test_object = namespace
                                         elif step[0] == 'veth':
                                             test_object = veth
+                                        elif step[0] == 'settings':
+                                            test_object = S
+                                        elif step[0] == 'tools':
+                                            test_object = TestStepsTools()
+                                            step[1] = step[1].title()
                                         elif step[0] == 'trafficgen':
                                             test_object = self._traffic_ctl
                                             # in case of send_traffic method, ensure that specified
@@ -149,10 +157,15 @@ class IntegrationTestCase(TestCase):
                                                 # initialize new VM
                                                 vnf_list[step[0]] = loader.get_vnf_class()()
                                             test_object = vnf_list[step[0]]
+                                        elif step[0] == 'wait':
+                                            input(os.linesep + "Step {}: Press Enter to continue with "
+                                                  "the next step...".format(i) + os.linesep + os.linesep)
+                                            continue
                                         else:
                                             self._logger.error("Unsupported test object %s", step[0])
                                             self._inttest = {'status' : False, 'details' : ' '.join(step)}
-                                            self.report_status("Step '{}'".format(' '.join(step)), self._inttest['status'])
+                                            self.report_status("Step '{}'".format(' '.join(step)),
+                                                               self._inttest['status'])
                                             break
 
                                         test_method = getattr(test_object, step[1])
@@ -163,7 +176,9 @@ class IntegrationTestCase(TestCase):
                                             callable(test_method) and callable(test_method_check):
 
                                             try:
-                                                step_params = eval_step_params(step[2:], step_result)
+                                                # eval parameters, but use only valid step_results
+                                                # to support negative indexes
+                                                step_params = eval_step_params(step[2:], step_result[:i])
                                                 step_log = '{} {}'.format(' '.join(step[:2]), step_params)
                                                 step_result[i] = test_method(*step_params)
                                                 self._logger.debug("Step %s '%s' results '%s'", i,
@@ -185,7 +200,7 @@ class IntegrationTestCase(TestCase):
                                                 for vnf in vnf_list:
                                                     vnf_list[vnf].stop()
                                                 break
-    
+
                                         self.report_status("Step {} - '{}'".format(i, step_log), step_ok)
                                         if not step_ok:
                                             self._inttest = {'status' : False, 'details' : step_log}
diff --git a/tools/teststepstools.py b/tools/teststepstools.py
new file mode 100644
index 00000000..d39f7f40
--- /dev/null
+++ b/tools/teststepstools.py
@@ -0,0 +1,90 @@
+# Copyright 2016 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Various helper functions for step driven testcases
+"""
+
+import re
+import logging
+import subprocess
+import locale
+
+class TestStepsTools(object):
+    """ Various tools and functions used by step driven testcases
+    """
+    # Functions use nonstandard names to avoid conflicts with
+    # standard python keywords.
+    # pylint: disable=invalid-name
+    def __init__(self):
+        """ TestStepsTools initialization
+        """
+        self._logger = logging.getLogger(__name__)
+
+    def Assert(self, condition):
+        """ Evaluate given `condition' and raise AssertionError
+            in case, that evaluation fails
+        """
+        try:
+            assert self.Eval(condition)
+        except AssertionError:
+            self._logger.error('Condition %s is not True', condition)
+            raise
+
+        return True
+
+    @staticmethod
+    def validate_Assert(result, dummy_condition):
+        """ Validate evaluation of given `condition'
+        """
+        return result
+
+    @staticmethod
+    def Eval(expression):
+        """ Evaluate python `expression' and return its result
+        """
+        # pylint: disable=eval-used
+        return eval(expression)
+
+    @staticmethod
+    def validate_Eval(result, dummy_expression):
+        """ Validate result of python `expression' evaluation
+        """
+        return result is not None
+
+    @staticmethod
+    def Exec(command, regex=None):
+        """ Execute a shell `command' and return its output filtered
+            out by optional `regex' expression.
+        """
+        try:
+            output = subprocess.check_output(command, shell=True)
+        except OSError:
+            return None
+
+        output = output.decode(locale.getdefaultlocale()[1])
+
+        if regex:
+            for line in output.split('\n'):
+                result = re.findall(regex, line)
+                if result:
+                    return result
+            return []
+
+        return output
+
+    @staticmethod
+    def validate_Exec(result, dummy_command, dummy_regex=None):
+        """ validate result of shell `command' execution
+        """
+        return result is not None
diff --git a/vnfs/qemu/qemu.py b/vnfs/qemu/qemu.py
index 01c16a0f..977b7bc1 100644
--- a/vnfs/qemu/qemu.py
+++ b/vnfs/qemu/qemu.py
@@ -83,9 +83,11 @@ class IVnfQemu(IVnf):
 
         name = 'Client%d' % self._number
         vnc = ':%d' % self._number
-        # don't use taskset to affinize main qemu process; It causes hangup
-        # of 2nd VM in case of DPDK. It also slows down VM responsivnes.
-        self._cmd = ['sudo', '-E', S.getValue('TOOLS')['qemu-system'],
+        # NOTE: affinization of main qemu process can cause hangup of 2nd VM
+        # in case of DPDK usage. It can also slow down VM response time.
+        cpumask = ",".join(S.getValue('GUEST_CORE_BINDING')[self._number])
+        self._cmd = ['sudo', '-E', 'taskset', '-c', cpumask,
+                     S.getValue('TOOLS')['qemu-system'],
                      '-m', S.getValue('GUEST_MEMORY')[self._number],
                      '-smp', str(S.getValue('GUEST_SMP')[self._number]),
                      '-cpu', 'host,migratable=off',