Merge "integration: Test vHost User numa awareness"

7 files changed, 387 insertions, 14 deletions

diff --git a/conf/__init__.py b/conf/__init__.py
index 88e8cec6..4d6f57fe 100644
--- a/conf/__init__.py
+++ b/conf/__init__.py
@@ -170,6 +170,7 @@ class Settings(object):
                 for macro, args, param, _, step in re.findall(_PARSE_PATTERN, value):
                     multi = int(step) if len(step) and int(step) else 1
                     if macro == '#EVAL':
+                        # pylint: disable=eval-used
                         tmp_result = str(eval(param))
                     elif macro == '#MAC':
                         mac_value = netaddr.EUI(param).value
@@ -204,6 +205,20 @@ class Settings(object):
         """
         return pprint.pformat(self.__dict__)
 
+    #
+    # validation methods used by step driven testcases
+    #
+    def validate_getValue(self, result, attr):
+        """Verifies, that correct value was returned
+        """
+        assert result == self.__dict__[attr]
+        return True
+
+    def validate_setValue(self, dummy_result, name, value):
+        """Verifies, that value was correctly set
+        """
+        assert value == self.__dict__[name]
+        return True
 
 settings = Settings()
 
diff --git a/conf/integration/01_testcases.conf b/conf/integration/01_testcases.conf
index a21eb214..7b2f7079 100644
--- a/conf/integration/01_testcases.conf
+++ b/conf/integration/01_testcases.conf
@@ -788,6 +788,79 @@ INTEGRATION_TESTS = [
         ] + STEP_VSWITCH_FLOWS_FINIT +
         STEP_VSWITCH_2PHY_6VM_FINIT
     },
+    {
+        # Testcase for verification of vHost User NUMA awareness feature
+        # introduced in DPDK v2.2. Test case will execute two VNFs, each
+        # pinned to different NUMA slot. After that it will verify that
+        # QEMU and PMD threads serving its interfaces are co-located
+        # at the same NUMA slot.
+        #
+        # Prerequisites:
+        #     * architecture with at least 2 NUMA slots
+        #     * OVS with DPDK support and DPDK v2.2 and newer
+        #     * OVS configuration utilizing both NUMA slots
+        #
+        #     Example of OVS configuration valid for DPDK v16.04 and cores
+        #     split between NUMA slots as follows:
+        #           node 0 cpus: 0 1 2 3 4 5 6 7 8 9
+        #           node 1 cpus: 10 11 12 13 14 15 16 17 18 19
+        #
+        #     VSWITCH_PMD_CPU_MASK = '1010'
+        #     VSWITCHD_DPDK_CONFIG = {
+        #         'dpdk-init' : 'true',
+        #         'dpdk-lcore-mask' : '0x4004',
+        #         'pmd-cpu-mask' : 'FF0FF',
+        #         'dpdk-socket-mem' : '1024,1024',
+        #     }
+        #
+        "Name": "vhost_numa_awareness",
+        "Deployment": "clean",
+        "Description": "vSwitch DPDK - verify that PMD threads are served "
+                       "by the same NUMA slot as QEMU instances",
+        "vSwitch" : "OvsDpdkVhost",
+        "TestSteps": STEP_VSWITCH_PVVP_INIT +                                       # STEP 0-6
+            [
+                # check that at least 2 numa slots are available
+                ['tools', 'exec', 'numactl -H', 'available: ([0-9]+)'],             # STEP 7
+                ['tools', 'assert', '#STEP[-1][0]>1'],                              # STEP 8
+                # store last 2 cores from numa slot 0
+                ['tools', 'exec', 'numactl -H', 'node 0 cpus:.*\s+(\\d+) (\\d+)$'], # STEP 9
+                # store last 2 cores from numa slot 1
+                ['tools', 'exec', 'numactl -H', 'node 1 cpus:.*\s+(\\d+) (\\d+)$'], # STEP 10
+                # pin VNF1 to 1st NUMA slot and VNF2 to 2nd NUMA slot
+                ['settings', 'setValue', 'GUEST_CORE_BINDING',                      # STEP 11
+                    [("#STEP[-2][0][0]", "#STEP[-2][0][1]"),
+                     ("#STEP[-1][0][0]", "#STEP[-1][0][1]")]
+                ],
+                # start 2 VNFs
+                ['vnf1', 'start'],                                                  # STEP 12
+                ['vnf2', 'start'],                                                  # STEP 13
+                # read paths to ovs utilities
+                ['settings', 'getValue', 'TOOLS'],                                  # STEP 14
+                # check that PMD thread serving VNF1 runs at NUMA slot 0
+                ## i.e. get numa slot ID serving dpdhvhostuser0...
+                ['tools', 'exec', "sudo #STEP[-1]['ovs-appctl'] "                   # STEP 15
+                    "dpif-netdev/pmd-rxq-show | "
+                    "sed -e '/dpdkvhostuser0/,$d' | tac",
+                    'pmd thread numa_id ([0-9])+'
+                ],
+                ## ...and check that it is NUMA slot 0
+                ['tools', 'assert', '#STEP[-1][0]==0'],                             # STEP 16
+                # check that PMD thread serving VNF2 runs at NUMA slot 1
+                ## i.e. get numa slot ID serving dpdhvhostuser2...
+                ['tools', 'exec', "sudo #STEP[-3]['ovs-appctl'] "                   # STEP 17
+                    "dpif-netdev/pmd-rxq-show | "
+                    "sed -e '/dpdkvhostuser2/,$d' | tac",
+                    'pmd thread numa_id ([0-9])+'
+                ],
+                ## ...and check that it is NUMA slot 1
+                ['tools', 'assert', '#STEP[-1][0]==1'],                             # STEP 18
+                # clean up
+                ['vnf2', 'stop'],                                                   # STEP 19
+                ['vnf1', 'stop'],                                                   # STEP 20
+            ] +
+            STEP_VSWITCH_PVVP_FINIT                                                 # STEP 21...
+    },
 ]
 
 # Example of TC definition with exact vSwitch, VNF and TRAFFICGEN values.
diff --git a/docs/userguide/integration.rst b/docs/userguide/integration.rst
index b0926d89..ceb11635 100755
--- a/docs/userguide/integration.rst
+++ b/docs/userguide/integration.rst
@@ -49,6 +49,173 @@ test will fail and terminate. The test will continue until a failure is detected
 or all steps pass. A csv report file is generated after a test completes with an
 OK or FAIL result.
 
+Test objects and their functions
+--------------------------------
+
+Every test step can call a function of one of the supported test objects. The list
+of supported objects and their most common functions follows:
+
+    * ``vswitch`` - provides functions for vSwitch configuration
+
+      List of supported functions:
+
+        * ``add_switch br_name`` - creates a new switch (bridge) with given ``br_name``
+        * ``del_switch br_name`` - deletes switch (bridge) with given ``br_name``
+        * ``add_phy_port br_name`` - adds a physical port into bridge specified by ``br_name``
+        * ``add_vport br_name`` - adds a virtual port into bridge specified by ``br_name``
+        * ``del_port br_name port_name`` - removes physical or virtual port specified by
+          ``port_name`` from bridge ``br_name``
+        * ``add_flow br_name flow`` - adds flow specified by ``flow`` dictionary into
+          the bridge ``br_name``; Content of flow dictionary will be passed to the vSwitch.
+          In case of Open vSwitch it will be passed to the ``ovs-ofctl add-flow`` command.
+          Please see Open vSwitch documentation for the list of supported flow parameters.
+        * ``del_flow br_name [flow]`` - deletes flow specified by ``flow`` dictionary from
+          bridge ``br_name``; In case that optional parameter ``flow`` is not specified
+          or set to an empty dictionary ``{}``, then all flows from bridge ``br_name``
+          will be deleted.
+        * ``dump_flows br_name`` - dumps all flows from bridge specified by ``br_name``
+        * ``enable_stp br_name`` - enables Spanning Tree Protocol for bridge ``br_name``
+        * ``disable_stp br_name`` - disables Spanning Tree Protocol for bridge ``br_name``
+        * ``enable_rstp br_name`` - enables Rapid Spanning Tree Protocol for bridge ``br_name``
+        * ``disable_rstp br_name`` - disables Rapid Spanning Tree Protocol for bridge ``br_name``
+
+        Examples:
+
+        .. code-block:: python
+
+            ['vswitch', 'add_switch', 'int_br0']
+
+            ['vswitch', 'del_switch', 'int_br0']
+
+            ['vswitch', 'add_phy_port', 'int_br0']
+
+            ['vswitch', 'del_port', 'int_br0', '#STEP[2][0]']
+
+            ['vswitch', 'add_flow', 'int_br0', {'in_port': '1', 'actions': ['output:2'],
+             'idle_timeout': '0'}],
+
+            ['vswitch', 'enable_rstp', 'int_br0']
+
+    * ``vnf[ID]`` - provides functions for deployment and termination of VNFs; Optional
+      alfanumerical ``ID`` is used for VNF identification in case that testcase
+      deploys multiple VNFs.
+
+      List of supported functions:
+
+        * ``start`` - starts a VNF based on VSPERF configuration
+        * ``stop`` - gracefully terminates given VNF
+
+        Examples:
+
+        .. code-block:: python
+
+            ['vnf1', 'start']
+            ['vnf2', 'start']
+            ['vnf2', 'stop']
+            ['vnf1', 'stop']
+
+    * ``trafficgen`` - triggers traffic generation
+
+      List of supported functions:
+
+        * ``send_traffic traffic`` - starts a traffic based on the vsperf configuration
+          and given ``traffic`` dictionary. More details about ``traffic`` dictionary
+          and its possible values are available at `Traffic Generator Integration Guide
+          <http://artifacts.opnfv.org/vswitchperf/docs/design/trafficgen_integration_guide.html#step-5-supported-traffic-types>`__
+
+        Examples:
+
+        .. code-block:: python
+
+            ['trafficgen', 'send_traffic', {'traffic_type' : 'throughput'}]
+
+            ['trafficgen', 'send_traffic', {'traffic_type' : 'back2back', 'bidir' : 'True'}]
+
+    * ``settings`` - reads or modifies VSPERF configuration
+
+      List of supported functions:
+
+        * ``getValue param`` - returns value of given ``param``
+        * ``setValue param value`` - sets value of ``param`` to given ``value``
+
+        Examples:
+
+        .. code-block:: python
+
+            ['settings', 'getValue', 'TOOLS']
+
+            ['settings', 'setValue', 'GUEST_USERNAME', ['root']]
+
+    * ``namespace`` - creates or modifies network namespaces
+
+      List of supported functions:
+
+        * ``create_namespace name`` - creates new namespace with given ``name``
+        * ``delete_namespace name`` - deletes namespace specified by its ``name``
+        * ``assign_port_to_namespace port name [port_up]`` - assigns NIC specified by ``port``
+          into given namespace ``name``; If optional parameter ``port_up`` is set to ``True``,
+          then port will be brought up.
+        * ``add_ip_to_namespace_eth port name addr cidr`` - assigns an IP address ``addr``/``cidr``
+          to the NIC specified by ``port`` within namespace ``name``
+        * ``reset_port_to_root port name`` - returns given ``port`` from namespace ``name`` back
+          to the root namespace
+
+        Examples:
+
+        .. code-block:: python
+
+            ['namespace', 'create_namespace', 'testns']
+
+            ['namespace', 'assign_port_to_namespace', 'eth0', 'testns']
+
+    * ``veth`` - manipulates with eth and veth devices
+
+      List of supported functions:
+
+        * ``add_veth_port port peer_port`` - adds a pair of veth ports named ``port`` and
+          ``peer_port``
+        * ``del_veth_port port peer_port`` - deletes a veth port pair specified by ``port``
+          and ``peer_port``
+        * ``bring_up_eth_port eth_port [namespace]`` - brings up ``eth_port`` in (optional)
+          ``namespace``
+
+        Examples:
+
+        .. code-block:: python
+
+            ['veth', 'add_veth_port', 'veth', 'veth1']
+
+            ['veth', 'bring_up_eth_port', 'eth1']
+
+    * ``tools`` - provides a set of helper functions
+
+      List of supported functions:
+
+        * ``Assert condition`` - evaluates given ``condition`` and raises ``AssertionError``
+          in case that condition is not ``True``
+        * ``Eval expression`` - evaluates given expression as a python code and returns
+          its result
+        * ``Exec command [regex]`` - executes a shell command and filters its output by
+          (optional) regular expression
+
+        Examples:
+
+        .. code-block:: python
+
+            ['tools', 'exec', 'numactl -H', 'available: ([0-9]+)']
+            ['tools', 'assert', '#STEP[-1][0]>1']
+
+    * ``wait`` - is used for test case interruption. This object doesn't have
+      any functions. Once reached, vsperf will pause test execution and waits
+      for press of ``Enter key``. It can be used during testcase design
+      for debugging purposes.
+
+      Examples:
+
+      .. code-block:: python
+
+        ['wait']
+
 Test Macros
 -----------
 
@@ -75,6 +242,16 @@ This test profile uses the vswitch add_vport method which returns a string
 value of the port added. This is later called by the del_port method using the
 name from step 1.
 
+It is also possible to use negative indexes in step macros. In that case
+``#STEP[-1]`` will refer to the result from previous step, ``#STEP[-2]``
+will refer to result of step called before previous step, etc. It means,
+that you could change ``STEP 2`` from previous example to achieve the same
+functionality:
+
+.. code-block:: python
+
+                ['vswitch', 'del_port', 'int_br0', '#STEP[-1][0]'],  # STEP 2
+
 Also commonly used steps can be created as a separate profile.
 
 .. code-block:: python
diff --git a/src/dpdk/Makefile b/src/dpdk/Makefile
index 1cd85e3e..29967029 100755
--- a/src/dpdk/Makefile
+++ b/src/dpdk/Makefile
@@ -72,6 +72,7 @@ ifdef CONFIG_FILE_BASE
 	$(AT)sed -i -e 's/CONFIG_RTE_LIBRTE_VHOST_USER=.\+/CONFIG_RTE_LIBRTE_VHOST_USER=$(VHOST_USER)/g' $(CONFIG_FILE_BASE)
 	$(AT)sed -i -e 's/CONFIG_RTE_LIBRTE_VHOST=./CONFIG_RTE_LIBRTE_VHOST=y/g' $(CONFIG_FILE_BASE)
 	$(AT)sed -i -e 's/CONFIG_RTE_LIBRTE_KNI=./CONFIG_RTE_LIBRTE_KNI=n/g' $(CONFIG_FILE_BASE)
+	$(AT)sed -i -e 's/CONFIG_RTE_LIBRTE_VHOST_NUMA=./CONFIG_RTE_LIBRTE_VHOST_NUMA=y/g' $(CONFIG_FILE_BASE)
 else
 	$(AT)sed -i -e 's/CONFIG_RTE_LIBRTE_VHOST_USER=.\+/CONFIG_RTE_LIBRTE_VHOST_USER=$(VHOST_USER)/g' $(CONFIG_FILE_LINUXAPP)
 	$(AT)sed -i -e 's/CONFIG_RTE_BUILD_COMBINE_LIBS=./CONFIG_RTE_BUILD_COMBINE_LIBS=y/g' $(CONFIG_FILE_LINUXAPP)
diff --git a/testcases/integration.py b/testcases/integration.py
index 88a6f12c..bec38624 100644
--- a/testcases/integration.py
+++ b/testcases/integration.py
@@ -18,12 +18,14 @@ import os
 import time
 import logging
 import copy
+import re
 
+from collections import OrderedDict
 from testcases import TestCase
 from conf import settings as S
-from collections import OrderedDict
 from tools import namespace
 from tools import veth
+from tools.teststepstools import TestStepsTools
 from core.loader import Loader
 
 CHECK_PREFIX = 'validate_'
@@ -61,17 +63,18 @@ class IntegrationTestCase(TestCase):
             """ Evaluates referrences to results from previous steps
             """
             def eval_param(param, STEP):
+                # pylint: disable=invalid-name
                 """ Helper function
                 """
                 if isinstance(param, str):
-                    tmp_param = ''
                     # evaluate every #STEP reference inside parameter itself
-                    for chunk in param.split('#'):
-                        if chunk.startswith('STEP['):
-                            tmp_param = tmp_param + str(eval(chunk))
-                        else:
-                            tmp_param = tmp_param + chunk
-                    return tmp_param
+                    macros = re.findall(r'#STEP\[[\w\[\]\-\'\"]+\]', param)
+                    if macros:
+                        for macro in macros:
+                            # pylint: disable=eval-used
+                            tmp_val = str(eval(macro[1:]))
+                            param = param.replace(macro, tmp_val)
+                    return param
                 elif isinstance(param, list) or isinstance(param, tuple):
                     tmp_list = []
                     for item in param:
@@ -136,6 +139,11 @@ class IntegrationTestCase(TestCase):
                                             test_object = namespace
                                         elif step[0] == 'veth':
                                             test_object = veth
+                                        elif step[0] == 'settings':
+                                            test_object = S
+                                        elif step[0] == 'tools':
+                                            test_object = TestStepsTools()
+                                            step[1] = step[1].title()
                                         elif step[0] == 'trafficgen':
                                             test_object = self._traffic_ctl
                                             # in case of send_traffic method, ensure that specified
@@ -149,10 +157,15 @@ class IntegrationTestCase(TestCase):
                                                 # initialize new VM
                                                 vnf_list[step[0]] = loader.get_vnf_class()()
                                             test_object = vnf_list[step[0]]
+                                        elif step[0] == 'wait':
+                                            input(os.linesep + "Step {}: Press Enter to continue with "
+                                                  "the next step...".format(i) + os.linesep + os.linesep)
+                                            continue
                                         else:
                                             self._logger.error("Unsupported test object %s", step[0])
                                             self._inttest = {'status' : False, 'details' : ' '.join(step)}
-                                            self.report_status("Step '{}'".format(' '.join(step)), self._inttest['status'])
+                                            self.report_status("Step '{}'".format(' '.join(step)),
+                                                               self._inttest['status'])
                                             break
 
                                         test_method = getattr(test_object, step[1])
@@ -163,7 +176,9 @@ class IntegrationTestCase(TestCase):
                                             callable(test_method) and callable(test_method_check):
 
                                             try:
-                                                step_params = eval_step_params(step[2:], step_result)
+                                                # eval parameters, but use only valid step_results
+                                                # to support negative indexes
+                                                step_params = eval_step_params(step[2:], step_result[:i])
                                                 step_log = '{} {}'.format(' '.join(step[:2]), step_params)
                                                 step_result[i] = test_method(*step_params)
                                                 self._logger.debug("Step %s '%s' results '%s'", i,
@@ -185,7 +200,7 @@ class IntegrationTestCase(TestCase):
                                                 for vnf in vnf_list:
                                                     vnf_list[vnf].stop()
                                                 break
-    
+
                                         self.report_status("Step {} - '{}'".format(i, step_log), step_ok)
                                         if not step_ok:
                                             self._inttest = {'status' : False, 'details' : step_log}
diff --git a/tools/teststepstools.py b/tools/teststepstools.py
new file mode 100644
index 00000000..d39f7f40
--- /dev/null
+++ b/tools/teststepstools.py
@@ -0,0 +1,90 @@
+# Copyright 2016 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Various helper functions for step driven testcases
+"""
+
+import re
+import logging
+import subprocess
+import locale
+
+class TestStepsTools(object):
+    """ Various tools and functions used by step driven testcases
+    """
+    # Functions use nonstandard names to avoid conflicts with
+    # standard python keywords.
+    # pylint: disable=invalid-name
+    def __init__(self):
+        """ TestStepsTools initialization
+        """
+        self._logger = logging.getLogger(__name__)
+
+    def Assert(self, condition):
+        """ Evaluate given `condition' and raise AssertionError
+            in case, that evaluation fails
+        """
+        try:
+            assert self.Eval(condition)
+        except AssertionError:
+            self._logger.error('Condition %s is not True', condition)
+            raise
+
+        return True
+
+    @staticmethod
+    def validate_Assert(result, dummy_condition):
+        """ Validate evaluation of given `condition'
+        """
+        return result
+
+    @staticmethod
+    def Eval(expression):
+        """ Evaluate python `expression' and return its result
+        """
+        # pylint: disable=eval-used
+        return eval(expression)
+
+    @staticmethod
+    def validate_Eval(result, dummy_expression):
+        """ Validate result of python `expression' evaluation
+        """
+        return result is not None
+
+    @staticmethod
+    def Exec(command, regex=None):
+        """ Execute a shell `command' and return its output filtered
+            out by optional `regex' expression.
+        """
+        try:
+            output = subprocess.check_output(command, shell=True)
+        except OSError:
+            return None
+
+        output = output.decode(locale.getdefaultlocale()[1])
+
+        if regex:
+            for line in output.split('\n'):
+                result = re.findall(regex, line)
+                if result:
+                    return result
+            return []
+
+        return output
+
+    @staticmethod
+    def validate_Exec(result, dummy_command, dummy_regex=None):
+        """ validate result of shell `command' execution
+        """
+        return result is not None
diff --git a/vnfs/qemu/qemu.py b/vnfs/qemu/qemu.py
index e654ac7e..ae0056dc 100644
--- a/vnfs/qemu/qemu.py
+++ b/vnfs/qemu/qemu.py
@@ -88,9 +88,11 @@ class IVnfQemu(IVnf):
 
         name = 'Client%d' % self._number
         vnc = ':%d' % self._number
-        # don't use taskset to affinize main qemu process; It causes hangup
-        # of 2nd VM in case of DPDK. It also slows down VM responsivnes.
-        self._cmd = ['sudo', '-E', S.getValue('TOOLS')['qemu-system'],
+        # NOTE: affinization of main qemu process can cause hangup of 2nd VM
+        # in case of DPDK usage. It can also slow down VM response time.
+        cpumask = ",".join(S.getValue('GUEST_CORE_BINDING')[self._number])
+        self._cmd = ['sudo', '-E', 'taskset', '-c', cpumask,
+                     S.getValue('TOOLS')['qemu-system'],
                      '-m', S.getValue('GUEST_MEMORY')[self._number],
                      '-smp', str(S.getValue('GUEST_SMP')[self._number]),
                      '-cpu', 'host,migratable=off',