diff options
| -rw-r--r-- | conf/__init__.py | 15 | ||||
| -rw-r--r-- | conf/integration/01_testcases.conf | 73 | ||||
| -rwxr-xr-x | docs/userguide/integration.rst | 177 | ||||
| -rwxr-xr-x | src/dpdk/Makefile | 1 | ||||
| -rw-r--r-- | testcases/integration.py | 37 | ||||
| -rw-r--r-- | tools/teststepstools.py | 90 | ||||
| -rw-r--r-- | vnfs/qemu/qemu.py | 8 |
7 files changed, 387 insertions, 14 deletions
diff --git a/conf/__init__.py b/conf/__init__.py index 88e8cec6..4d6f57fe 100644 --- a/conf/__init__.py +++ b/conf/__init__.py @@ -170,6 +170,7 @@ class Settings(object): for macro, args, param, _, step in re.findall(_PARSE_PATTERN, value): multi = int(step) if len(step) and int(step) else 1 if macro == '#EVAL': + # pylint: disable=eval-used tmp_result = str(eval(param)) elif macro == '#MAC': mac_value = netaddr.EUI(param).value @@ -204,6 +205,20 @@ class Settings(object): """ return pprint.pformat(self.__dict__) + # + # validation methods used by step driven testcases + # + def validate_getValue(self, result, attr): + """Verifies, that correct value was returned + """ + assert result == self.__dict__[attr] + return True + + def validate_setValue(self, dummy_result, name, value): + """Verifies, that value was correctly set + """ + assert value == self.__dict__[name] + return True settings = Settings() diff --git a/conf/integration/01_testcases.conf b/conf/integration/01_testcases.conf index a67702f8..084323b2 100644 --- a/conf/integration/01_testcases.conf +++ b/conf/integration/01_testcases.conf @@ -649,6 +649,79 @@ INTEGRATION_TESTS = [ ] + STEP_VSWITCH_FLOWS_FINIT + STEP_VSWITCH_2PHY_6VM_FINIT }, + { + # Testcase for verification of vHost User NUMA awareness feature + # introduced in DPDK v2.2. Test case will execute two VNFs, each + # pinned to different NUMA slot. After that it will verify that + # QEMU and PMD threads serving its interfaces are co-located + # at the same NUMA slot. + # + # Prerequisites: + # * architecture with at least 2 NUMA slots + # * OVS with DPDK support and DPDK v2.2 and newer + # * OVS configuration utilizing both NUMA slots + # + # Example of OVS configuration valid for DPDK v16.04 and cores + # split between NUMA slots as follows: + # node 0 cpus: 0 1 2 3 4 5 6 7 8 9 + # node 1 cpus: 10 11 12 13 14 15 16 17 18 19 + # + # VSWITCH_PMD_CPU_MASK = '1010' + # VSWITCHD_DPDK_CONFIG = { + # 'dpdk-init' : 'true', + # 'dpdk-lcore-mask' : '0x4004', + # 'pmd-cpu-mask' : 'FF0FF', + # 'dpdk-socket-mem' : '1024,1024', + # } + # + "Name": "vhost_numa_awareness", + "Deployment": "clean", + "Description": "vSwitch DPDK - verify that PMD threads are served " + "by the same NUMA slot as QEMU instances", + "vSwitch" : "OvsDpdkVhost", + "TestSteps": STEP_VSWITCH_PVVP_INIT + # STEP 0-6 + [ + # check that at least 2 numa slots are available + ['tools', 'exec', 'numactl -H', 'available: ([0-9]+)'], # STEP 7 + ['tools', 'assert', '#STEP[-1][0]>1'], # STEP 8 + # store last 2 cores from numa slot 0 + ['tools', 'exec', 'numactl -H', 'node 0 cpus:.*\s+(\\d+) (\\d+)$'], # STEP 9 + # store last 2 cores from numa slot 1 + ['tools', 'exec', 'numactl -H', 'node 1 cpus:.*\s+(\\d+) (\\d+)$'], # STEP 10 + # pin VNF1 to 1st NUMA slot and VNF2 to 2nd NUMA slot + ['settings', 'setValue', 'GUEST_CORE_BINDING', # STEP 11 + [("#STEP[-2][0][0]", "#STEP[-2][0][1]"), + ("#STEP[-1][0][0]", "#STEP[-1][0][1]")] + ], + # start 2 VNFs + ['vnf1', 'start'], # STEP 12 + ['vnf2', 'start'], # STEP 13 + # read paths to ovs utilities + ['settings', 'getValue', 'TOOLS'], # STEP 14 + # check that PMD thread serving VNF1 runs at NUMA slot 0 + ## i.e. get numa slot ID serving dpdhvhostuser0... + ['tools', 'exec', "sudo #STEP[-1]['ovs-appctl'] " # STEP 15 + "dpif-netdev/pmd-rxq-show | " + "sed -e '/dpdkvhostuser0/,$d' | tac", + 'pmd thread numa_id ([0-9])+' + ], + ## ...and check that it is NUMA slot 0 + ['tools', 'assert', '#STEP[-1][0]==0'], # STEP 16 + # check that PMD thread serving VNF2 runs at NUMA slot 1 + ## i.e. get numa slot ID serving dpdhvhostuser2... + ['tools', 'exec', "sudo #STEP[-3]['ovs-appctl'] " # STEP 17 + "dpif-netdev/pmd-rxq-show | " + "sed -e '/dpdkvhostuser2/,$d' | tac", + 'pmd thread numa_id ([0-9])+' + ], + ## ...and check that it is NUMA slot 1 + ['tools', 'assert', '#STEP[-1][0]==1'], # STEP 18 + # clean up + ['vnf2', 'stop'], # STEP 19 + ['vnf1', 'stop'], # STEP 20 + ] + + STEP_VSWITCH_PVVP_FINIT # STEP 21... + }, ] # Example of TC definition with exact vSwitch, VNF and TRAFFICGEN values. diff --git a/docs/userguide/integration.rst b/docs/userguide/integration.rst index b0926d89..ceb11635 100755 --- a/docs/userguide/integration.rst +++ b/docs/userguide/integration.rst @@ -49,6 +49,173 @@ test will fail and terminate. The test will continue until a failure is detected or all steps pass. A csv report file is generated after a test completes with an OK or FAIL result. +Test objects and their functions +-------------------------------- + +Every test step can call a function of one of the supported test objects. The list +of supported objects and their most common functions follows: + + * ``vswitch`` - provides functions for vSwitch configuration + + List of supported functions: + + * ``add_switch br_name`` - creates a new switch (bridge) with given ``br_name`` + * ``del_switch br_name`` - deletes switch (bridge) with given ``br_name`` + * ``add_phy_port br_name`` - adds a physical port into bridge specified by ``br_name`` + * ``add_vport br_name`` - adds a virtual port into bridge specified by ``br_name`` + * ``del_port br_name port_name`` - removes physical or virtual port specified by + ``port_name`` from bridge ``br_name`` + * ``add_flow br_name flow`` - adds flow specified by ``flow`` dictionary into + the bridge ``br_name``; Content of flow dictionary will be passed to the vSwitch. + In case of Open vSwitch it will be passed to the ``ovs-ofctl add-flow`` command. + Please see Open vSwitch documentation for the list of supported flow parameters. + * ``del_flow br_name [flow]`` - deletes flow specified by ``flow`` dictionary from + bridge ``br_name``; In case that optional parameter ``flow`` is not specified + or set to an empty dictionary ``{}``, then all flows from bridge ``br_name`` + will be deleted. + * ``dump_flows br_name`` - dumps all flows from bridge specified by ``br_name`` + * ``enable_stp br_name`` - enables Spanning Tree Protocol for bridge ``br_name`` + * ``disable_stp br_name`` - disables Spanning Tree Protocol for bridge ``br_name`` + * ``enable_rstp br_name`` - enables Rapid Spanning Tree Protocol for bridge ``br_name`` + * ``disable_rstp br_name`` - disables Rapid Spanning Tree Protocol for bridge ``br_name`` + + Examples: + + .. code-block:: python + + ['vswitch', 'add_switch', 'int_br0'] + + ['vswitch', 'del_switch', 'int_br0'] + + ['vswitch', 'add_phy_port', 'int_br0'] + + ['vswitch', 'del_port', 'int_br0', '#STEP[2][0]'] + + ['vswitch', 'add_flow', 'int_br0', {'in_port': '1', 'actions': ['output:2'], + 'idle_timeout': '0'}], + + ['vswitch', 'enable_rstp', 'int_br0'] + + * ``vnf[ID]`` - provides functions for deployment and termination of VNFs; Optional + alfanumerical ``ID`` is used for VNF identification in case that testcase + deploys multiple VNFs. + + List of supported functions: + + * ``start`` - starts a VNF based on VSPERF configuration + * ``stop`` - gracefully terminates given VNF + + Examples: + + .. code-block:: python + + ['vnf1', 'start'] + ['vnf2', 'start'] + ['vnf2', 'stop'] + ['vnf1', 'stop'] + + * ``trafficgen`` - triggers traffic generation + + List of supported functions: + + * ``send_traffic traffic`` - starts a traffic based on the vsperf configuration + and given ``traffic`` dictionary. More details about ``traffic`` dictionary + and its possible values are available at `Traffic Generator Integration Guide + <http://artifacts.opnfv.org/vswitchperf/docs/design/trafficgen_integration_guide.html#step-5-supported-traffic-types>`__ + + Examples: + + .. code-block:: python + + ['trafficgen', 'send_traffic', {'traffic_type' : 'throughput'}] + + ['trafficgen', 'send_traffic', {'traffic_type' : 'back2back', 'bidir' : 'True'}] + + * ``settings`` - reads or modifies VSPERF configuration + + List of supported functions: + + * ``getValue param`` - returns value of given ``param`` + * ``setValue param value`` - sets value of ``param`` to given ``value`` + + Examples: + + .. code-block:: python + + ['settings', 'getValue', 'TOOLS'] + + ['settings', 'setValue', 'GUEST_USERNAME', ['root']] + + * ``namespace`` - creates or modifies network namespaces + + List of supported functions: + + * ``create_namespace name`` - creates new namespace with given ``name`` + * ``delete_namespace name`` - deletes namespace specified by its ``name`` + * ``assign_port_to_namespace port name [port_up]`` - assigns NIC specified by ``port`` + into given namespace ``name``; If optional parameter ``port_up`` is set to ``True``, + then port will be brought up. + * ``add_ip_to_namespace_eth port name addr cidr`` - assigns an IP address ``addr``/``cidr`` + to the NIC specified by ``port`` within namespace ``name`` + * ``reset_port_to_root port name`` - returns given ``port`` from namespace ``name`` back + to the root namespace + + Examples: + + .. code-block:: python + + ['namespace', 'create_namespace', 'testns'] + + ['namespace', 'assign_port_to_namespace', 'eth0', 'testns'] + + * ``veth`` - manipulates with eth and veth devices + + List of supported functions: + + * ``add_veth_port port peer_port`` - adds a pair of veth ports named ``port`` and + ``peer_port`` + * ``del_veth_port port peer_port`` - deletes a veth port pair specified by ``port`` + and ``peer_port`` + * ``bring_up_eth_port eth_port [namespace]`` - brings up ``eth_port`` in (optional) + ``namespace`` + + Examples: + + .. code-block:: python + + ['veth', 'add_veth_port', 'veth', 'veth1'] + + ['veth', 'bring_up_eth_port', 'eth1'] + + * ``tools`` - provides a set of helper functions + + List of supported functions: + + * ``Assert condition`` - evaluates given ``condition`` and raises ``AssertionError`` + in case that condition is not ``True`` + * ``Eval expression`` - evaluates given expression as a python code and returns + its result + * ``Exec command [regex]`` - executes a shell command and filters its output by + (optional) regular expression + + Examples: + + .. code-block:: python + + ['tools', 'exec', 'numactl -H', 'available: ([0-9]+)'] + ['tools', 'assert', '#STEP[-1][0]>1'] + + * ``wait`` - is used for test case interruption. This object doesn't have + any functions. Once reached, vsperf will pause test execution and waits + for press of ``Enter key``. It can be used during testcase design + for debugging purposes. + + Examples: + + .. code-block:: python + + ['wait'] + Test Macros ----------- @@ -75,6 +242,16 @@ This test profile uses the vswitch add_vport method which returns a string value of the port added. This is later called by the del_port method using the name from step 1. +It is also possible to use negative indexes in step macros. In that case +``#STEP[-1]`` will refer to the result from previous step, ``#STEP[-2]`` +will refer to result of step called before previous step, etc. It means, +that you could change ``STEP 2`` from previous example to achieve the same +functionality: + +.. code-block:: python + + ['vswitch', 'del_port', 'int_br0', '#STEP[-1][0]'], # STEP 2 + Also commonly used steps can be created as a separate profile. .. code-block:: python diff --git a/src/dpdk/Makefile b/src/dpdk/Makefile index 1cd85e3e..29967029 100755 --- a/src/dpdk/Makefile +++ b/src/dpdk/Makefile @@ -72,6 +72,7 @@ ifdef CONFIG_FILE_BASE $(AT)sed -i -e 's/CONFIG_RTE_LIBRTE_VHOST_USER=.\+/CONFIG_RTE_LIBRTE_VHOST_USER=$(VHOST_USER)/g' $(CONFIG_FILE_BASE) $(AT)sed -i -e 's/CONFIG_RTE_LIBRTE_VHOST=./CONFIG_RTE_LIBRTE_VHOST=y/g' $(CONFIG_FILE_BASE) $(AT)sed -i -e 's/CONFIG_RTE_LIBRTE_KNI=./CONFIG_RTE_LIBRTE_KNI=n/g' $(CONFIG_FILE_BASE) + $(AT)sed -i -e 's/CONFIG_RTE_LIBRTE_VHOST_NUMA=./CONFIG_RTE_LIBRTE_VHOST_NUMA=y/g' $(CONFIG_FILE_BASE) else $(AT)sed -i -e 's/CONFIG_RTE_LIBRTE_VHOST_USER=.\+/CONFIG_RTE_LIBRTE_VHOST_USER=$(VHOST_USER)/g' $(CONFIG_FILE_LINUXAPP) $(AT)sed -i -e 's/CONFIG_RTE_BUILD_COMBINE_LIBS=./CONFIG_RTE_BUILD_COMBINE_LIBS=y/g' $(CONFIG_FILE_LINUXAPP) diff --git a/testcases/integration.py b/testcases/integration.py index 88a6f12c..bec38624 100644 --- a/testcases/integration.py +++ b/testcases/integration.py @@ -18,12 +18,14 @@ import os import time import logging import copy +import re +from collections import OrderedDict from testcases import TestCase from conf import settings as S -from collections import OrderedDict from tools import namespace from tools import veth +from tools.teststepstools import TestStepsTools from core.loader import Loader CHECK_PREFIX = 'validate_' @@ -61,17 +63,18 @@ class IntegrationTestCase(TestCase): """ Evaluates referrences to results from previous steps """ def eval_param(param, STEP): + # pylint: disable=invalid-name """ Helper function """ if isinstance(param, str): - tmp_param = '' # evaluate every #STEP reference inside parameter itself - for chunk in param.split('#'): - if chunk.startswith('STEP['): - tmp_param = tmp_param + str(eval(chunk)) - else: - tmp_param = tmp_param + chunk - return tmp_param + macros = re.findall(r'#STEP\[[\w\[\]\-\'\"]+\]', param) + if macros: + for macro in macros: + # pylint: disable=eval-used + tmp_val = str(eval(macro[1:])) + param = param.replace(macro, tmp_val) + return param elif isinstance(param, list) or isinstance(param, tuple): tmp_list = [] for item in param: @@ -136,6 +139,11 @@ class IntegrationTestCase(TestCase): test_object = namespace elif step[0] == 'veth': test_object = veth + elif step[0] == 'settings': + test_object = S + elif step[0] == 'tools': + test_object = TestStepsTools() + step[1] = step[1].title() elif step[0] == 'trafficgen': test_object = self._traffic_ctl # in case of send_traffic method, ensure that specified @@ -149,10 +157,15 @@ class IntegrationTestCase(TestCase): # initialize new VM vnf_list[step[0]] = loader.get_vnf_class()() test_object = vnf_list[step[0]] + elif step[0] == 'wait': + input(os.linesep + "Step {}: Press Enter to continue with " + "the next step...".format(i) + os.linesep + os.linesep) + continue else: self._logger.error("Unsupported test object %s", step[0]) self._inttest = {'status' : False, 'details' : ' '.join(step)} - self.report_status("Step '{}'".format(' '.join(step)), self._inttest['status']) + self.report_status("Step '{}'".format(' '.join(step)), + self._inttest['status']) break test_method = getattr(test_object, step[1]) @@ -163,7 +176,9 @@ class IntegrationTestCase(TestCase): callable(test_method) and callable(test_method_check): try: - step_params = eval_step_params(step[2:], step_result) + # eval parameters, but use only valid step_results + # to support negative indexes + step_params = eval_step_params(step[2:], step_result[:i]) step_log = '{} {}'.format(' '.join(step[:2]), step_params) step_result[i] = test_method(*step_params) self._logger.debug("Step %s '%s' results '%s'", i, @@ -185,7 +200,7 @@ class IntegrationTestCase(TestCase): for vnf in vnf_list: vnf_list[vnf].stop() break - + self.report_status("Step {} - '{}'".format(i, step_log), step_ok) if not step_ok: self._inttest = {'status' : False, 'details' : step_log} diff --git a/tools/teststepstools.py b/tools/teststepstools.py new file mode 100644 index 00000000..d39f7f40 --- /dev/null +++ b/tools/teststepstools.py @@ -0,0 +1,90 @@ +# Copyright 2016 Intel Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Various helper functions for step driven testcases +""" + +import re +import logging +import subprocess +import locale + +class TestStepsTools(object): + """ Various tools and functions used by step driven testcases + """ + # Functions use nonstandard names to avoid conflicts with + # standard python keywords. + # pylint: disable=invalid-name + def __init__(self): + """ TestStepsTools initialization + """ + self._logger = logging.getLogger(__name__) + + def Assert(self, condition): + """ Evaluate given `condition' and raise AssertionError + in case, that evaluation fails + """ + try: + assert self.Eval(condition) + except AssertionError: + self._logger.error('Condition %s is not True', condition) + raise + + return True + + @staticmethod + def validate_Assert(result, dummy_condition): + """ Validate evaluation of given `condition' + """ + return result + + @staticmethod + def Eval(expression): + """ Evaluate python `expression' and return its result + """ + # pylint: disable=eval-used + return eval(expression) + + @staticmethod + def validate_Eval(result, dummy_expression): + """ Validate result of python `expression' evaluation + """ + return result is not None + + @staticmethod + def Exec(command, regex=None): + """ Execute a shell `command' and return its output filtered + out by optional `regex' expression. + """ + try: + output = subprocess.check_output(command, shell=True) + except OSError: + return None + + output = output.decode(locale.getdefaultlocale()[1]) + + if regex: + for line in output.split('\n'): + result = re.findall(regex, line) + if result: + return result + return [] + + return output + + @staticmethod + def validate_Exec(result, dummy_command, dummy_regex=None): + """ validate result of shell `command' execution + """ + return result is not None diff --git a/vnfs/qemu/qemu.py b/vnfs/qemu/qemu.py index 01c16a0f..977b7bc1 100644 --- a/vnfs/qemu/qemu.py +++ b/vnfs/qemu/qemu.py @@ -83,9 +83,11 @@ class IVnfQemu(IVnf): name = 'Client%d' % self._number vnc = ':%d' % self._number - # don't use taskset to affinize main qemu process; It causes hangup - # of 2nd VM in case of DPDK. It also slows down VM responsivnes. - self._cmd = ['sudo', '-E', S.getValue('TOOLS')['qemu-system'], + # NOTE: affinization of main qemu process can cause hangup of 2nd VM + # in case of DPDK usage. It can also slow down VM response time. + cpumask = ",".join(S.getValue('GUEST_CORE_BINDING')[self._number]) + self._cmd = ['sudo', '-E', 'taskset', '-c', cpumask, + S.getValue('TOOLS')['qemu-system'], '-m', S.getValue('GUEST_MEMORY')[self._number], '-smp', str(S.getValue('GUEST_SMP')[self._number]), '-cpu', 'host,migratable=off', |