diff options
| -rw-r--r-- | core/__init__.py | 2 | ||||
| -rw-r--r-- | core/component_factory.py | 10 | ||||
| -rw-r--r-- | core/traffic_controller.py | 133 | ||||
| -rw-r--r-- | core/traffic_controller_rfc2544.py | 108 | ||||
| -rw-r--r-- | core/traffic_controller_rfc2889.py | 107 | ||||
| -rw-r--r-- | core/vnf_controller.py | 17 | ||||
| -rw-r--r-- | docs/userguide/index.rst | 1 | ||||
| -rwxr-xr-x | docs/userguide/integration.rst | 573 | ||||
| -rw-r--r-- | docs/userguide/teststeps.rst | 651 | ||||
| -rw-r--r-- | testcases/integration.py | 192 | ||||
| -rw-r--r-- | testcases/performance.py | 2 | ||||
| -rw-r--r-- | testcases/testcase.py | 224 | ||||
| -rw-r--r-- | vnfs/qemu/qemu.py | 43 |
13 files changed, 1055 insertions, 1008 deletions
diff --git a/core/__init__.py b/core/__init__.py index e39ec88e..d16401ea 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -15,5 +15,5 @@ """Core structural interfaces and their implementations """ import core.component_factory -from core.traffic_controller import (ITrafficController) +from core.traffic_controller import (TrafficController) from core.vswitch_controller import (IVswitchController) diff --git a/core/component_factory.py b/core/component_factory.py index ef7ba86f..236a61ed 100644 --- a/core/component_factory.py +++ b/core/component_factory.py @@ -46,7 +46,7 @@ def create_traffic(traffic_type, trafficgen_class): :param traffic_type: Name of traffic type :param trafficgen_class: Reference to traffic generator class to be used. - :return: A new ITrafficController + :return: A new TrafficController """ if traffic_type.lower().startswith('rfc2889'): return TrafficControllerRFC2889(trafficgen_class) @@ -87,7 +87,7 @@ def create_vswitch(deployment_scenario, vswitch_class, traffic, raise RuntimeError("Unknown deployment scenario '{}'.".format(deployment_scenario)) -def create_vnf(deployment_scenario, vnf_class): +def create_vnf(deployment_scenario, vnf_class, extra_vnfs): """Return a new VnfController for the deployment_scenario. The returned controller is configured with the given VNF class. @@ -96,9 +96,13 @@ def create_vnf(deployment_scenario, vnf_class): :param deployment_scenario: The deployment scenario name :param vswitch_class: Reference to vSwitch class to be used. + :param extra_vnfs: The number of VNFs not involved in given + deployment scenario. It will be used to correctly expand + configuration values and initialize shared dirs. This parameter + is used in case, that additional VNFs are executed by TestSteps. :return: VnfController for the deployment_scenario """ - return VnfController(deployment_scenario, vnf_class) + return VnfController(deployment_scenario, vnf_class, extra_vnfs) def create_collector(collector_class, result_dir, test_name): """Return a new Collector of the given class diff --git a/core/traffic_controller.py b/core/traffic_controller.py index 428e91f8..a6c1bd8d 100644 --- a/core/traffic_controller.py +++ b/core/traffic_controller.py @@ -1,4 +1,4 @@ -# Copyright 2015 Intel Corporation. +# Copyright 2015-2016 Intel Corporation. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,17 +12,105 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Interface to traffic controllers +"""Base class for traffic controllers """ -class ITrafficController(object): - """Abstract class which defines a traffic controller object +import logging +import os +import time + +from core.results.results_constants import ResultsConstants +from conf import settings +from conf import get_test_param + +class TrafficController(object): + """Base class which defines a common functionality for all traffic + controller classes. Used to setup and control a traffic generator for a particular deployment scenario. """ + def __init__(self, traffic_gen_class): + """Initialization common for all types of traffic controllers + + :param traffic_gen_class: The traffic generator class to be used. + """ + self._type = None + self._logger = logging.getLogger(__name__) + self._logger.debug("__init__") + self._traffic_gen_class = traffic_gen_class() + self._traffic_started = False + self._traffic_started_call_count = 0 + self._duration = int(get_test_param('duration', 30)) + self._lossrate = float(get_test_param('lossrate', 0.0)) + self._mode = settings.getValue('mode').lower() + self._results = [] + + # If set, comma separated packet_sizes value from --test_params + # on cli takes precedence over value in settings file. + self._packet_sizes = None + packet_sizes_cli = get_test_param('pkt_sizes') + if packet_sizes_cli: + self._packet_sizes = [int(x.strip()) + for x in packet_sizes_cli.split(',')] + else: + self._packet_sizes = settings.getValue('TRAFFICGEN_PKT_SIZES') + + def __enter__(self): + """Call initialisation function. + """ + self._traffic_gen_class.connect() + + def __exit__(self, type_, value, traceback): + """Stop traffic, clean up. + """ + if self._traffic_started: + self.stop_traffic() + + def _append_results(self, result_dict, packet_size): + """Adds common values to traffic generator results. + + :param result_dict: Dictionary containing results from trafficgen + :param packet_size: Packet size value. + + :returns: dictionary of results with additional entries. + """ + + ret_value = result_dict - def send_traffic(self, traffic): + ret_value[ResultsConstants.TYPE] = self._type + ret_value[ResultsConstants.PACKET_SIZE] = str(packet_size) + + return ret_value + + def traffic_required(self): + """Checks selected '--mode' of traffic generator and performs + its specific handling. + + :returns: True - in case that traffic generator should be executed + False - if traffic generation is not required + """ + if self._mode == 'trafficgen-off': + time.sleep(2) + self._logger.debug("All is set. Please run traffic generator manually.") + input(os.linesep + "Press Enter to terminate vswitchperf..." + + os.linesep + os.linesep) + return False + elif self._mode == 'trafficgen-pause': + time.sleep(2) + while True: + choice = input(os.linesep + 'Transmission paused, should' + ' transmission be resumed? [y/n]' + os.linesep).lower() + if choice in ('yes', 'y', 'ye'): + return True + elif choice in ('no', 'n'): + self._logger.info("Traffic transmission will be skipped.") + return False + else: + print("Please respond with 'yes', 'y', 'no' or 'n' ", end='') + return True + + def send_traffic(self, dummy_traffic): """Triggers traffic to be sent from the traffic generator. This is a blocking function. @@ -33,7 +121,7 @@ class ITrafficController(object): "The TrafficController does not implement", "the \"send_traffic\" function.") - def send_traffic_async(self, traffic, function): + def send_traffic_async(self, dummy_traffic, dummy_function): """Triggers traffic to be sent asynchronously. This is not a blocking function. @@ -55,6 +143,33 @@ class ITrafficController(object): def stop_traffic(self): """Kills traffic being sent from the traffic generator. """ - raise NotImplementedError( - "The TrafficController does not implement", - "the \"stop_traffic\" function.") + self._logger.debug("stop_traffic()") + + def print_results(self): + """IResult interface implementation. + """ + counter = 0 + for item in self._results: + logging.info("Record: " + str(counter)) + counter += 1 + for(key, value) in list(item.items()): + logging.info(" Key: " + str(key) + + ", Value: " + str(value)) + + def get_results(self): + """IResult interface implementation. + """ + return self._results + + def validate_send_traffic(self, dummy_result, dummy_traffic): + """Verify that send traffic has succeeded + """ + if len(self._results): + if 'b2b_frames' in self._results[-1]: + return float(self._results[-1]['b2b_frames']) > 0 + elif 'throughput_rx_fps' in self._results[-1]: + return float(self._results[-1]['throughput_rx_fps']) > 0 + else: + return True + else: + return False diff --git a/core/traffic_controller_rfc2544.py b/core/traffic_controller_rfc2544.py index af09deff..874c3ae7 100644 --- a/core/traffic_controller_rfc2544.py +++ b/core/traffic_controller_rfc2544.py @@ -1,4 +1,4 @@ -# Copyright 2015 Intel Corporation. +# Copyright 2015-2016 Intel Corporation. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,16 +13,12 @@ # limitations under the License. """RFC2544 Traffic Controller implementation. """ -import logging - -from core.traffic_controller import ITrafficController -from core.results.results_constants import ResultsConstants +from core.traffic_controller import TrafficController from core.results.results import IResults -from conf import settings from conf import get_test_param -class TrafficControllerRFC2544(ITrafficController, IResults): +class TrafficControllerRFC2544(TrafficController, IResults): """Traffic controller for RFC2544 traffic Used to setup and control a traffic generator for an RFC2544 deployment @@ -34,60 +30,15 @@ class TrafficControllerRFC2544(ITrafficController, IResults): :param traffic_gen_class: The traffic generator class to be used. """ - self._logger = logging.getLogger(__name__) - self._logger.debug("__init__") - self._traffic_gen_class = traffic_gen_class() - self._traffic_started = False - self._traffic_started_call_count = 0 + super(TrafficControllerRFC2544, self).__init__(traffic_gen_class) + self._type = 'rfc2544' self._tests = int(get_test_param('rfc2544_tests', 1)) - self._duration = int(get_test_param('duration', 30)) - self._lossrate = float(get_test_param('lossrate', 0.0)) - self._results = [] - - # If set, comma separated packet_sizes value from --test_params - # on cli takes precedence over value in settings file. - self._packet_sizes = None - packet_sizes_cli = get_test_param('pkt_sizes') - if packet_sizes_cli: - self._packet_sizes = [int(x.strip()) - for x in packet_sizes_cli.split(',')] - else: - self._packet_sizes = settings.getValue('TRAFFICGEN_PKT_SIZES') - - def __enter__(self): - """Call initialisation function. - """ - self._traffic_gen_class.connect() - - def __exit__(self, type_, value, traceback): - """Stop traffic, clean up. - """ - if self._traffic_started: - self.stop_traffic() - - @staticmethod - def _append_results(result_dict, packet_size): - """Adds common values to traffic generator results. - - :param result_dict: Dictionary containing results from trafficgen - :param packet_size: Packet size value. - - :returns: dictionary of results with additional entries. - """ - - ret_value = result_dict - - # TODO Old TOIT controller had knowledge about scenario beeing - # executed, should new controller also fill Configuration & ID, - # or this should be passed to TestCase? - ret_value[ResultsConstants.TYPE] = 'rfc2544' - ret_value[ResultsConstants.PACKET_SIZE] = str(packet_size) - - return ret_value def send_traffic(self, traffic): - """See ITrafficController for description + """See TrafficController for description """ + if not self.traffic_required(): + return self._logger.debug('send_traffic with ' + str(self._traffic_gen_class)) @@ -109,13 +60,14 @@ class TrafficControllerRFC2544(ITrafficController, IResults): result = self._traffic_gen_class.send_rfc2544_throughput( traffic, tests=self._tests, duration=self._duration, lossrate=self._lossrate) - result = TrafficControllerRFC2544._append_results(result, - packet_size) + result = self._append_results(result, packet_size) self._results.append(result) def send_traffic_async(self, traffic, function): - """See ITrafficController for description + """See TrafficController for description """ + if not self.traffic_required(): + return self._logger.debug('send_traffic_async with ' + str(self._traffic_gen_class)) @@ -131,40 +83,6 @@ class TrafficControllerRFC2544(ITrafficController, IResults): else: function['function']() result = self._traffic_gen_class.wait_rfc2544_throughput() - result = TrafficControllerRFC2544._append_results(result, - packet_size) + result = self._append_results(result, packet_size) self._results.append(result) - def stop_traffic(self): - """Kills traffic being sent from the traffic generator. - """ - self._logger.debug("stop_traffic()") - - def print_results(self): - """IResult interface implementation. - """ - counter = 0 - for item in self._results: - logging.info("Record: " + str(counter)) - counter += 1 - for(key, value) in list(item.items()): - logging.info(" Key: " + str(key) + - ", Value: " + str(value)) - - def get_results(self): - """IResult interface implementation. - """ - return self._results - - def validate_send_traffic(self, dummy_result, dummy_traffic): - """Verify that send traffic has succeeded - """ - if len(self._results): - if 'b2b_frames' in self._results[-1]: - return float(self._results[-1]['b2b_frames']) > 0 - elif 'throughput_rx_fps' in self._results[-1]: - return float(self._results[-1]['throughput_rx_fps']) > 0 - else: - return True - else: - return False diff --git a/core/traffic_controller_rfc2889.py b/core/traffic_controller_rfc2889.py index a97a47d3..a2e12e6d 100644 --- a/core/traffic_controller_rfc2889.py +++ b/core/traffic_controller_rfc2889.py @@ -1,15 +1,24 @@ +# Copyright 2016 Spirent Communications, Intel Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """RFC2889 Traffic Controller implementation. """ -import logging - -from core.traffic_controller import ITrafficController -from core.results.results_constants import ResultsConstants +from core.traffic_controller import TrafficController from core.results.results import IResults -from conf import settings from conf import get_test_param -class TrafficControllerRFC2889(ITrafficController, IResults): +class TrafficControllerRFC2889(TrafficController, IResults): """Traffic controller for RFC2889 traffic Used to setup and control a traffic generator for an RFC2889 deployment @@ -21,56 +30,15 @@ class TrafficControllerRFC2889(ITrafficController, IResults): :param traffic_gen_class: The traffic generator class to be used. """ - self._logger = logging.getLogger(__name__) - self._logger.debug("__init__") - self._traffic_gen_class = traffic_gen_class() - self._traffic_started = False - self._traffic_started_call_count = 0 + super(TrafficControllerRFC2889, self).__init__(traffic_gen_class) + self._type = 'rfc2889' self._trials = int(get_test_param('rfc2889_trials', 1)) - self._duration = int(get_test_param('duration', 30)) - self._results = [] - - # If set, comma separated packet_sizes value from --test_params - # on cli takes precedence over value in settings file. - self._packet_sizes = None - packet_sizes_cli = get_test_param('pkt_sizes') - if packet_sizes_cli: - self._packet_sizes = [int(x.strip()) - for x in packet_sizes_cli.split(',')] - else: - self._packet_sizes = settings.getValue('TRAFFICGEN_PKT_SIZES') - - def __enter__(self): - """Call initialisation function. - """ - self._traffic_gen_class.connect() - - def __exit__(self, type_, value, traceback): - """Stop traffic, clean up. - """ - if self._traffic_started: - self.stop_traffic() - - @staticmethod - def _append_results(result_dict, packet_size): - """Adds common values to traffic generator results. - - :param result_dict: Dictionary containing results from trafficgen - :param packet_size: Packet size value. - - :returns: dictionary of results with additional entries. - """ - - ret_value = result_dict - - ret_value[ResultsConstants.TYPE] = 'rfc2889' - ret_value[ResultsConstants.PACKET_SIZE] = str(packet_size) - - return ret_value def send_traffic(self, traffic): - """See ITrafficController for description + """See TrafficController for description """ + if not self.traffic_required(): + return self._logger.debug('send_traffic with ' + str(self._traffic_gen_class)) @@ -92,13 +60,14 @@ class TrafficControllerRFC2889(ITrafficController, IResults): result = self._traffic_gen_class.send_rfc2889_forwarding( traffic, tests=self._trials, duration=self._duration) - result = TrafficControllerRFC2889._append_results(result, - packet_size) + result = self._append_results(result, packet_size) self._results.append(result) def send_traffic_async(self, traffic, function): - """See ITrafficController for description + """See TrafficController for description """ + if not self.traffic_required(): + return self._logger.debug('send_traffic_async with ' + str(self._traffic_gen_class)) @@ -115,32 +84,6 @@ class TrafficControllerRFC2889(ITrafficController, IResults): function['function']() result = self._traffic_gen_class.wait_rfc2889_forwarding( traffic, trials=self._trials, duration=self._duration) - result = TrafficControllerRFC2889._append_results(result, - packet_size) + result = self._append_results(result, packet_size) self._results.append(result) - def stop_traffic(self): - """Kills traffic being sent from the traffic generator. - """ - self._logger.debug("stop_traffic()") - - def print_results(self): - """IResult interface implementation. - """ - counter = 0 - for item in self._results: - logging.info("Record: " + str(counter)) - counter += 1 - for(key, value) in list(item.items()): - logging.info(" Key: " + str(key) + - ", Value: " + str(value)) - - def get_results(self): - """IResult interface implementation. - """ - return self._results - - def validate_send_traffic(self, result, traffic): - """Verify that send traffic has succeeded - """ - return True diff --git a/core/vnf_controller.py b/core/vnf_controller.py index 3e472f04..937cd5cc 100644 --- a/core/vnf_controller.py +++ b/core/vnf_controller.py @@ -31,10 +31,14 @@ class VnfController(object): _vnfs: A list of vnfs controlled by the controller. """ - def __init__(self, deployment, vnf_class): + def __init__(self, deployment, vnf_class, extra_vnfs): """Sets up the VNF infrastructure based on deployment scenario :param vnf_class: The VNF class to be used. + :param extra_vnfs: The number of VNFs not involved in given + deployment scenario. It will be used to correctly expand + configuration values and initialize shared dirs. This parameter + is used in case, that additional VNFs are executed by TestSteps. """ # reset VNF ID counter for each testcase IVnf.reset_vnf_counter() @@ -57,9 +61,9 @@ class VnfController(object): # without VNFs like p2p vm_number = 0 - if vm_number: - self._logger.debug('Check configuration for %s guests.', vm_number) - settings.check_vm_settings(vm_number) + if vm_number + extra_vnfs > 0: + self._logger.debug('Check configuration for %s guests.', vm_number + extra_vnfs) + settings.check_vm_settings(vm_number + extra_vnfs) # enforce that GUEST_NIC_NR is 1 or even number of NICs updated = False nics_nr = settings.getValue('GUEST_NICS_NR') @@ -73,10 +77,11 @@ class VnfController(object): 'was updated to GUEST_NICS_NR = %s', settings.getValue('GUEST_NICS_NR')) + if vm_number: self._vnfs = [vnf_class() for _ in range(vm_number)] - self._logger.debug('__init__ ' + str(len(self._vnfs)) + - ' VNF[s] with ' + ' '.join(map(str, self._vnfs))) + self._logger.debug('__init__ ' + str(len(self._vnfs)) + + ' VNF[s] with ' + ' '.join(map(str, self._vnfs))) def get_vnfs(self): """Returns a list of vnfs controlled by this controller. diff --git a/docs/userguide/index.rst b/docs/userguide/index.rst index 1a796dbf..a1cce262 100644 --- a/docs/userguide/index.rst +++ b/docs/userguide/index.rst @@ -11,5 +11,6 @@ VSPERF User Guide :maxdepth: 3 testusage.rst + teststeps.rst integration.rst yardstick.rst diff --git a/docs/userguide/integration.rst b/docs/userguide/integration.rst index 20d6b70c..5b26a716 100755 --- a/docs/userguide/integration.rst +++ b/docs/userguide/integration.rst @@ -33,579 +33,6 @@ view the current test list simply execute the following command: The standard tests included are defined inside the ``conf/integration/01_testcases.conf`` file. -Test Steps ----------- - -Execution of integration tests are done on a step by step work flow starting -with step 0 as defined inside the test case. Each step of the test increments -the step number by one which is indicated in the log. - -.. code-block:: console - - (testcases.integration) - Step 1 - 'vswitch add_switch ['int_br1']' ... OK - -Each step in the test case is validated. If a step does not pass validation the -test will fail and terminate. The test will continue until a failure is detected -or all steps pass. A csv report file is generated after a test completes with an -OK or FAIL result. - -Test objects and their functions --------------------------------- - -Every test step can call a function of one of the supported test objects. The list -of supported objects and their most common functions follows: - - * ``vswitch`` - provides functions for vSwitch configuration - - List of supported functions: - - * ``add_switch br_name`` - creates a new switch (bridge) with given ``br_name`` - * ``del_switch br_name`` - deletes switch (bridge) with given ``br_name`` - * ``add_phy_port br_name`` - adds a physical port into bridge specified by ``br_name`` - * ``add_vport br_name`` - adds a virtual port into bridge specified by ``br_name`` - * ``del_port br_name port_name`` - removes physical or virtual port specified by - ``port_name`` from bridge ``br_name`` - * ``add_flow br_name flow`` - adds flow specified by ``flow`` dictionary into - the bridge ``br_name``; Content of flow dictionary will be passed to the vSwitch. - In case of Open vSwitch it will be passed to the ``ovs-ofctl add-flow`` command. - Please see Open vSwitch documentation for the list of supported flow parameters. - * ``del_flow br_name [flow]`` - deletes flow specified by ``flow`` dictionary from - bridge ``br_name``; In case that optional parameter ``flow`` is not specified - or set to an empty dictionary ``{}``, then all flows from bridge ``br_name`` - will be deleted. - * ``dump_flows br_name`` - dumps all flows from bridge specified by ``br_name`` - * ``enable_stp br_name`` - enables Spanning Tree Protocol for bridge ``br_name`` - * ``disable_stp br_name`` - disables Spanning Tree Protocol for bridge ``br_name`` - * ``enable_rstp br_name`` - enables Rapid Spanning Tree Protocol for bridge ``br_name`` - * ``disable_rstp br_name`` - disables Rapid Spanning Tree Protocol for bridge ``br_name`` - - Examples: - - .. code-block:: python - - ['vswitch', 'add_switch', 'int_br0'] - - ['vswitch', 'del_switch', 'int_br0'] - - ['vswitch', 'add_phy_port', 'int_br0'] - - ['vswitch', 'del_port', 'int_br0', '#STEP[2][0]'] - - ['vswitch', 'add_flow', 'int_br0', {'in_port': '1', 'actions': ['output:2'], - 'idle_timeout': '0'}], - - ['vswitch', 'enable_rstp', 'int_br0'] - - * ``vnf[ID]`` - provides functions for deployment and termination of VNFs; Optional - alfanumerical ``ID`` is used for VNF identification in case that testcase - deploys multiple VNFs. - - List of supported functions: - - * ``start`` - starts a VNF based on VSPERF configuration - * ``stop`` - gracefully terminates given VNF - - Examples: - - .. code-block:: python - - ['vnf1', 'start'] - ['vnf2', 'start'] - ['vnf2', 'stop'] - ['vnf1', 'stop'] - - * ``trafficgen`` - triggers traffic generation - - List of supported functions: - - * ``send_traffic traffic`` - starts a traffic based on the vsperf configuration - and given ``traffic`` dictionary. More details about ``traffic`` dictionary - and its possible values are available at `Traffic Generator Integration Guide - <http://artifacts.opnfv.org/vswitchperf/docs/design/trafficgen_integration_guide.html#step-5-supported-traffic-types>`__ - - Examples: - - .. code-block:: python - - ['trafficgen', 'send_traffic', {'traffic_type' : 'throughput'}] - - ['trafficgen', 'send_traffic', {'traffic_type' : 'back2back', 'bidir' : 'True'}] - - * ``settings`` - reads or modifies VSPERF configuration - - List of supported functions: - - * ``getValue param`` - returns value of given ``param`` - * ``setValue param value`` - sets value of ``param`` to given ``value`` - - Examples: - - .. code-block:: python - - ['settings', 'getValue', 'TOOLS'] - - ['settings', 'setValue', 'GUEST_USERNAME', ['root']] - - * ``namespace`` - creates or modifies network namespaces - - List of supported functions: - - * ``create_namespace name`` - creates new namespace with given ``name`` - * ``delete_namespace name`` - deletes namespace specified by its ``name`` - * ``assign_port_to_namespace port name [port_up]`` - assigns NIC specified by ``port`` - into given namespace ``name``; If optional parameter ``port_up`` is set to ``True``, - then port will be brought up. - * ``add_ip_to_namespace_eth port name addr cidr`` - assigns an IP address ``addr``/``cidr`` - to the NIC specified by ``port`` within namespace ``name`` - * ``reset_port_to_root port name`` - returns given ``port`` from namespace ``name`` back - to the root namespace - - Examples: - - .. code-block:: python - - ['namespace', 'create_namespace', 'testns'] - - ['namespace', 'assign_port_to_namespace', 'eth0', 'testns'] - - * ``veth`` - manipulates with eth and veth devices - - List of supported functions: - - * ``add_veth_port port peer_port`` - adds a pair of veth ports named ``port`` and - ``peer_port`` - * ``del_veth_port port peer_port`` - deletes a veth port pair specified by ``port`` - and ``peer_port`` - * ``bring_up_eth_port eth_port [namespace]`` - brings up ``eth_port`` in (optional) - ``namespace`` - - Examples: - - .. code-block:: python - - ['veth', 'add_veth_port', 'veth', 'veth1'] - - ['veth', 'bring_up_eth_port', 'eth1'] - - * ``tools`` - provides a set of helper functions - - List of supported functions: - - * ``Assert condition`` - evaluates given ``condition`` and raises ``AssertionError`` - in case that condition is not ``True`` - * ``Eval expression`` - evaluates given expression as a python code and returns - its result - * ``Exec command [regex]`` - executes a shell command and filters its output by - (optional) regular expression - - Examples: - - .. code-block:: python - - ['tools', 'exec', 'numactl -H', 'available: ([0-9]+)'] - ['tools', 'assert', '#STEP[-1][0]>1'] - - * ``wait`` - is used for test case interruption. This object doesn't have - any functions. Once reached, vsperf will pause test execution and waits - for press of ``Enter key``. It can be used during testcase design - for debugging purposes. - - Examples: - - .. code-block:: python - - ['wait'] - -Test Macros ------------ - -Test profiles can include macros as part of the test step. Each step in the -profile may return a value such as a port name. Recall macros use #STEP to -indicate the recalled value inside the return structure. If the method the -test step calls returns a value it can be later recalled, for example: - -.. code-block:: python - - { - "Name": "vswitch_add_del_vport", - "Deployment": "clean", - "Description": "vSwitch - add and delete virtual port", - "TestSteps": [ - ['vswitch', 'add_switch', 'int_br0'], # STEP 0 - ['vswitch', 'add_vport', 'int_br0'], # STEP 1 - ['vswitch', 'del_port', 'int_br0', '#STEP[1][0]'], # STEP 2 - ['vswitch', 'del_switch', 'int_br0'], # STEP 3 - ] - } - -This test profile uses the vswitch add_vport method which returns a string -value of the port added. This is later called by the del_port method using the -name from step 1. - -It is also possible to use negative indexes in step macros. In that case -``#STEP[-1]`` will refer to the result from previous step, ``#STEP[-2]`` -will refer to result of step called before previous step, etc. It means, -that you could change ``STEP 2`` from previous example to achieve the same -functionality: - -.. code-block:: python - - ['vswitch', 'del_port', 'int_br0', '#STEP[-1][0]'], # STEP 2 - -Also commonly used steps can be created as a separate profile. - -.. code-block:: python - - STEP_VSWITCH_PVP_INIT = [ - ['vswitch', 'add_switch', 'int_br0'], # STEP 0 - ['vswitch', 'add_phy_port', 'int_br0'], # STEP 1 - ['vswitch', 'add_phy_port', 'int_br0'], # STEP 2 - ['vswitch', 'add_vport', 'int_br0'], # STEP 3 - ['vswitch', 'add_vport', 'int_br0'], # STEP 4 - ] - -This profile can then be used inside other testcases - -.. code-block:: python - - { - "Name": "vswitch_pvp", - "Deployment": "clean", - "Description": "vSwitch - configure switch and one vnf", - "TestSteps": STEP_VSWITCH_PVP_INIT + - [ - ['vnf', 'start'], - ['vnf', 'stop'], - ] + - STEP_VSWITCH_PVP_FINIT - } - -HelloWorld and other basic Testcases ------------------------------------- - -The following examples are for demonstration purposes. -You can run them by copying and pasting into the -conf/integration/01_testcases.conf file. -A command-line instruction is shown at the end of each -example. - -HelloWorld -^^^^^^^^^^ - -The first example is a HelloWorld testcase. -It simply creates a bridge with 2 physical ports, then sets up a flow to drop -incoming packets from the port that was instantiated at the STEP #1. -There's no interaction with the traffic generator. -Then the flow, the 2 ports and the bridge are deleted. -'add_phy_port' method creates a 'dpdk' type interface that will manage the -physical port. The string value returned is the port name that will be referred -by 'del_port' later on. - -.. code-block:: python - - { - "Name": "HelloWorld", - "Description": "My first testcase", - "Deployment": "clean", - "TestSteps": [ - ['vswitch', 'add_switch', 'int_br0'], # STEP 0 - ['vswitch', 'add_phy_port', 'int_br0'], # STEP 1 - ['vswitch', 'add_phy_port', 'int_br0'], # STEP 2 - ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[1][1]', \ - 'actions': ['drop'], 'idle_timeout': '0'}], - ['vswitch', 'del_flow', 'int_br0'], - ['vswitch', 'del_port', 'int_br0', '#STEP[1][0]'], - ['vswitch', 'del_port', 'int_br0', '#STEP[2][0]'], - ['vswitch', 'del_switch', 'int_br0'], - ] - - } - -To run HelloWorld test: - - .. code-block:: console - - ./vsperf --conf-file user_settings.py --integration HelloWorld - -Specify a Flow by the IP address -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The next example shows how to explicitly set up a flow by specifying a -destination IP address. -All packets received from the port created at STEP #1 that have a destination -IP address = 90.90.90.90 will be forwarded to the port created at the STEP #2. - -.. code-block:: python - - { - "Name": "p2p_rule_l3da", - "Description": "Phy2Phy with rule on L3 Dest Addr", - "Deployment": "clean", - "biDirectional": "False", - "TestSteps": [ - ['vswitch', 'add_switch', 'int_br0'], # STEP 0 - ['vswitch', 'add_phy_port', 'int_br0'], # STEP 1 - ['vswitch', 'add_phy_port', 'int_br0'], # STEP 2 - ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[1][1]', \ - 'dl_type': '0x0800', 'nw_dst': '90.90.90.90', \ - 'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}], - ['trafficgen', 'send_traffic', {'traffic_type' : 'continuous'}], - ['vswitch', 'dump_flows', 'int_br0'], # STEP 5 - ['vswitch', 'del_flow', 'int_br0'], # STEP 7 == del-flows - ['vswitch', 'del_port', 'int_br0', '#STEP[1][0]'], - ['vswitch', 'del_port', 'int_br0', '#STEP[2][0]'], - ['vswitch', 'del_switch', 'int_br0'], - ] - }, - -To run the test: - - .. code-block:: console - - ./vsperf --conf-file user_settings.py --integration p2p_rule_l3da - -Multistream feature -^^^^^^^^^^^^^^^^^^^ - -The next testcase uses the multistream feature. -The traffic generator will send packets with different UDP ports. -That is accomplished by using "Stream Type" and "MultiStream" keywords. -4 different flows are set to forward all incoming packets. - -.. code-block:: python - - { - "Name": "multistream_l4", - "Description": "Multistream on UDP ports", - "Deployment": "clean", - "Stream Type": "L4", - "MultiStream": 4, - "TestSteps": [ - ['vswitch', 'add_switch', 'int_br0'], # STEP 0 - ['vswitch', 'add_phy_port', 'int_br0'], # STEP 1 - ['vswitch', 'add_phy_port', 'int_br0'], # STEP 2 - # Setup Flows - ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[1][1]', \ - 'dl_type': '0x0800', 'nw_proto': '17', 'udp_dst': '0', \ - 'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}], - ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[1][1]', \ - 'dl_type': '0x0800', 'nw_proto': '17', 'udp_dst': '1', \ - 'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}], - ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[1][1]', \ - 'dl_type': '0x0800', 'nw_proto': '17', 'udp_dst': '2', \ - 'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}], - ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[1][1]', \ - 'dl_type': '0x0800', 'nw_proto': '17', 'udp_dst': '3', \ - 'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}], - # Send mono-dir traffic - ['trafficgen', 'send_traffic', {'traffic_type' : 'continuous', \ - 'bidir' : 'False'}], - # Clean up - ['vswitch', 'del_flow', 'int_br0'], - ['vswitch', 'del_port', 'int_br0', '#STEP[1][0]'], - ['vswitch', 'del_port', 'int_br0', '#STEP[2][0]'], - ['vswitch', 'del_switch', 'int_br0'], - ] - }, - -To run the test: - - .. code-block:: console - - ./vsperf --conf-file user_settings.py --integration multistream_l4 - -PVP with a VM Replacement -^^^^^^^^^^^^^^^^^^^^^^^^^ - -This example launches a 1st VM in a PVP topology, then the VM is replaced -by another VM. -When VNF setup parameter in ./conf/04_vnf.conf is "QemuDpdkVhostUser" -'add_vport' method creates a 'dpdkvhostuser' type port to connect a VM. - -.. code-block:: python - - { - "Name": "ex_replace_vm", - "Description": "PVP with VM replacement", - "Deployment": "clean", - "TestSteps": [ - ['vswitch', 'add_switch', 'int_br0'], # STEP 0 - ['vswitch', 'add_phy_port', 'int_br0'], # STEP 1 - ['vswitch', 'add_phy_port', 'int_br0'], # STEP 2 - ['vswitch', 'add_vport', 'int_br0'], # STEP 3 vm1 - ['vswitch', 'add_vport', 'int_br0'], # STEP 4 - - # Setup Flows - ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[1][1]', \ - 'actions': ['output:#STEP[3][1]'], 'idle_timeout': '0'}], - ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[4][1]', \ - 'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}], - ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[2][1]', \ - 'actions': ['output:#STEP[4][1]'], 'idle_timeout': '0'}], - ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[3][1]', \ - 'actions': ['output:#STEP[1][1]'], 'idle_timeout': '0'}], - - # Start VM 1 - ['vnf1', 'start'], - # Now we want to replace VM 1 with another VM - ['vnf1', 'stop'], - - ['vswitch', 'add_vport', 'int_br0'], # STEP 11 vm2 - ['vswitch', 'add_vport', 'int_br0'], # STEP 12 - ['vswitch', 'del_flow', 'int_br0'], - ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[1][1]', \ - 'actions': ['output:#STEP[11][1]'], 'idle_timeout': '0'}], - ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[12][1]', \ - 'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}], - - # Start VM 2 - ['vnf2', 'start'], - ['vnf2', 'stop'], - ['vswitch', 'dump_flows', 'int_br0'], - - # Clean up - ['vswitch', 'del_flow', 'int_br0'], - ['vswitch', 'del_port', 'int_br0', '#STEP[1][0]'], - ['vswitch', 'del_port', 'int_br0', '#STEP[2][0]'], - ['vswitch', 'del_port', 'int_br0', '#STEP[3][0]'], # vm1 - ['vswitch', 'del_port', 'int_br0', '#STEP[4][0]'], - ['vswitch', 'del_port', 'int_br0', '#STEP[11][0]'], # vm2 - ['vswitch', 'del_port', 'int_br0', '#STEP[12][0]'], - ['vswitch', 'del_switch', 'int_br0'], - ] - }, - -To run the test: - - .. code-block:: console - - ./vsperf --conf-file user_settings.py --integration ex_replace_vm - -VM with a Linux bridge -^^^^^^^^^^^^^^^^^^^^^^ - -In this example a command-line parameter allows to set up a Linux bridge into -the guest VM. -That's one of the available ways to specify the guest application. -Packets matching the flow will be forwarded to the VM. - -.. code-block:: python - - { - "Name": "ex_pvp_rule_l3da", - "Description": "PVP with flow on L3 Dest Addr", - "Deployment": "clean", - "TestSteps": [ - ['vswitch', 'add_switch', 'int_br0'], # STEP 0 - ['vswitch', 'add_phy_port', 'int_br0'], # STEP 1 - ['vswitch', 'add_phy_port', 'int_br0'], # STEP 2 - ['vswitch', 'add_vport', 'int_br0'], # STEP 3 vm1 - ['vswitch', 'add_vport', 'int_br0'], # STEP 4 - # Setup Flows - ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[1][1]', \ - 'dl_type': '0x0800', 'nw_dst': '90.90.90.90', \ - 'actions': ['output:#STEP[3][1]'], 'idle_timeout': '0'}], - # Each pkt from the VM is forwarded to the 2nd dpdk port - ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[4][1]', \ - 'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}], - # Start VMs - ['vnf1', 'start'], - ['trafficgen', 'send_traffic', {'traffic_type' : 'continuous', \ - 'bidir' : 'False'}], - ['vnf1', 'stop'], - # Clean up - ['vswitch', 'dump_flows', 'int_br0'], # STEP 10 - ['vswitch', 'del_flow', 'int_br0'], # STEP 11 - ['vswitch', 'del_port', 'int_br0', '#STEP[1][0]'], - ['vswitch', 'del_port', 'int_br0', '#STEP[2][0]'], - ['vswitch', 'del_port', 'int_br0', '#STEP[3][0]'], # vm1 ports - ['vswitch', 'del_port', 'int_br0', '#STEP[4][0]'], - ['vswitch', 'del_switch', 'int_br0'], - ] - }, - -To run the test: - - .. code-block:: console - - ./vsperf --conf-file user_settings.py --test-params - "guest_loopback=linux_bridge" --integration ex_pvp_rule_l3da - -Forward packets based on UDP port -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -This examples launches 2 VMs connected in parallel. -Incoming packets will be forwarded to one specific VM depending on the -destination UDP port. - -.. code-block:: python - - { - "Name": "ex_2pvp_rule_l4dp", - "Description": "2 PVP with flows on L4 Dest Port", - "Deployment": "clean", - "Stream Type": "L4", # loop UDP ports - "MultiStream": 2, - "TestSteps": [ - ['vswitch', 'add_switch', 'int_br0'], # STEP 0 - ['vswitch', 'add_phy_port', 'int_br0'], # STEP 1 - ['vswitch', 'add_phy_port', 'int_br0'], # STEP 2 - ['vswitch', 'add_vport', 'int_br0'], # STEP 3 vm1 - ['vswitch', 'add_vport', 'int_br0'], # STEP 4 - ['vswitch', 'add_vport', 'int_br0'], # STEP 5 vm2 - ['vswitch', 'add_vport', 'int_br0'], # STEP 6 - # Setup Flows to reply ICMPv6 and similar packets, so to - # avoid flooding internal port with their re-transmissions - ['vswitch', 'add_flow', 'int_br0', \ - {'priority': '1', 'dl_src': '00:00:00:00:00:01', \ - 'actions': ['output:#STEP[3][1]'], 'idle_timeout': '0'}], - ['vswitch', 'add_flow', 'int_br0', \ - {'priority': '1', 'dl_src': '00:00:00:00:00:02', \ - 'actions': ['output:#STEP[4][1]'], 'idle_timeout': '0'}], - ['vswitch', 'add_flow', 'int_br0', \ - {'priority': '1', 'dl_src': '00:00:00:00:00:03', \ - 'actions': ['output:#STEP[5][1]'], 'idle_timeout': '0'}], - ['vswitch', 'add_flow', 'int_br0', \ - {'priority': '1', 'dl_src': '00:00:00:00:00:04', \ - 'actions': ['output:#STEP[6][1]'], 'idle_timeout': '0'}], - # Forward UDP packets depending on dest port - ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[1][1]', \ - 'dl_type': '0x0800', 'nw_proto': '17', 'udp_dst': '0', \ - 'actions': ['output:#STEP[3][1]'], 'idle_timeout': '0'}], - ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[1][1]', \ - 'dl_type': '0x0800', 'nw_proto': '17', 'udp_dst': '1', \ - 'actions': ['output:#STEP[5][1]'], 'idle_timeout': '0'}], - # Send VM output to phy port #2 - ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[4][1]', \ - 'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}], - ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[6][1]', \ - 'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}], - # Start VMs - ['vnf1', 'start'], # STEP 16 - ['vnf2', 'start'], # STEP 17 - ['trafficgen', 'send_traffic', {'traffic_type' : 'continuous', \ - 'bidir' : 'False'}], - ['vnf1', 'stop'], - ['vnf2', 'stop'], - ['vswitch', 'dump_flows', 'int_br0'], - # Clean up - ['vswitch', 'del_flow', 'int_br0'], - ['vswitch', 'del_port', 'int_br0', '#STEP[1][0]'], - ['vswitch', 'del_port', 'int_br0', '#STEP[2][0]'], - ['vswitch', 'del_port', 'int_br0', '#STEP[3][0]'], # vm1 ports - ['vswitch', 'del_port', 'int_br0', '#STEP[4][0]'], - ['vswitch', 'del_port', 'int_br0', '#STEP[5][0]'], # vm2 ports - ['vswitch', 'del_port', 'int_br0', '#STEP[6][0]'], - ['vswitch', 'del_switch', 'int_br0'], - ] - }, - -To run the test: - - .. code-block:: console - - ./vsperf --conf-file user_settings.py --integration ex_2pvp_rule_l4dp - Executing Tunnel encapsulation tests ------------------------------------ diff --git a/docs/userguide/teststeps.rst b/docs/userguide/teststeps.rst new file mode 100644 index 00000000..51e62020 --- /dev/null +++ b/docs/userguide/teststeps.rst @@ -0,0 +1,651 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) OPNFV, Intel Corporation, AT&T and others. + +Step driven tests +================= + +In general, test scenarios are defined by a ``deployment`` used in the particular +test case definition. The chosen deployment scenario will take care of the vSwitch +configuration, deployment of VNFs and it can also affect configuration of a traffic +generator. In order to allow a more flexible way of testcase scripting, VSPERF supports +a detailed step driven testcase definition. It can be used to configure and +program vSwitch, deploy and terminate VNFs, execute a traffic generator, +modify a VSPERF configuration, execute external commands, etc. + +Execution of step driven tests is done on a step by step work flow starting +with step 0 as defined inside the test case. Each step of the test increments +the step number by one which is indicated in the log. + +.. code-block:: console + + (testcases.integration) - Step 0 'vswitch add_vport ['br0']' start + +Step driven tests can be used for both performance and integration testing. +In case of integration test, each step in the test case is validated. If a step +does not pass validation the test will fail and terminate. The test will continue +until a failure is detected or all steps pass. A csv report file is generated after +a test completes with an OK or FAIL result. + +In case of performance test, the validation of steps is not performed and +standard output files with results from traffic generator and underlying OS +details are generated by vsperf. + +Step driven testcases can be used in two different ways: + + # description of full testcase - in this case ``clean`` deployment is used + to indicate that vsperf should neither configure vSwitch nor deploy any VNF. + Test shall perform all required vSwitch configuration and programming and + deploy required number of VNFs. + + # modification of existing deployment - in this case, any of supported + deployments can be used to perform initial vSwitch configuration and + deployment of VNFs. Additional actions defined by TestSteps can be used + to alter vSwitch configuration or deploy additional VNFs. After the last + step is processed, the test execution will continue with traffic execution. + +Test objects and their functions +-------------------------------- + +Every test step can call a function of one of the supported test objects. The list +of supported objects and their most common functions follows: + + * ``vswitch`` - provides functions for vSwitch configuration + + List of supported functions: + + * ``add_switch br_name`` - creates a new switch (bridge) with given ``br_name`` + * ``del_switch br_name`` - deletes switch (bridge) with given ``br_name`` + * ``add_phy_port br_name`` - adds a physical port into bridge specified by ``br_name`` + * ``add_vport br_name`` - adds a virtual port into bridge specified by ``br_name`` + * ``del_port br_name port_name`` - removes physical or virtual port specified by + ``port_name`` from bridge ``br_name`` + * ``add_flow br_name flow`` - adds flow specified by ``flow`` dictionary into + the bridge ``br_name``; Content of flow dictionary will be passed to the vSwitch. + In case of Open vSwitch it will be passed to the ``ovs-ofctl add-flow`` command. + Please see Open vSwitch documentation for the list of supported flow parameters. + * ``del_flow br_name [flow]`` - deletes flow specified by ``flow`` dictionary from + bridge ``br_name``; In case that optional parameter ``flow`` is not specified + or set to an empty dictionary ``{}``, then all flows from bridge ``br_name`` + will be deleted. + * ``dump_flows br_name`` - dumps all flows from bridge specified by ``br_name`` + * ``enable_stp br_name`` - enables Spanning Tree Protocol for bridge ``br_name`` + * ``disable_stp br_name`` - disables Spanning Tree Protocol for bridge ``br_name`` + * ``enable_rstp br_name`` - enables Rapid Spanning Tree Protocol for bridge ``br_name`` + * ``disable_rstp br_name`` - disables Rapid Spanning Tree Protocol for bridge ``br_name`` + + Examples: + + .. code-block:: python + + ['vswitch', 'add_switch', 'int_br0'] + + ['vswitch', 'del_switch', 'int_br0'] + + ['vswitch', 'add_phy_port', 'int_br0'] + + ['vswitch', 'del_port', 'int_br0', '#STEP[2][0]'] + + ['vswitch', 'add_flow', 'int_br0', {'in_port': '1', 'actions': ['output:2'], + 'idle_timeout': '0'}], + + ['vswitch', 'enable_rstp', 'int_br0'] + + * ``vnf[ID]`` - provides functions for deployment and termination of VNFs; Optional + alfanumerical ``ID`` is used for VNF identification in case that testcase + deploys multiple VNFs. + + List of supported functions: + + * ``start`` - starts a VNF based on VSPERF configuration + * ``stop`` - gracefully terminates given VNF + + Examples: + + .. code-block:: python + + ['vnf1', 'start'] + ['vnf2', 'start'] + ['vnf2', 'stop'] + ['vnf1', 'stop'] + + * ``trafficgen`` - triggers traffic generation + + List of supported functions: + + * ``send_traffic traffic`` - starts a traffic based on the vsperf configuration + and given ``traffic`` dictionary. More details about ``traffic`` dictionary + and its possible values are available at `Traffic Generator Integration Guide + <http://artifacts.opnfv.org/vswitchperf/docs/design/trafficgen_integration_guide.html#step-5-supported-traffic-types>`__ + + Examples: + + .. code-block:: python + + ['trafficgen', 'send_traffic', {'traffic_type' : 'throughput'}] + + ['trafficgen', 'send_traffic', {'traffic_type' : 'back2back', 'bidir' : 'True'}] + + * ``settings`` - reads or modifies VSPERF configuration + + List of supported functions: + + * ``getValue param`` - returns value of given ``param`` + * ``setValue param value`` - sets value of ``param`` to given ``value`` + + Examples: + + .. code-block:: python + + ['settings', 'getValue', 'TOOLS'] + + ['settings', 'setValue', 'GUEST_USERNAME', ['root']] + + * ``namespace`` - creates or modifies network namespaces + + List of supported functions: + + * ``create_namespace name`` - creates new namespace with given ``name`` + * ``delete_namespace name`` - deletes namespace specified by its ``name`` + * ``assign_port_to_namespace port name [port_up]`` - assigns NIC specified by ``port`` + into given namespace ``name``; If optional parameter ``port_up`` is set to ``True``, + then port will be brought up. + * ``add_ip_to_namespace_eth port name addr cidr`` - assigns an IP address ``addr``/``cidr`` + to the NIC specified by ``port`` within namespace ``name`` + * ``reset_port_to_root port name`` - returns given ``port`` from namespace ``name`` back + to the root namespace + + Examples: + + .. code-block:: python + + ['namespace', 'create_namespace', 'testns'] + + ['namespace', 'assign_port_to_namespace', 'eth0', 'testns'] + + * ``veth`` - manipulates with eth and veth devices + + List of supported functions: + + * ``add_veth_port port peer_port`` - adds a pair of veth ports named ``port`` and + ``peer_port`` + * ``del_veth_port port peer_port`` - deletes a veth port pair specified by ``port`` + and ``peer_port`` + * ``bring_up_eth_port eth_port [namespace]`` - brings up ``eth_port`` in (optional) + ``namespace`` + + Examples: + + .. code-block:: python + + ['veth', 'add_veth_port', 'veth', 'veth1'] + + ['veth', 'bring_up_eth_port', 'eth1'] + + * ``tools`` - provides a set of helper functions + + List of supported functions: + + * ``Assert condition`` - evaluates given ``condition`` and raises ``AssertionError`` + in case that condition is not ``True`` + * ``Eval expression`` - evaluates given expression as a python code and returns + its result + * ``Exec command [regex]`` - executes a shell command and filters its output by + (optional) regular expression + + Examples: + + .. code-block:: python + + ['tools', 'exec', 'numactl -H', 'available: ([0-9]+)'] + ['tools', 'assert', '#STEP[-1][0]>1'] + + * ``wait`` - is used for test case interruption. This object doesn't have + any functions. Once reached, vsperf will pause test execution and waits + for press of ``Enter key``. It can be used during testcase design + for debugging purposes. + + Examples: + + .. code-block:: python + + ['wait'] + +Test Macros +----------- + +Test profiles can include macros as part of the test step. Each step in the +profile may return a value such as a port name. Recall macros use #STEP to +indicate the recalled value inside the return structure. If the method the +test step calls returns a value it can be later recalled, for example: + +.. code-block:: python + + { + "Name": "vswitch_add_del_vport", + "Deployment": "clean", + "Description": "vSwitch - add and delete virtual port", + "TestSteps": [ + ['vswitch', 'add_switch', 'int_br0'], # STEP 0 + ['vswitch', 'add_vport', 'int_br0'], # STEP 1 + ['vswitch', 'del_port', 'int_br0', '#STEP[1][0]'], # STEP 2 + ['vswitch', 'del_switch', 'int_br0'], # STEP 3 + ] + } + +This test profile uses the vswitch add_vport method which returns a string +value of the port added. This is later called by the del_port method using the +name from step 1. + +It is also possible to use negative indexes in step macros. In that case +``#STEP[-1]`` will refer to the result from previous step, ``#STEP[-2]`` +will refer to result of step called before previous step, etc. It means, +that you could change ``STEP 2`` from previous example to achieve the same +functionality: + +.. code-block:: python + + ['vswitch', 'del_port', 'int_br0', '#STEP[-1][0]'], # STEP 2 + +Also commonly used steps can be created as a separate profile. + +.. code-block:: python + + STEP_VSWITCH_PVP_INIT = [ + ['vswitch', 'add_switch', 'int_br0'], # STEP 0 + ['vswitch', 'add_phy_port', 'int_br0'], # STEP 1 + ['vswitch', 'add_phy_port', 'int_br0'], # STEP 2 + ['vswitch', 'add_vport', 'int_br0'], # STEP 3 + ['vswitch', 'add_vport', 'int_br0'], # STEP 4 + ] + +This profile can then be used inside other testcases + +.. code-block:: python + + { + "Name": "vswitch_pvp", + "Deployment": "clean", + "Description": "vSwitch - configure switch and one vnf", + "TestSteps": STEP_VSWITCH_PVP_INIT + + [ + ['vnf', 'start'], + ['vnf', 'stop'], + ] + + STEP_VSWITCH_PVP_FINIT + } + +HelloWorld and other basic Testcases +------------------------------------ + +The following examples are for demonstration purposes. +You can run them by copying and pasting into the +conf/integration/01_testcases.conf file. +A command-line instruction is shown at the end of each +example. + +HelloWorld +^^^^^^^^^^ + +The first example is a HelloWorld testcase. +It simply creates a bridge with 2 physical ports, then sets up a flow to drop +incoming packets from the port that was instantiated at the STEP #1. +There's no interaction with the traffic generator. +Then the flow, the 2 ports and the bridge are deleted. +'add_phy_port' method creates a 'dpdk' type interface that will manage the +physical port. The string value returned is the port name that will be referred +by 'del_port' later on. + +.. code-block:: python + + { + "Name": "HelloWorld", + "Description": "My first testcase", + "Deployment": "clean", + "TestSteps": [ + ['vswitch', 'add_switch', 'int_br0'], # STEP 0 + ['vswitch', 'add_phy_port', 'int_br0'], # STEP 1 + ['vswitch', 'add_phy_port', 'int_br0'], # STEP 2 + ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[1][1]', \ + 'actions': ['drop'], 'idle_timeout': '0'}], + ['vswitch', 'del_flow', 'int_br0'], + ['vswitch', 'del_port', 'int_br0', '#STEP[1][0]'], + ['vswitch', 'del_port', 'int_br0', '#STEP[2][0]'], + ['vswitch', 'del_switch', 'int_br0'], + ] + + }, + +To run HelloWorld test: + + .. code-block:: console + + ./vsperf --conf-file user_settings.py --integration HelloWorld + +Specify a Flow by the IP address +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The next example shows how to explicitly set up a flow by specifying a +destination IP address. +All packets received from the port created at STEP #1 that have a destination +IP address = 90.90.90.90 will be forwarded to the port created at the STEP #2. + +.. code-block:: python + + { + "Name": "p2p_rule_l3da", + "Description": "Phy2Phy with rule on L3 Dest Addr", + "Deployment": "clean", + "biDirectional": "False", + "TestSteps": [ + ['vswitch', 'add_switch', 'int_br0'], # STEP 0 + ['vswitch', 'add_phy_port', 'int_br0'], # STEP 1 + ['vswitch', 'add_phy_port', 'int_br0'], # STEP 2 + ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[1][1]', \ + 'dl_type': '0x0800', 'nw_dst': '90.90.90.90', \ + 'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}], + ['trafficgen', 'send_traffic', {'traffic_type' : 'continuous'}], + ['vswitch', 'dump_flows', 'int_br0'], # STEP 5 + ['vswitch', 'del_flow', 'int_br0'], # STEP 7 == del-flows + ['vswitch', 'del_port', 'int_br0', '#STEP[1][0]'], + ['vswitch', 'del_port', 'int_br0', '#STEP[2][0]'], + ['vswitch', 'del_switch', 'int_br0'], + ] + }, + +To run the test: + + .. code-block:: console + + ./vsperf --conf-file user_settings.py --integration p2p_rule_l3da + +Multistream feature +^^^^^^^^^^^^^^^^^^^ + +The next testcase uses the multistream feature. +The traffic generator will send packets with different UDP ports. +That is accomplished by using "Stream Type" and "MultiStream" keywords. +4 different flows are set to forward all incoming packets. + +.. code-block:: python + + { + "Name": "multistream_l4", + "Description": "Multistream on UDP ports", + "Deployment": "clean", + "Stream Type": "L4", + "MultiStream": 4, + "TestSteps": [ + ['vswitch', 'add_switch', 'int_br0'], # STEP 0 + ['vswitch', 'add_phy_port', 'int_br0'], # STEP 1 + ['vswitch', 'add_phy_port', 'int_br0'], # STEP 2 + # Setup Flows + ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[1][1]', \ + 'dl_type': '0x0800', 'nw_proto': '17', 'udp_dst': '0', \ + 'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}], + ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[1][1]', \ + 'dl_type': '0x0800', 'nw_proto': '17', 'udp_dst': '1', \ + 'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}], + ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[1][1]', \ + 'dl_type': '0x0800', 'nw_proto': '17', 'udp_dst': '2', \ + 'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}], + ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[1][1]', \ + 'dl_type': '0x0800', 'nw_proto': '17', 'udp_dst': '3', \ + 'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}], + # Send mono-dir traffic + ['trafficgen', 'send_traffic', {'traffic_type' : 'continuous', \ + 'bidir' : 'False'}], + # Clean up + ['vswitch', 'del_flow', 'int_br0'], + ['vswitch', 'del_port', 'int_br0', '#STEP[1][0]'], + ['vswitch', 'del_port', 'int_br0', '#STEP[2][0]'], + ['vswitch', 'del_switch', 'int_br0'], + ] + }, + +To run the test: + + .. code-block:: console + + ./vsperf --conf-file user_settings.py --integration multistream_l4 + +PVP with a VM Replacement +^^^^^^^^^^^^^^^^^^^^^^^^^ + +This example launches a 1st VM in a PVP topology, then the VM is replaced +by another VM. +When VNF setup parameter in ./conf/04_vnf.conf is "QemuDpdkVhostUser" +'add_vport' method creates a 'dpdkvhostuser' type port to connect a VM. + +.. code-block:: python + + { + "Name": "ex_replace_vm", + "Description": "PVP with VM replacement", + "Deployment": "clean", + "TestSteps": [ + ['vswitch', 'add_switch', 'int_br0'], # STEP 0 + ['vswitch', 'add_phy_port', 'int_br0'], # STEP 1 + ['vswitch', 'add_phy_port', 'int_br0'], # STEP 2 + ['vswitch', 'add_vport', 'int_br0'], # STEP 3 vm1 + ['vswitch', 'add_vport', 'int_br0'], # STEP 4 + + # Setup Flows + ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[1][1]', \ + 'actions': ['output:#STEP[3][1]'], 'idle_timeout': '0'}], + ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[4][1]', \ + 'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}], + ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[2][1]', \ + 'actions': ['output:#STEP[4][1]'], 'idle_timeout': '0'}], + ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[3][1]', \ + 'actions': ['output:#STEP[1][1]'], 'idle_timeout': '0'}], + + # Start VM 1 + ['vnf1', 'start'], + # Now we want to replace VM 1 with another VM + ['vnf1', 'stop'], + + ['vswitch', 'add_vport', 'int_br0'], # STEP 11 vm2 + ['vswitch', 'add_vport', 'int_br0'], # STEP 12 + ['vswitch', 'del_flow', 'int_br0'], + ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[1][1]', \ + 'actions': ['output:#STEP[11][1]'], 'idle_timeout': '0'}], + ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[12][1]', \ + 'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}], + + # Start VM 2 + ['vnf2', 'start'], + ['vnf2', 'stop'], + ['vswitch', 'dump_flows', 'int_br0'], + + # Clean up + ['vswitch', 'del_flow', 'int_br0'], + ['vswitch', 'del_port', 'int_br0', '#STEP[1][0]'], + ['vswitch', 'del_port', 'int_br0', '#STEP[2][0]'], + ['vswitch', 'del_port', 'int_br0', '#STEP[3][0]'], # vm1 + ['vswitch', 'del_port', 'int_br0', '#STEP[4][0]'], + ['vswitch', 'del_port', 'int_br0', '#STEP[11][0]'], # vm2 + ['vswitch', 'del_port', 'int_br0', '#STEP[12][0]'], + ['vswitch', 'del_switch', 'int_br0'], + ] + }, + +To run the test: + + .. code-block:: console + + ./vsperf --conf-file user_settings.py --integration ex_replace_vm + +VM with a Linux bridge +^^^^^^^^^^^^^^^^^^^^^^ + +This example setups a PVP topology and routes traffic to the VM based on +the destination IP address. A command-line parameter is used to select a Linux +bridge as a guest loopback application. It is also possible to select a guest +loopback application by a configuration option ``GUEST_LOOPBACK``. + +.. code-block:: python + + { + "Name": "ex_pvp_rule_l3da", + "Description": "PVP with flow on L3 Dest Addr", + "Deployment": "clean", + "TestSteps": [ + ['vswitch', 'add_switch', 'int_br0'], # STEP 0 + ['vswitch', 'add_phy_port', 'int_br0'], # STEP 1 + ['vswitch', 'add_phy_port', 'int_br0'], # STEP 2 + ['vswitch', 'add_vport', 'int_br0'], # STEP 3 vm1 + ['vswitch', 'add_vport', 'int_br0'], # STEP 4 + # Setup Flows + ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[1][1]', \ + 'dl_type': '0x0800', 'nw_dst': '90.90.90.90', \ + 'actions': ['output:#STEP[3][1]'], 'idle_timeout': '0'}], + # Each pkt from the VM is forwarded to the 2nd dpdk port + ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[4][1]', \ + 'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}], + # Start VMs + ['vnf1', 'start'], + ['trafficgen', 'send_traffic', {'traffic_type' : 'continuous', \ + 'bidir' : 'False'}], + ['vnf1', 'stop'], + # Clean up + ['vswitch', 'dump_flows', 'int_br0'], # STEP 10 + ['vswitch', 'del_flow', 'int_br0'], # STEP 11 + ['vswitch', 'del_port', 'int_br0', '#STEP[1][0]'], + ['vswitch', 'del_port', 'int_br0', '#STEP[2][0]'], + ['vswitch', 'del_port', 'int_br0', '#STEP[3][0]'], # vm1 ports + ['vswitch', 'del_port', 'int_br0', '#STEP[4][0]'], + ['vswitch', 'del_switch', 'int_br0'], + ] + }, + +To run the test: + + .. code-block:: console + + ./vsperf --conf-file user_settings.py --test-params + "guest_loopback=linux_bridge" --integration ex_pvp_rule_l3da + +Forward packets based on UDP port +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This examples launches 2 VMs connected in parallel. +Incoming packets will be forwarded to one specific VM depending on the +destination UDP port. + +.. code-block:: python + + { + "Name": "ex_2pvp_rule_l4dp", + "Description": "2 PVP with flows on L4 Dest Port", + "Deployment": "clean", + "Stream Type": "L4", # loop UDP ports + "MultiStream": 2, + "TestSteps": [ + ['vswitch', 'add_switch', 'int_br0'], # STEP 0 + ['vswitch', 'add_phy_port', 'int_br0'], # STEP 1 + ['vswitch', 'add_phy_port', 'int_br0'], # STEP 2 + ['vswitch', 'add_vport', 'int_br0'], # STEP 3 vm1 + ['vswitch', 'add_vport', 'int_br0'], # STEP 4 + ['vswitch', 'add_vport', 'int_br0'], # STEP 5 vm2 + ['vswitch', 'add_vport', 'int_br0'], # STEP 6 + # Setup Flows to reply ICMPv6 and similar packets, so to + # avoid flooding internal port with their re-transmissions + ['vswitch', 'add_flow', 'int_br0', \ + {'priority': '1', 'dl_src': '00:00:00:00:00:01', \ + 'actions': ['output:#STEP[3][1]'], 'idle_timeout': '0'}], + ['vswitch', 'add_flow', 'int_br0', \ + {'priority': '1', 'dl_src': '00:00:00:00:00:02', \ + 'actions': ['output:#STEP[4][1]'], 'idle_timeout': '0'}], + ['vswitch', 'add_flow', 'int_br0', \ + {'priority': '1', 'dl_src': '00:00:00:00:00:03', \ + 'actions': ['output:#STEP[5][1]'], 'idle_timeout': '0'}], + ['vswitch', 'add_flow', 'int_br0', \ + {'priority': '1', 'dl_src': '00:00:00:00:00:04', \ + 'actions': ['output:#STEP[6][1]'], 'idle_timeout': '0'}], + # Forward UDP packets depending on dest port + ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[1][1]', \ + 'dl_type': '0x0800', 'nw_proto': '17', 'udp_dst': '0', \ + 'actions': ['output:#STEP[3][1]'], 'idle_timeout': '0'}], + ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[1][1]', \ + 'dl_type': '0x0800', 'nw_proto': '17', 'udp_dst': '1', \ + 'actions': ['output:#STEP[5][1]'], 'idle_timeout': '0'}], + # Send VM output to phy port #2 + ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[4][1]', \ + 'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}], + ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[6][1]', \ + 'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}], + # Start VMs + ['vnf1', 'start'], # STEP 16 + ['vnf2', 'start'], # STEP 17 + ['trafficgen', 'send_traffic', {'traffic_type' : 'continuous', \ + 'bidir' : 'False'}], + ['vnf1', 'stop'], + ['vnf2', 'stop'], + ['vswitch', 'dump_flows', 'int_br0'], + # Clean up + ['vswitch', 'del_flow', 'int_br0'], + ['vswitch', 'del_port', 'int_br0', '#STEP[1][0]'], + ['vswitch', 'del_port', 'int_br0', '#STEP[2][0]'], + ['vswitch', 'del_port', 'int_br0', '#STEP[3][0]'], # vm1 ports + ['vswitch', 'del_port', 'int_br0', '#STEP[4][0]'], + ['vswitch', 'del_port', 'int_br0', '#STEP[5][0]'], # vm2 ports + ['vswitch', 'del_port', 'int_br0', '#STEP[6][0]'], + ['vswitch', 'del_switch', 'int_br0'], + ] + }, + +To run the test: + + .. code-block:: console + + ./vsperf --conf-file user_settings.py --integration ex_2pvp_rule_l4dp + +Modification of existing PVVP deployment +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This is an example of modification of a standard deployment scenario with additional TestSteps. +Standard PVVP scenario is used to configure a vSwitch and to deploy two VNFs connected +in series. Additional TestSteps will deploy a 3rd VNF and connect it in parallel to +already configured VNFs. Traffic generator is instructed (by Multistream feature) to send +two separate traffic streams. One stream will be sent to the standalone VNF and second +to two chained VNFs. + +In case, that test is defined as a performance test, then traffic results will be collected +and available in both csv and rst report files. + +.. code-block:: python + + { + "Name": "pvvp_pvp_cont", + "Traffic Type": "continuous", + "Deployment": "pvvp", + "Description": "PVVP and PVP in parallel with Continuous Stream", + "biDirectional": "True", + "iLoad": "100", + "MultiStream": "2", + "TestSteps": [ + ['vswitch', 'add_vport', 'br0'], + ['vswitch', 'add_vport', 'br0'], + # priority must be higher than default 32768, otherwise flows won't match + ['vswitch', 'add_flow', 'br0', + {'in_port': '1', 'actions': ['output:#STEP[-2][1]'], 'idle_timeout': '0', 'dl_type':'0x800', + 'nw_proto':'17', 'tp_dst':'0', 'priority': '33000'}], + ['vswitch', 'add_flow', 'br0', + {'in_port': '2', 'actions': ['output:#STEP[-2][1]'], 'idle_timeout': '0', 'dl_type':'0x800', + 'nw_proto':'17', 'tp_dst':'0', 'priority': '33000'}], + ['vswitch', 'add_flow', 'br0', {'in_port': '#STEP[-4][1]', 'actions': ['output:1'], + 'idle_timeout': '0'}], + ['vswitch', 'add_flow', 'br0', {'in_port': '#STEP[-4][1]', 'actions': ['output:2'], + 'idle_timeout': '0'}], + ['vswitch', 'dump_flows', 'br0'], + ['vnf1', 'start'], + ] + }, + +To run the test: + + .. code-block:: console + + ./vsperf --conf-file user_settings.py pvvp_pvp_cont + diff --git a/testcases/integration.py b/testcases/integration.py index bec38624..4b9dacfd 100644 --- a/testcases/integration.py +++ b/testcases/integration.py @@ -28,9 +28,6 @@ from tools import veth from tools.teststepstools import TestStepsTools from core.loader import Loader -CHECK_PREFIX = 'validate_' - - class IntegrationTestCase(TestCase): """IntegrationTestCase class """ @@ -41,193 +38,20 @@ class IntegrationTestCase(TestCase): self._type = 'integration' super(IntegrationTestCase, self).__init__(cfg) self._logger = logging.getLogger(__name__) - self._inttest = None - - def report_status(self, label, status): - """ Log status of test step - """ - self._logger.info("%s ... %s", label, 'OK' if status else 'FAILED') - - def run_initialize(self): - """ Prepare test execution environment - """ - super(IntegrationTestCase, self).run_initialize() - self._inttest = {'status' : True, 'details' : ''} - - def run(self): - """Run the test - - All setup and teardown through controllers is included. - """ - def eval_step_params(params, step_result): - """ Evaluates referrences to results from previous steps - """ - def eval_param(param, STEP): - # pylint: disable=invalid-name - """ Helper function - """ - if isinstance(param, str): - # evaluate every #STEP reference inside parameter itself - macros = re.findall(r'#STEP\[[\w\[\]\-\'\"]+\]', param) - if macros: - for macro in macros: - # pylint: disable=eval-used - tmp_val = str(eval(macro[1:])) - param = param.replace(macro, tmp_val) - return param - elif isinstance(param, list) or isinstance(param, tuple): - tmp_list = [] - for item in param: - tmp_list.append(eval_param(item, STEP)) - return tmp_list - elif isinstance(param, dict): - tmp_dict = {} - for (key, value) in param.items(): - tmp_dict[key] = eval_param(value, STEP) - return tmp_dict - else: - return param - - eval_params = [] - # evaluate all parameters if needed - for param in params: - eval_params.append(eval_param(param, step_result)) - return eval_params - - # prepare test execution environment - self.run_initialize() - - try: - with self._vswitch_ctl, self._loadgen: - with self._vnf_ctl, self._collector: - if not self._vswitch_none: - self._add_flows() - - # run traffic generator if requested, otherwise wait for manual termination - if S.getValue('mode') == 'trafficgen-off': - time.sleep(2) - self._logger.debug("All is set. Please run traffic generator manually.") - input(os.linesep + "Press Enter to terminate vswitchperf..." + os.linesep + os.linesep) - else: - with self._traffic_ctl: - if not self.test: - self._traffic_ctl.send_traffic(self._traffic) - else: - vnf_list = {} - loader = Loader() - # execute test based on TestSteps definition - if self.test: - # initialize list with results - step_result = [None] * len(self.test) - - # count how many VNFs are involved in the test - for step in self.test: - if step[0].startswith('vnf'): - vnf_list[step[0]] = None - - # check/expand GUEST configuration and copy data to shares - if len(vnf_list): - S.check_vm_settings(len(vnf_list)) - self._copy_fwd_tools_for_all_guests(len(vnf_list)) - - # run test step by step... - for i, step in enumerate(self.test): - step_ok = False - if step[0] == 'vswitch': - test_object = self._vswitch_ctl.get_vswitch() - elif step[0] == 'namespace': - test_object = namespace - elif step[0] == 'veth': - test_object = veth - elif step[0] == 'settings': - test_object = S - elif step[0] == 'tools': - test_object = TestStepsTools() - step[1] = step[1].title() - elif step[0] == 'trafficgen': - test_object = self._traffic_ctl - # in case of send_traffic method, ensure that specified - # traffic values are merged with existing self._traffic - if step[1] == 'send_traffic': - tmp_traffic = copy.deepcopy(self._traffic) - tmp_traffic.update(step[2]) - step[2] = tmp_traffic - elif step[0].startswith('vnf'): - if not vnf_list[step[0]]: - # initialize new VM - vnf_list[step[0]] = loader.get_vnf_class()() - test_object = vnf_list[step[0]] - elif step[0] == 'wait': - input(os.linesep + "Step {}: Press Enter to continue with " - "the next step...".format(i) + os.linesep + os.linesep) - continue - else: - self._logger.error("Unsupported test object %s", step[0]) - self._inttest = {'status' : False, 'details' : ' '.join(step)} - self.report_status("Step '{}'".format(' '.join(step)), - self._inttest['status']) - break - - test_method = getattr(test_object, step[1]) - test_method_check = getattr(test_object, CHECK_PREFIX + step[1]) - - step_params = [] - if test_method and test_method_check and \ - callable(test_method) and callable(test_method_check): - - try: - # eval parameters, but use only valid step_results - # to support negative indexes - step_params = eval_step_params(step[2:], step_result[:i]) - step_log = '{} {}'.format(' '.join(step[:2]), step_params) - step_result[i] = test_method(*step_params) - self._logger.debug("Step %s '%s' results '%s'", i, - step_log, step_result[i]) - time.sleep(5) - step_ok = test_method_check(step_result[i], *step_params) - except AssertionError: - self._inttest = {'status' : False, 'details' : step_log} - self._logger.error("Step %s raised assertion error", i) - # stop vnfs in case of error - for vnf in vnf_list: - vnf_list[vnf].stop() - break - except IndexError: - self._inttest = {'status' : False, 'details' : step_log} - self._logger.error("Step %s result index error %s", i, - ' '.join(step[2:])) - # stop vnfs in case of error - for vnf in vnf_list: - vnf_list[vnf].stop() - break - - self.report_status("Step {} - '{}'".format(i, step_log), step_ok) - if not step_ok: - self._inttest = {'status' : False, 'details' : step_log} - # stop vnfs in case of error - for vnf in vnf_list: - vnf_list[vnf].stop() - break - - # dump vswitch flows before they are affected by VNF termination - if not self._vswitch_none: - self._vswitch_ctl.dump_vswitch_flows() - finally: - # tear down test execution environment and log results - self.run_finalize() - - # report test results - self.run_report() + # enforce check of step result for step driven testcases + self._step_check = True def run_report(self): """ Report test results """ if self.test: results = OrderedDict() - results['status'] = 'OK' if self._inttest['status'] else 'FAILED' - results['details'] = self._inttest['details'] + results['status'] = 'OK' if self._step_status['status'] else 'FAILED' + results['details'] = self._step_status['details'] TestCase.write_result_to_file([results], self._output_file) - self.report_status("Test '{}'".format(self.name), self._inttest['status']) + self.step_report_status("Test '{}'".format(self.name), self._step_status['status']) # inform vsperf about testcase failure - if not self._inttest['status']: + if not self._step_status['status']: raise Exception + else: + super(IntegrationTestCase, self).run_report() diff --git a/testcases/performance.py b/testcases/performance.py index a4769a28..240d04a9 100644 --- a/testcases/performance.py +++ b/testcases/performance.py @@ -34,5 +34,5 @@ class PerformanceTestCase(TestCase): def run_report(self): super(PerformanceTestCase, self).run_report() - if S.getValue('mode') != 'trafficgen-off': + if self._tc_results: report.generate(self._output_file, self._tc_results, self._collector.get_results(), self._type) diff --git a/testcases/testcase.py b/testcases/testcase.py index 7f22c18f..00164ea2 100644 --- a/testcases/testcase.py +++ b/testcases/testcase.py @@ -34,6 +34,7 @@ from tools import hugepages from tools import functions from tools.pkt_gen.trafficgen.trafficgenhelper import TRAFFIC_DEFAULTS +CHECK_PREFIX = 'validate_' class TestCase(object): """TestCase base class @@ -60,6 +61,11 @@ class TestCase(object): self._settings_original = {} self._settings_paths_modified = False self._testcast_run_time = None + # initialization of step driven specific members + self._step_check = False # by default don't check result for step driven testcases + self._step_vnf_list = {} + self._step_result = [] + self._step_status = None # store all GUEST_ specific settings to keep original values before their expansion for key in S.__dict__: @@ -170,6 +176,12 @@ class TestCase(object): else: self._logger.debug("MAC addresses can not be read") + # count how many VNFs are involved in TestSteps + if self.test: + for step in self.test: + if step[0].startswith('vnf'): + self._step_vnf_list[step[0]] = None + def run_initialize(self): """ Prepare test execution environment """ @@ -186,26 +198,31 @@ class TestCase(object): self._vnf_ctl = component_factory.create_vnf( self.deployment, - loader.get_vnf_class()) + loader.get_vnf_class(), + len(self._step_vnf_list)) # verify enough hugepages are free to run the testcase if not self._check_for_enough_hugepages(): raise RuntimeError('Not enough hugepages free to run test.') # perform guest related handling - if self._vnf_ctl.get_vnfs_number(): + tmp_vm_count = self._vnf_ctl.get_vnfs_number() + len(self._step_vnf_list) + if tmp_vm_count: # copy sources of l2 forwarding tools into VM shared dir if needed - self._copy_fwd_tools_for_all_guests(self._vnf_ctl.get_vnfs_number()) + self._copy_fwd_tools_for_all_guests(tmp_vm_count) # in case of multi VM in parallel, set the number of streams to the number of VMs if self.deployment.startswith('pvpv'): # for each VM NIC pair we need an unique stream streams = 0 - for vm_nic in S.getValue('GUEST_NICS_NR')[:self._vnf_ctl.get_vnfs_number()]: + for vm_nic in S.getValue('GUEST_NICS_NR')[:tmp_vm_count]: streams += int(vm_nic / 2) if vm_nic > 1 else 1 self._logger.debug("VMs with parallel connection were detected. " "Thus Number of streams was set to %s", streams) - self._traffic.update({'multistream': streams}) + # update streams if needed; In case of additional VNFs deployed by TestSteps + # user can define a proper stream count manually + if 'multistream' not in self._traffic or self._traffic['multistream'] < streams: + self._traffic.update({'multistream': streams}) # OVS Vanilla requires guest VM MAC address and IPs to work if 'linux_bridge' in S.getValue('GUEST_LOOPBACK'): @@ -235,11 +252,16 @@ class TestCase(object): self._output_file = os.path.join(self._results_dir, "result_" + self.name + "_" + self.deployment + ".csv") + self._step_status = {'status' : True, 'details' : ''} + self._logger.debug("Setup:") def run_finalize(self): """ Tear down test execution environment and record test results """ + # Stop all VNFs started by TestSteps in case that something went wrong + self.step_stop_vnfs() + # umount hugepages if mounted self._umount_hugepages() @@ -272,11 +294,12 @@ class TestCase(object): self._logger.debug("self._collector Results:") self._collector.print_results() - if S.getValue('mode') != 'trafficgen-off': + results = self._traffic_ctl.get_results() + if results: self._logger.debug("Traffic Results:") self._traffic_ctl.print_results() - self._tc_results = self._append_results(self._traffic_ctl.get_results()) + self._tc_results = self._append_results(results) TestCase.write_result_to_file(self._tc_results, self._output_file) def run(self): @@ -287,36 +310,31 @@ class TestCase(object): # prepare test execution environment self.run_initialize() - with self._vswitch_ctl, self._loadgen: - with self._vnf_ctl, self._collector: - if not self._vswitch_none: - self._add_flows() + try: + with self._vswitch_ctl, self._loadgen: + with self._vnf_ctl, self._collector: + if not self._vswitch_none: + self._add_flows() - # run traffic generator if requested, otherwise wait for manual termination - if S.getValue('mode') == 'trafficgen-off': - time.sleep(2) - self._logger.debug("All is set. Please run traffic generator manually.") - input(os.linesep + "Press Enter to terminate vswitchperf..." + os.linesep + os.linesep) - else: - if S.getValue('mode') == 'trafficgen-pause': - time.sleep(2) - true_vals = ('yes', 'y', 'ye', None) - while True: - choice = input(os.linesep + 'Transmission paused, should' - ' transmission be resumed? ' + os.linesep).lower() - if not choice or choice not in true_vals: - print('Please respond with \'yes\' or \'y\' ', end='') - else: - break with self._traffic_ctl: - self._traffic_ctl.send_traffic(self._traffic) + # execute test based on TestSteps definition if needed... + if self.step_run(): + # ...and continue with traffic generation, but keep + # in mind, that clean deployment does not configure + # OVS nor executes the traffic + if self.deployment != 'clean': + self._traffic_ctl.send_traffic(self._traffic) - # dump vswitch flows before they are affected by VNF termination - if not self._vswitch_none: - self._vswitch_ctl.dump_vswitch_flows() + # dump vswitch flows before they are affected by VNF termination + if not self._vswitch_none: + self._vswitch_ctl.dump_vswitch_flows() - # tear down test execution environment and log results - self.run_finalize() + # garbage collection for case that TestSteps modify existing deployment + self.step_stop_vnfs() + + finally: + # tear down test execution environment and log results + self.run_finalize() self._testcase_run_time = time.strftime("%H:%M:%S", time.gmtime(time.time() - self._testcase_start_time)) @@ -635,3 +653,143 @@ class TestCase(object): vswitch.add_flow(bridge, flow) else: pass + + + # + # TestSteps realted methods + # + def step_report_status(self, label, status): + """ Log status of test step + """ + self._logger.info("%s ... %s", label, 'OK' if status else 'FAILED') + + def step_stop_vnfs(self): + """ Stop all VNFs started by TestSteps + """ + for vnf in self._step_vnf_list: + self._step_vnf_list[vnf].stop() + + @staticmethod + def step_eval_param(param, STEP): + # pylint: disable=invalid-name + """ Helper function for #STEP macro evaluation + """ + if isinstance(param, str): + # evaluate every #STEP reference inside parameter itself + macros = re.findall(r'#STEP\[[\w\[\]\-\'\"]+\]', param) + if macros: + for macro in macros: + # pylint: disable=eval-used + tmp_val = str(eval(macro[1:])) + param = param.replace(macro, tmp_val) + return param + elif isinstance(param, list) or isinstance(param, tuple): + tmp_list = [] + for item in param: + tmp_list.append(TestCase.step_eval_param(item, STEP)) + return tmp_list + elif isinstance(param, dict): + tmp_dict = {} + for (key, value) in param.items(): + tmp_dict[key] = TestCase.step_eval_param(value, STEP) + return tmp_dict + else: + return param + + @staticmethod + def step_eval_params(params, step_result): + """ Evaluates referrences to results from previous steps + """ + eval_params = [] + # evaluate all parameters if needed + for param in params: + eval_params.append(TestCase.step_eval_param(param, step_result)) + return eval_params + + def step_run(self): + """ Execute actions specified by TestSteps list + + :return: False if any error was detected + True otherwise + """ + # anything to do? + if not self.test: + return True + + # required for VNFs initialization + loader = Loader() + # initialize list with results + self._step_result = [None] * len(self.test) + + # run test step by step... + for i, step in enumerate(self.test): + step_ok = not self._step_check + if step[0] == 'vswitch': + test_object = self._vswitch_ctl.get_vswitch() + elif step[0] == 'namespace': + test_object = namespace + elif step[0] == 'veth': + test_object = veth + elif step[0] == 'settings': + test_object = S + elif step[0] == 'tools': + test_object = TestStepsTools() + step[1] = step[1].title() + elif step[0] == 'trafficgen': + test_object = self._traffic_ctl + # in case of send_traffic or send_traffic_async methods, ensure + # that specified traffic values are merged with existing self._traffic + if step[1].startswith('send_traffic'): + tmp_traffic = copy.deepcopy(self._traffic) + tmp_traffic.update(step[2]) + step[2] = tmp_traffic + elif step[0].startswith('vnf'): + if not self._step_vnf_list[step[0]]: + # initialize new VM + self._step_vnf_list[step[0]] = loader.get_vnf_class()() + test_object = self._step_vnf_list[step[0]] + elif step[0] == 'wait': + input(os.linesep + "Step {}: Press Enter to continue with " + "the next step...".format(i) + os.linesep + os.linesep) + continue + else: + self._logger.error("Unsupported test object %s", step[0]) + self._step_status = {'status' : False, 'details' : ' '.join(step)} + self.step_report_status("Step '{}'".format(' '.join(step)), + self._step_status['status']) + return False + + test_method = getattr(test_object, step[1]) + if self._step_check: + test_method_check = getattr(test_object, CHECK_PREFIX + step[1]) + else: + test_method_check = None + + step_params = [] + try: + # eval parameters, but use only valid step_results + # to support negative indexes + step_params = TestCase.step_eval_params(step[2:], self._step_result[:i]) + step_log = '{} {}'.format(' '.join(step[:2]), step_params) + self._logger.debug("Step %s '%s' start", i, step_log) + self._step_result[i] = test_method(*step_params) + self._logger.debug("Step %s '%s' results '%s'", i, + step_log, self._step_result[i]) + time.sleep(5) + if self._step_check: + step_ok = test_method_check(self._step_result[i], *step_params) + except (AssertionError, AttributeError, IndexError) as ex: + step_ok = False + self._logger.error("Step %s raised %s", i, type(ex).__name__) + + if self._step_check: + self.step_report_status("Step {} - '{}'".format(i, step_log), step_ok) + + if not step_ok: + self._step_status = {'status' : False, 'details' : step_log} + # Stop all VNFs started by TestSteps + self.step_stop_vnfs() + return False + + # all steps processed without any issue + return True diff --git a/vnfs/qemu/qemu.py b/vnfs/qemu/qemu.py index 87126da8..51553277 100644 --- a/vnfs/qemu/qemu.py +++ b/vnfs/qemu/qemu.py @@ -151,27 +151,28 @@ class IVnfQemu(IVnf): """ Stops VNF instance gracefully first. """ - try: - # exit testpmd if needed - if self._guest_loopback == 'testpmd': - self.execute_and_wait('stop', 120, "Done") - self.execute_and_wait('quit', 120, "[bB]ye") - - # turn off VM - self.execute_and_wait('poweroff', 120, "Power down") - - except pexpect.TIMEOUT: - self.kill() - - # wait until qemu shutdowns - self._logger.debug('Wait for QEMU to terminate') - for dummy in range(30): - time.sleep(1) - if not self.is_running(): - break - - # just for case that graceful shutdown failed - super(IVnfQemu, self).stop() + if self.is_running(): + try: + # exit testpmd if needed + if self._guest_loopback == 'testpmd': + self.execute_and_wait('stop', 120, "Done") + self.execute_and_wait('quit', 120, "[bB]ye") + + # turn off VM + self.execute_and_wait('poweroff', 120, "Power down") + + except pexpect.TIMEOUT: + self.kill() + + # wait until qemu shutdowns + self._logger.debug('Wait for QEMU to terminate') + for dummy in range(30): + time.sleep(1) + if not self.is_running(): + break + + # just for case that graceful shutdown failed + super(IVnfQemu, self).stop() # helper functions |