tests: Improvement of step driven testcases

A set of improvements was introduced to enhance step driven
testcases capabilities.

Details:
* delay among test steps is configurable by TEST_STEP_DELAY
  parameter
* step driven tool function exec was renamed to exec_shell
* new step driven tool function exec_python was introduced to execute
  a python code
* new step driven object sleep was introduced to pause test execution
  for defined number of seconds.
* fixed bug in settings.validate_getValue() to correctly validate
  access of parameters modified by TEST_PARAMS
* new #PARAM() macro was introduced to allow references among
  configuration parameters
* multistream support has been added into ixnetrfc2544v2.tcl,
  which is used for tunneling protocols test (op2p deployment)
* fixed bug in op2p deployment to list interfaces and flows
  from both bridges involved in the test
* test report updated to state exact rfcxxxx type of traffic
  type, e.g. rfc2544_continuous
* test report of step driven testcases was updated to contain
  measured values from traffic generator in CSV report
* method for ovs flow comparison was modified to normalize
  IPv4 CIDR network addr (e.g. 10.0.0.5/8 => 10.0.0.0/8)

JIRA: VSPERF-512

Change-Id: Ib4f38dcdfbf3820dd766b25520da0ad0c81f3293
Signed-off-by: Martin Klozik <martinx.klozik@intel.com>
Reviewed-by: Al Morton <acmorton@att.com>
Reviewed-by: Christian Trautman <ctrautma@redhat.com>
Reviewed-by: Sridhar Rao <sridhar.rao@spirent.com>
Reviewed-by: Trevor Cooper <trevor.cooper@intel.com>
Reviewed-by: Ciara Loftus <ciara.loftus@intel.com>

18 files changed, 271 insertions, 105 deletions

diff --git a/3rd_party/ixia/ixnetrfc2544v2.tcl b/3rd_party/ixia/ixnetrfc2544v2.tcl
index 5758f0e4..b5c0fe2a 100755
--- a/3rd_party/ixia/ixnetrfc2544v2.tcl
+++ b/3rd_party/ixia/ixnetrfc2544v2.tcl
@@ -83,6 +83,13 @@ proc startRfc2544Test { testSpec trafficSpec } {
 
     set duration                [dict get $testSpec duration]
 
+    set L2CountValue            1
+    set L2Increment             False
+    set L3ValueType             singleValue
+    set L3CountValue            1
+    set L4ValueType             singleValue
+    set L4CountValue            1
+
     # RFC2544 to IXIA terminology mapping (it affects Ixia configuration inside this script):
     # Test    => Trial
     # Trial   => Iteration
@@ -109,16 +116,23 @@ proc startRfc2544Test { testSpec trafficSpec } {
     }
 
     set multipleStreams         [dict get $testSpec multipleStreams]
+    set streamType              [dict get $testSpec streamType]
+
     if {($multipleStreams < 0)} {
-        set multipleStreams    0
+        set multipleStreams     0
     }
-    set numflows               64000
 
     if {$multipleStreams} {
-        set numflows       $multipleStreams
-        set multipleStreams     increment
-    } else {
-        set multipleStreams     singleValue
+        if {($streamType == "L2")} {
+            set L2CountValue    $multipleStreams
+            set L2Increment     True
+        } elseif {($streamType == "L3")} {
+            set L3ValueType     increment
+            set L3CountValue    $multipleStreams
+        } else {
+            set L4ValueType     increment
+            set L4CountValue    $multipleStreams
+        }
     }
 
     set fastConvergence         True
@@ -742,9 +756,9 @@ proc startRfc2544Test { testSpec trafficSpec } {
     set sg_lan [ixNet add $ixNetSG_Stack(1)/protocols/static lan]
     ixNet setMultiAttrs $sg_lan \
      -atmEncapsulation ::ixNet::OBJ-null \
-     -count 1 \
+     -count $L2CountValue \
      -countPerVc 1 \
-     -enableIncrementMac False \
+     -enableIncrementMac $L2Increment \
      -enableIncrementVlan False \
      -enableSiteId False \
      -enableVlan False \
@@ -1122,9 +1136,9 @@ proc startRfc2544Test { testSpec trafficSpec } {
     set sg_lan [ixNet add $ixNetSG_Stack(1)/protocols/static lan]
     ixNet setMultiAttrs $sg_lan \
      -atmEncapsulation ::ixNet::OBJ-null \
-     -count 1 \
+     -count $L2CountValue \
      -countPerVc 1 \
-     -enableIncrementMac False \
+     -enableIncrementMac $L2Increment \
      -enableIncrementVlan False \
      -enableSiteId False \
      -enableVlan False \
@@ -1570,10 +1584,10 @@ proc startRfc2544Test { testSpec trafficSpec } {
      -auto False \
      -randomMask {0.0.0.0} \
      -trackingEnabled False \
-     -valueType $multipleStreams \
+     -valueType $L3ValueType \
      -activeFieldChoice False \
      -startValue $dstIp \
-     -countValue $numflows
+     -countValue $L3CountValue
     #sg_commit
     #set sg_field [lindex [ixNet remapIds $sg_field] 0]
 
@@ -1651,10 +1665,10 @@ proc startRfc2544Test { testSpec trafficSpec } {
          -auto False \
          -randomMask {63} \
          -trackingEnabled False \
-         -valueType $multipleStreams \
+         -valueType $L4ValueType \
          -activeFieldChoice False \
          -startValue {0} \
-         -countValue $numflows
+         -countValue $L4CountValue
 
         #
         # configuring the object that corresponds to /traffic/trafficItem:1/configElement:1/stack:"udp-3"/field:"udp.header.length-3"
diff --git a/conf/00_common.conf b/conf/00_common.conf
index 7f30deb2..4c25b0b8 100644
--- a/conf/00_common.conf
+++ b/conf/00_common.conf
@@ -1,4 +1,4 @@
-# Copyright 2015 Intel Corporation.
+# Copyright 2015-2017 Intel Corporation.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -116,6 +116,9 @@ VERBOSITY = 'debug'
 # conventions
 TEST_PARAMS = {}
 
+# delay enforced after every step to allow system to process changes
+TEST_STEP_DELAY = 5
+
 # ############################
 # Modules
 # ############################
diff --git a/conf/01_testcases.conf b/conf/01_testcases.conf
index 2d5ab93e..df582df9 100755
--- a/conf/01_testcases.conf
+++ b/conf/01_testcases.conf
@@ -75,6 +75,16 @@
 #                                  # override any values defined by TEST_PARAMS option
 #                                  # stated in configuration files or values specified
 #                                  # on command line through --test-params parameter.
+#
+# "TestSteps": []                  # Definition of detailed test steps.
+#                                  # In case that this list is defined, then
+#                                  # vsperf will execute defined test steps
+#                                  # one by one. It can be used to configure
+#                                  # vswitch, insert flows and transmit traffic.
+#                                  # It is possible to refer to result of any
+#                                  # previous step through #STEP[i][j] macro.
+#                                  # Where i is a number of step (starts from 0)
+#                                  # and j is index of result returned by step i.
 # "Test Modifier": [FrameMod|Other],
 # "Dependency": [Test_Case_Name |None],
 
diff --git a/conf/__init__.py b/conf/__init__.py
index 2a2586ff..e24111dc 100644
--- a/conf/__init__.py
+++ b/conf/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2015-2016 Intel Corporation.
+# Copyright 2015-2017 Intel Corporation.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -49,6 +49,40 @@ class Settings(object):
     def __init__(self):
         pass
 
+    def _eval_param(self, param):
+        # pylint: disable=invalid-name
+        """ Helper function for expansion of references to vsperf parameters
+        """
+        if isinstance(param, str):
+            # evaluate every #PARAM reference inside parameter itself
+            macros = re.findall(r'#PARAM\((([\w\-]+)(\[[\w\[\]\-\'\"]+\])*)\)', param)
+            if macros:
+                for macro in macros:
+                    # pylint: disable=eval-used
+                    try:
+                        tmp_val = str(eval("self.getValue('{}'){}".format(macro[1], macro[2])))
+                        param = param.replace('#PARAM({})'.format(macro[0]), tmp_val)
+                    # silently ignore that option required by PARAM macro can't be evaluated;
+                    # It is possible, that referred parameter will be constructed during runtime
+                    # and re-read later.
+                    except IndexError:
+                        pass
+                    except AttributeError:
+                        pass
+            return param
+        elif isinstance(param, list) or isinstance(param, tuple):
+            tmp_list = []
+            for item in param:
+                tmp_list.append(self._eval_param(item))
+            return tmp_list
+        elif isinstance(param, dict):
+            tmp_dict = {}
+            for (key, value) in param.items():
+                tmp_dict[key] = self._eval_param(value)
+            return tmp_dict
+        else:
+            return param
+
     def getValue(self, attr):
         """Return a settings item value
         """
@@ -65,11 +99,11 @@ class Settings(object):
                     if attr == 'TRAFFIC':
                         tmp_value = copy.deepcopy(master_value)
                         tmp_value = merge_spec(tmp_value, cli_value)
-                        return tmp_value
+                        return self._eval_param(tmp_value)
                     else:
-                        return cli_value
+                        return self._eval_param(cli_value)
                 else:
-                    return master_value
+                    return self._eval_param(master_value)
         else:
             raise AttributeError("%r object has no attribute %r" %
                                  (self.__class__, attr))
@@ -189,7 +223,7 @@ class Settings(object):
         for key in self.__dict__:
             if key.startswith('GUEST_'):
                 value = self.getValue(key)
-                if isinstance(value, str) and value.find('#') >= 0:
+                if isinstance(value, str) and str(value).find('#') >= 0:
                     self._expand_vm_settings(key, 1)
 
                 if isinstance(value, list):
@@ -266,7 +300,9 @@ class Settings(object):
     def validate_getValue(self, result, attr):
         """Verifies, that correct value was returned
         """
-        assert result == self.__dict__[attr]
+        # getValue must be called to expand macros and apply
+        # values from TEST_PARAM option
+        assert result == self.getValue(attr)
         return True
 
     def validate_setValue(self, dummy_result, name, value):
diff --git a/conf/integration/01_testcases.conf b/conf/integration/01_testcases.conf
index 61766e33..b58fa965 100644
--- a/conf/integration/01_testcases.conf
+++ b/conf/integration/01_testcases.conf
@@ -1,4 +1,4 @@
-# Copyright 2015-2016 Intel Corporation.
+# Copyright 2015-2017 Intel Corporation.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -34,16 +34,6 @@ SUPPORTED_TUNNELING_PROTO = ['vxlan', 'gre', 'geneve']
 #
 # bidirectional testing for OP2P is not yet supported.
 # TRAFFIC['bidir'] must be set to 'False'.
-#
-# "TestSteps": []                   # Definition of integration test steps.
-#                                   # In case that this list is defined, then
-#                                   # vsperf will execute defined test steps
-#                                   # one by one. It can be used to configure
-#                                   # vswitch, insert flows and transmit traffic.
-#                                   # It is possible to refer to result of any
-#                                   # previous step through #STEP[i][j] macro.
-#                                   # Where i is a number of step (starts from 0)
-#                                   # and j is index of result returned by step i.
 
 #
 # Common TestSteps parts ("macros")
@@ -894,12 +884,12 @@ INTEGRATION_TESTS = [
         "TestSteps": STEP_VSWITCH_PVVP_INIT +                                       # STEP 0-6
             [
                 # check that at least 2 numa slots are available
-                ['tools', 'exec', 'numactl -H', 'available: ([0-9]+)'],             # STEP 7
+                ['tools', 'exec_shell', 'numactl -H', 'available: ([0-9]+)'],       # STEP 7
                 ['tools', 'assert', '#STEP[-1][0]>1'],                              # STEP 8
                 # store last 2 cores from numa slot 0
-                ['tools', 'exec', 'numactl -H', 'node 0 cpus:.*\s+(\\d+) (\\d+)$'], # STEP 9
+                ['tools', 'exec_shell', 'numactl -H', 'node 0 cpus:.*\s+(\\d+) (\\d+)$'], # STEP 9
                 # store last 2 cores from numa slot 1
-                ['tools', 'exec', 'numactl -H', 'node 1 cpus:.*\s+(\\d+) (\\d+)$'], # STEP 10
+                ['tools', 'exec_shell', 'numactl -H', 'node 1 cpus:.*\s+(\\d+) (\\d+)$'], # STEP 10
                 # pin VNF1 to 1st NUMA slot and VNF2 to 2nd NUMA slot
                 ['settings', 'setValue', 'GUEST_CORE_BINDING',                      # STEP 11
                     [("#STEP[-2][0][0]", "#STEP[-2][0][1]"),
@@ -912,7 +902,7 @@ INTEGRATION_TESTS = [
                 ['settings', 'getValue', 'TOOLS'],                                  # STEP 14
                 # check that PMD thread serving VNF1 runs at NUMA slot 0
                 ## i.e. get numa slot ID serving dpdhvhostuser0...
-                ['tools', 'exec', "sudo #STEP[-1]['ovs-appctl'] "                   # STEP 15
+                ['tools', 'exec_shell', "sudo #STEP[-1]['ovs-appctl'] "             # STEP 15
                     "dpif-netdev/pmd-rxq-show | "
                     "sed -e '/dpdkvhostuser0/,$d' | tac",
                     'pmd thread numa_id ([0-9])+'
@@ -921,7 +911,7 @@ INTEGRATION_TESTS = [
                 ['tools', 'assert', '#STEP[-1][0]==0'],                             # STEP 16
                 # check that PMD thread serving VNF2 runs at NUMA slot 1
                 ## i.e. get numa slot ID serving dpdhvhostuser2...
-                ['tools', 'exec', "sudo #STEP[-3]['ovs-appctl'] "                   # STEP 17
+                ['tools', 'exec_shell', "sudo #STEP[-3]['ovs-appctl'] "             # STEP 17
                     "dpif-netdev/pmd-rxq-show | "
                     "sed -e '/dpdkvhostuser2/,$d' | tac",
                     'pmd thread numa_id ([0-9])+'
diff --git a/core/traffic_controller.py b/core/traffic_controller.py
index b1911536..5ebdc0d9 100644
--- a/core/traffic_controller.py
+++ b/core/traffic_controller.py
@@ -1,4 +1,4 @@
-# Copyright 2015-2016 Intel Corporation.
+# Copyright 2015-2017 Intel Corporation.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -44,7 +44,7 @@ class TrafficController(object):
         self._lossrate = float(settings.getValue('TRAFFICGEN_LOSSRATE'))
         self._packet_sizes = settings.getValue('TRAFFICGEN_PKT_SIZES')
 
-        self._mode = settings.getValue('mode').lower()
+        self._mode = str(settings.getValue('mode')).lower()
         self._results = []
 
     def __enter__(self):
diff --git a/core/traffic_controller_rfc2544.py b/core/traffic_controller_rfc2544.py
index e230c832..cb839518 100644
--- a/core/traffic_controller_rfc2544.py
+++ b/core/traffic_controller_rfc2544.py
@@ -1,4 +1,4 @@
-# Copyright 2015-2016 Intel Corporation.
+# Copyright 2015-2017 Intel Corporation.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -42,6 +42,9 @@ class TrafficControllerRFC2544(TrafficController, IResults):
         self._logger.debug('send_traffic with ' +
                            str(self._traffic_gen_class))
 
+        # update type with detailed traffic value
+        self._type = traffic['traffic_type']
+
         for packet_size in self._packet_sizes:
             # Merge framesize with the default traffic definition
             if 'l2' in traffic:
@@ -60,8 +63,8 @@ class TrafficControllerRFC2544(TrafficController, IResults):
                 result = self._traffic_gen_class.send_rfc2544_throughput(
                     traffic, tests=self._tests, duration=self._duration, lossrate=self._lossrate)
             else:
-                raise RuntimeError("Unsupported traffic type {} was \
-                                   detected".format(traffic['traffic_type']))
+                raise RuntimeError("Unsupported traffic type {} was "
+                                   "detected".format(traffic['traffic_type']))
 
             result = self._append_results(result, packet_size)
             self._results.append(result)
@@ -74,6 +77,9 @@ class TrafficControllerRFC2544(TrafficController, IResults):
         self._logger.debug('send_traffic_async with ' +
                            str(self._traffic_gen_class))
 
+        # update type with detailed traffic value
+        self._type = traffic['traffic_type']
+
         for packet_size in self._packet_sizes:
             traffic['l2'] = {'framesize': packet_size}
             self._traffic_gen_class.start_rfc2544_throughput(
diff --git a/core/traffic_controller_rfc2889.py b/core/traffic_controller_rfc2889.py
index 05955e65..01aaa722 100644
--- a/core/traffic_controller_rfc2889.py
+++ b/core/traffic_controller_rfc2889.py
@@ -1,4 +1,4 @@
-# Copyright 2016 Spirent Communications, Intel Corporation.
+# Copyright 2016-2017 Spirent Communications, Intel Corporation.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -42,6 +42,9 @@ class TrafficControllerRFC2889(TrafficController, IResults):
         self._logger.debug('send_traffic with ' +
                            str(self._traffic_gen_class))
 
+        # update type with detailed traffic value
+        self._type = traffic['traffic_type']
+
         for packet_size in self._packet_sizes:
             # Merge framesize with the default traffic definition
             if 'l2' in traffic:
@@ -71,6 +74,9 @@ class TrafficControllerRFC2889(TrafficController, IResults):
         self._logger.debug('send_traffic_async with ' +
                            str(self._traffic_gen_class))
 
+        # update type with detailed traffic value
+        self._type = traffic['traffic_type']
+
         for packet_size in self._packet_sizes:
             traffic['l2'] = {'framesize': packet_size}
             self._traffic_gen_class.start_rfc2889_forwarding(
diff --git a/core/vswitch_controller_op2p.py b/core/vswitch_controller_op2p.py
index ee8ada8b..85bf79bd 100644
--- a/core/vswitch_controller_op2p.py
+++ b/core/vswitch_controller_op2p.py
@@ -1,4 +1,4 @@
-# Copyright 2015-2016 Intel Corporation.
+# Copyright 2015-2017 Intel Corporation.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,7 +19,7 @@ import logging
 
 from core.vswitch_controller import IVswitchController
 from vswitches.utils import add_ports_to_flow
-from conf import settings
+from conf import settings as S
 from tools import tasks
 
 _FLOW_TEMPLATE = {
@@ -55,7 +55,7 @@ class VswitchControllerOP2P(IVswitchController):
         if self._tunnel_operation == "encapsulation":
             self._setup_encap()
         else:
-            if settings.getValue('VSWITCH').endswith('Vanilla'):
+            if str(S.getValue('VSWITCH')).endswith('Vanilla'):
                 self._setup_decap_vanilla()
             else:
                 self._setup_decap()
@@ -70,17 +70,17 @@ class VswitchControllerOP2P(IVswitchController):
 
         try:
             self._vswitch.start()
-            bridge = settings.getValue('TUNNEL_INTEGRATION_BRIDGE')
-            bridge_ext = settings.getValue('TUNNEL_EXTERNAL_BRIDGE')
-            bridge_ext_ip = settings.getValue('TUNNEL_EXTERNAL_BRIDGE_IP')
-            tg_port2_mac = settings.getValue('TRAFFICGEN_PORT2_MAC')
-            vtep_ip2 = settings.getValue('VTEP_IP2')
+            bridge = S.getValue('TUNNEL_INTEGRATION_BRIDGE')
+            bridge_ext = S.getValue('TUNNEL_EXTERNAL_BRIDGE')
+            bridge_ext_ip = S.getValue('TUNNEL_EXTERNAL_BRIDGE_IP')
+            tg_port2_mac = S.getValue('TRAFFICGEN_PORT2_MAC')
+            vtep_ip2 = S.getValue('VTEP_IP2')
             self._vswitch.add_switch(bridge)
 
             tasks.run_task(['sudo', 'ip', 'addr', 'add',
-                            settings.getValue('VTEP_IP1'), 'dev', bridge],
+                            S.getValue('VTEP_IP1'), 'dev', bridge],
                            self._logger, 'Assign ' +
-                           settings.getValue('VTEP_IP1') + ' to ' + bridge,
+                           S.getValue('VTEP_IP1') + ' to ' + bridge,
                            False)
             tasks.run_task(['sudo', 'ip', 'link', 'set', 'dev', bridge, 'up'],
                            self._logger, 'Bring up ' + bridge, False)
@@ -104,10 +104,10 @@ class VswitchControllerOP2P(IVswitchController):
                            'Set ' + bridge_ext + 'status to up')
 
             self._vswitch.add_route(bridge,
-                                    settings.getValue('VTEP_IP2_SUBNET'),
+                                    S.getValue('VTEP_IP2_SUBNET'),
                                     bridge_ext)
 
-            if settings.getValue('VSWITCH').endswith('Vanilla'):
+            if str(S.getValue('VSWITCH')).endswith('Vanilla'):
                 tasks.run_task(['sudo', 'arp', '-s', vtep_ip2, tg_port2_mac],
                                self._logger,
                                'Set ' + bridge_ext + ' status to up')
@@ -133,16 +133,16 @@ class VswitchControllerOP2P(IVswitchController):
 
         try:
             self._vswitch.start()
-            bridge = settings.getValue('TUNNEL_INTEGRATION_BRIDGE')
-            bridge_ext = settings.getValue('TUNNEL_EXTERNAL_BRIDGE')
-            bridge_ext_ip = settings.getValue('TUNNEL_EXTERNAL_BRIDGE_IP')
-            tgen_ip1 = settings.getValue('TRAFFICGEN_PORT1_IP')
+            bridge = S.getValue('TUNNEL_INTEGRATION_BRIDGE')
+            bridge_ext = S.getValue('TUNNEL_EXTERNAL_BRIDGE')
+            bridge_ext_ip = S.getValue('TUNNEL_EXTERNAL_BRIDGE_IP')
+            tgen_ip1 = S.getValue('TRAFFICGEN_PORT1_IP')
             self._vswitch.add_switch(bridge)
 
             tasks.run_task(['sudo', 'ip', 'addr', 'add',
-                            settings.getValue('VTEP_IP1'), 'dev', bridge],
+                            S.getValue('VTEP_IP1'), 'dev', bridge],
                            self._logger, 'Assign ' +
-                           settings.getValue('VTEP_IP1') + ' to ' + bridge, False)
+                           S.getValue('VTEP_IP1') + ' to ' + bridge, False)
             tasks.run_task(['sudo', 'ip', 'link', 'set', 'dev', bridge, 'up'],
                            self._logger, 'Bring up ' + bridge, False)
 
@@ -152,7 +152,7 @@ class VswitchControllerOP2P(IVswitchController):
             self._vswitch.add_phy_port(bridge)
             (_, phy2_number) = self._vswitch.add_phy_port(bridge_ext)
             if tunnel_type == "vxlan":
-                vxlan_vni = 'options:key=' + settings.getValue('VXLAN_VNI')
+                vxlan_vni = 'options:key=' + S.getValue('VXLAN_VNI')
                 (_, phy3_number) = self._vswitch.add_tunnel_port(bridge_ext,
                                                                  tgen_ip1,
                                                                  tunnel_type,
@@ -174,7 +174,7 @@ class VswitchControllerOP2P(IVswitchController):
                            'Set ' + bridge_ext + ' status to up')
 
             self._vswitch.set_tunnel_arp(tgen_ip1,
-                                         settings.getValue('TRAFFICGEN_PORT1_MAC'),
+                                         S.getValue('TRAFFICGEN_PORT1_MAC'),
                                          bridge)
             # Test is unidirectional for now
             self._vswitch.del_flow(bridge_ext)
@@ -193,16 +193,16 @@ class VswitchControllerOP2P(IVswitchController):
 
         try:
             self._vswitch.start()
-            bridge = settings.getValue('TUNNEL_INTEGRATION_BRIDGE')
-            bridge_ext = settings.getValue('TUNNEL_EXTERNAL_BRIDGE')
-            bridge_ext_ip = settings.getValue('TUNNEL_EXTERNAL_BRIDGE_IP')
-            tgen_ip1 = settings.getValue('TRAFFICGEN_PORT1_IP')
+            bridge = S.getValue('TUNNEL_INTEGRATION_BRIDGE')
+            bridge_ext = S.getValue('TUNNEL_EXTERNAL_BRIDGE')
+            bridge_ext_ip = S.getValue('TUNNEL_EXTERNAL_BRIDGE_IP')
+            tgen_ip1 = S.getValue('TRAFFICGEN_PORT1_IP')
             self._vswitch.add_switch(bridge)
 
             tasks.run_task(['sudo', 'ip', 'addr', 'add',
-                            settings.getValue('TUNNEL_INT_BRIDGE_IP'), 'dev', bridge],
+                            S.getValue('TUNNEL_INT_BRIDGE_IP'), 'dev', bridge],
                            self._logger, 'Assign ' +
-                           settings.getValue('TUNNEL_INT_BRIDGE_IP') + ' to ' + bridge, False)
+                           S.getValue('TUNNEL_INT_BRIDGE_IP') + ' to ' + bridge, False)
             tasks.run_task(['sudo', 'ip', 'link', 'set', 'dev', bridge, 'up'],
                            self._logger, 'Bring up ' + bridge, False)
 
@@ -213,7 +213,7 @@ class VswitchControllerOP2P(IVswitchController):
             (_, phy2_number) = self._vswitch.add_phy_port(bridge)
 
             if tunnel_type == "vxlan":
-                vxlan_vni = 'options:key=' + settings.getValue('VXLAN_VNI')
+                vxlan_vni = 'options:key=' + S.getValue('VXLAN_VNI')
                 self._vswitch.add_tunnel_port(bridge, tgen_ip1, tunnel_type,
                                               params=[vxlan_vni])
             else:
@@ -231,15 +231,15 @@ class VswitchControllerOP2P(IVswitchController):
                            self._logger,
                            'Set ' + bridge_ext + ' status to up')
 
-            tg_port2_mac = settings.getValue('TRAFFICGEN_PORT2_MAC')
-            vtep_ip2 = settings.getValue('TRAFFICGEN_PORT2_IP')
+            tg_port2_mac = S.getValue('TRAFFICGEN_PORT2_MAC')
+            vtep_ip2 = S.getValue('TRAFFICGEN_PORT2_IP')
 
             self._vswitch.set_tunnel_arp(vtep_ip2,
                                          tg_port2_mac,
                                          bridge_ext)
 
             self._vswitch.add_route(bridge,
-                                    settings.getValue('VTEP_IP2_SUBNET'),
+                                    S.getValue('VTEP_IP2_SUBNET'),
                                     bridge)
 
 
@@ -278,10 +278,16 @@ class VswitchControllerOP2P(IVswitchController):
     def get_ports_info(self):
         """See IVswitchController for description
         """
-        self._logger.debug('get_ports_info  using ' + str(self._vswitch_class))
-        return self._vswitch.get_ports(settings.getValue('VSWITCH_BRIDGE_NAME'))
+        self._logger.debug('get_ports_info for bridges: %s, %s',
+                           S.getValue('TUNNEL_INTEGRATION_BRIDGE'),
+                           S.getValue('TUNNEL_EXTERNAL_BRIDGE'))
+        return self._vswitch.get_ports(
+            S.getValue('TUNNEL_INTEGRATION_BRIDGE')) +\
+                self._vswitch.get_ports(
+                    S.getValue('TUNNEL_EXTERNAL_BRIDGE'))
 
     def dump_vswitch_flows(self):
         """See IVswitchController for description
         """
-        self._vswitch.dump_flows(settings.getValue('VSWITCH_BRIDGE_NAME'))
+        self._vswitch.dump_flows(S.getValue('TUNNEL_INTEGRATION_BRIDGE'))
+        self._vswitch.dump_flows(S.getValue('TUNNEL_EXTERNAL_BRIDGE'))
diff --git a/docs/testing/user/userguide/teststeps.rst b/docs/testing/user/userguide/teststeps.rst
index 870c3d80..71f19714 100644
--- a/docs/testing/user/userguide/teststeps.rst
+++ b/docs/testing/user/userguide/teststeps.rst
@@ -192,14 +192,16 @@ of supported objects and their most common functions follows:
           in case that condition is not ``True``
         * ``Eval expression`` - evaluates given expression as a python code and returns
           its result
-        * ``Exec command [regex]`` - executes a shell command and filters its output by
+        * ``Exec_Shell command [regex]`` - executes a shell command and filters its output by
           (optional) regular expression
+        * ``Exec_Python code`` - executes a python code
+
 
         Examples:
 
         .. code-block:: python
 
-            ['tools', 'exec', 'numactl -H', 'available: ([0-9]+)']
+            ['tools', 'exec_shell', 'numactl -H', 'available: ([0-9]+)']
             ['tools', 'assert', '#STEP[-1][0]>1']
 
     * ``wait`` - is used for test case interruption. This object doesn't have
@@ -213,6 +215,14 @@ of supported objects and their most common functions follows:
 
         ['wait']
 
+    * ``sleep`` - is used to pause testcase execution for defined number of seconds.
+
+      Examples:
+
+      .. code-block:: python
+
+        ['sleep', '60']
+
 Test Macros
 -----------
 
diff --git a/docs/testing/user/userguide/testusage.rst b/docs/testing/user/userguide/testusage.rst
index f8490768..b6939e57 100644
--- a/docs/testing/user/userguide/testusage.rst
+++ b/docs/testing/user/userguide/testusage.rst
@@ -112,6 +112,32 @@ of options with ``GUEST_`` prefix could be found at :ref:`design document
 
 .. _overriding-parameters-documentation:
 
+Referencing parameter values
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+It is possible to use a special macro ``#PARAM()`` to refer to the value of
+another configuration parameter. This reference is evaluated during
+access of the parameter value (by ``settings.getValue()`` call), so it
+can refer to parameters created during VSPERF runtime, e.g. NICS dictionary.
+It can be used to reflect DUT HW details in the testcase definition.
+
+Example:
+
+.. code:: python
+
+    {
+        ...
+        "Name": "testcase",
+        "Parameters" : {
+            "TRAFFIC" : {
+                'l2': {
+                    # set destination MAC to the MAC of the first
+                    # interface from WHITELIST_NICS list
+                    'dstmac' : '#PARAM(NICS[0]["mac"])',
+                },
+            },
+        ...
+
 Overriding values defined in configuration files
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/src/ovs/ofctl.py b/src/ovs/ofctl.py
index f0d116da..64d54466 100644
--- a/src/ovs/ofctl.py
+++ b/src/ovs/ofctl.py
@@ -22,6 +22,7 @@ https://github.com/openstack/neutron/blob/6eac1dc99124ca024d6a69b3abfa3bc69c7356
 import logging
 import string
 import re
+import netaddr
 
 from tools import tasks
 from conf import settings
@@ -31,6 +32,9 @@ _OVS_CMD_TIMEOUT = settings.getValue('OVS_CMD_TIMEOUT')
 
 _CACHE_FILE_NAME = '/tmp/vsperf_flows_cache'
 
+# only simple regex is used; validity of IPv4 is not checked by regex
+_IPV4_REGEX = r"([0-9]{1,3}(\.[0-9]{1,3}){3}(\/[0-9]{1,2})?)"
+
 class OFBase(object):
     """Add/remove/show datapaths using ``ovs-ofctl``.
     """
@@ -446,10 +450,17 @@ def flow_match(flow_dump, flow_src):
     flow_src = flow_src.replace('udp_dst', 'tp_dst')
     flow_src = flow_src.replace('tcp_src', 'tp_src')
     flow_src = flow_src.replace('tcp_dst', 'tp_dst')
+    flow_src = flow_src.replace('0x800', '0x0800')
+
+    # modify IPv4 CIDR to real network addresses
+    for ipv4_cidr in re.findall(_IPV4_REGEX, flow_src):
+        if ipv4_cidr[2]:
+            tmp_cidr = str(netaddr.IPNetwork(ipv4_cidr[0]).cidr)
+            flow_src = flow_src.replace(ipv4_cidr[0], tmp_cidr)
 
     # split flow strings into lists of comparable elements
-    flow_dump_list = re.findall(r"[\w.:=()]+", flow_dump)
-    flow_src_list = re.findall(r"[\w.:=()]+", flow_src)
+    flow_dump_list = re.findall(r"[\w.:=()/]+", flow_dump)
+    flow_src_list = re.findall(r"[\w.:=()/]+", flow_src)
 
     # check if all items from source flow are present in dump flow
     flow_src_ctrl = list(flow_src_list)
diff --git a/testcases/integration.py b/testcases/integration.py
index f2a5fecf..f87a8ee2 100644
--- a/testcases/integration.py
+++ b/testcases/integration.py
@@ -1,4 +1,4 @@
-# Copyright 2015-2016 Intel Corporation.
+# Copyright 2015-2017 Intel Corporation.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -36,10 +36,13 @@ class IntegrationTestCase(TestCase):
         """ Report test results
         """
         if self.test:
-            results = OrderedDict()
-            results['status'] = 'OK' if self._step_status['status'] else 'FAILED'
-            results['details'] = self._step_status['details']
-            TestCase.write_result_to_file([results], self._output_file)
+            tmp_results = OrderedDict()
+            tmp_results['status'] = 'OK' if self._step_status['status'] else 'FAILED'
+            tmp_results['details'] = self._step_status['details']
+            self._tc_results = [tmp_results]
+
+            super(IntegrationTestCase, self).run_report()
+
             self.step_report_status("Test '{}'".format(self.name), self._step_status['status'])
             # inform vsperf about testcase failure
             if not self._step_status['status']:
diff --git a/testcases/testcase.py b/testcases/testcase.py
index 75ed1a5c..1537186f 100644
--- a/testcases/testcase.py
+++ b/testcases/testcase.py
@@ -143,7 +143,7 @@ class TestCase(object):
         self._traffic = functions.check_traffic(self._traffic)
 
         # Packet Forwarding mode
-        self._vswitch_none = S.getValue('VSWITCH').strip().lower() == 'none'
+        self._vswitch_none = str(S.getValue('VSWITCH')).strip().lower() == 'none'
 
         # trafficgen configuration required for tests of tunneling protocols
         if self.deployment == "op2p":
@@ -289,8 +289,31 @@ class TestCase(object):
             self._logger.debug("Traffic Results:")
             self._traffic_ctl.print_results()
 
+        if self._tc_results is None:
             self._tc_results = self._append_results(results)
-            TestCase.write_result_to_file(self._tc_results, self._output_file)
+        else:
+            # integration step driven tests have their status and possible
+            # failure details stored inside self._tc_results
+            results = self._append_results(results)
+            if len(self._tc_results) < len(results):
+                if len(self._tc_results) > 1:
+                    raise RuntimeError('Testcase results do not match:'
+                                       'results: {}\n'
+                                       'trafficgen results: {}\n',
+                                       self._tc_results,
+                                       results)
+                else:
+                    tmp_results = copy.deepcopy(self._tc_results[0])
+                    self._tc_results = []
+                    for res in results:
+                        tmp_res = copy.deepcopy(tmp_results)
+                        tmp_res.update(res)
+                        self._tc_results.append(tmp_res)
+            else:
+                for i, result in enumerate(results):
+                    self._tc_results[i].update(result)
+
+        TestCase.write_result_to_file(self._tc_results, self._output_file)
 
     def run(self):
         """Run the test
@@ -440,7 +463,7 @@ class TestCase(object):
         # hugepages are needed by DPDK and Qemu
         if not self._hugepages_mounted and \
             (self.deployment.count('v') or \
-             S.getValue('VSWITCH').lower().count('dpdk') or \
+             str(S.getValue('VSWITCH')).lower().count('dpdk') or \
              self._vswitch_none or \
              self.test and 'vnf' in [step[0][0:3] for step in self.test]):
             hugepages.mount_hugepages()
@@ -467,7 +490,7 @@ class TestCase(object):
         # get hugepage amounts for each socket on dpdk
         sock0_mem, sock1_mem = 0, 0
 
-        if S.getValue('VSWITCH').lower().count('dpdk'):
+        if str(S.getValue('VSWITCH')).lower().count('dpdk'):
             sock_mem = S.getValue('DPDK_SOCKET_MEM')
             sock0_mem, sock1_mem = (int(sock_mem[0]) * 1024 / hugepage_size,
                                     int(sock_mem[1]) * 1024 / hugepage_size)
@@ -739,6 +762,10 @@ class TestCase(object):
                 input(os.linesep + "Step {}: Press Enter to continue with "
                       "the next step...".format(i) + os.linesep + os.linesep)
                 continue
+            elif step[0] == 'sleep':
+                self._logger.debug("Sleep %s seconds", step[1])
+                time.sleep(int(step[1]))
+                continue
             else:
                 self._logger.error("Unsupported test object %s", step[0])
                 self._step_status = {'status' : False, 'details' : ' '.join(step)}
@@ -762,7 +789,7 @@ class TestCase(object):
                 self._step_result[i] = test_method(*step_params)
                 self._logger.debug("Step %s '%s' results '%s'", i,
                                    step_log, self._step_result[i])
-                time.sleep(5)
+                time.sleep(S.getValue('TEST_STEP_DELAY'))
                 if self._step_check:
                     step_ok = test_method_check(self._step_result[i], *step_params)
             except (AssertionError, AttributeError, IndexError) as ex:
diff --git a/tools/report/report.py b/tools/report/report.py
index 4d892075..b3f15c1b 100644
--- a/tools/report/report.py
+++ b/tools/report/report.py
@@ -67,7 +67,7 @@ def _get_env(result, versions):
         'vswitch': _get_version(S.getValue('VSWITCH'), versions),
     }
 
-    if S.getValue('VSWITCH').lower().count('dpdk'):
+    if str(S.getValue('VSWITCH')).lower().count('dpdk'):
         env.update({'dpdk': _get_version('dpdk', versions)})
 
     if result[ResultsConstants.DEPLOYMENT].count('v'):
diff --git a/tools/teststepstools.py b/tools/teststepstools.py
index d39f7f40..5d551c68 100644
--- a/tools/teststepstools.py
+++ b/tools/teststepstools.py
@@ -1,4 +1,4 @@
-# Copyright 2016 Intel Corporation.
+# Copyright 2016-2017 Intel Corporation.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,25 +20,23 @@ import logging
 import subprocess
 import locale
 
+_LOGGER = logging.getLogger(__name__)
+
 class TestStepsTools(object):
     """ Various tools and functions used by step driven testcases
     """
     # Functions use nonstandard names to avoid conflicts with
     # standard python keywords.
     # pylint: disable=invalid-name
-    def __init__(self):
-        """ TestStepsTools initialization
-        """
-        self._logger = logging.getLogger(__name__)
-
-    def Assert(self, condition):
+    @staticmethod
+    def Assert(condition):
         """ Evaluate given `condition' and raise AssertionError
             in case, that evaluation fails
         """
         try:
-            assert self.Eval(condition)
+            assert TestStepsTools.Eval(condition)
         except AssertionError:
-            self._logger.error('Condition %s is not True', condition)
+            _LOGGER.error('Condition %s is not True', condition)
             raise
 
         return True
@@ -63,7 +61,27 @@ class TestStepsTools(object):
         return result is not None
 
     @staticmethod
-    def Exec(command, regex=None):
+    def Exec_Python(code):
+        """ Execute a python `code' and return True on success
+        """
+        # pylint: disable=exec-used
+        try:
+            exec(code, globals())
+        # pylint: disable=broad-except
+        # pylint: disable=bare-except
+        except:
+            _LOGGER.error('Execution of following code has failed %s', code)
+            return False
+        return True
+
+    @staticmethod
+    def validate_Exec_Python(result, dummy_code):
+        """ Validate result of python `code' execution
+        """
+        return result
+
+    @staticmethod
+    def Exec_Shell(command, regex=None):
         """ Execute a shell `command' and return its output filtered
             out by optional `regex' expression.
         """
@@ -84,7 +102,7 @@ class TestStepsTools(object):
         return output
 
     @staticmethod
-    def validate_Exec(result, dummy_command, dummy_regex=None):
+    def validate_Exec_Shell(result, dummy_command, dummy_regex=None):
         """ validate result of shell `command' execution
         """
         return result is not None
diff --git a/vnfs/qemu/qemu.py b/vnfs/qemu/qemu.py
index b48f7630..20b4d62a 100644
--- a/vnfs/qemu/qemu.py
+++ b/vnfs/qemu/qemu.py
@@ -1,4 +1,4 @@
-# Copyright 2015-2016 Intel Corporation.
+# Copyright 2015-2017 Intel Corporation.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -73,7 +73,7 @@ class IVnfQemu(IVnf):
         self._testpmd_fwd_mode = S.getValue('GUEST_TESTPMD_FWD_MODE')[self._number]
         # in case of SRIOV we must ensure, that MAC addresses are not swapped
         if S.getValue('SRIOV_ENABLED') and self._testpmd_fwd_mode.startswith('mac') and \
-           not S.getValue('VNF').endswith('PciPassthrough'):
+           not str(S.getValue('VNF')).endswith('PciPassthrough'):
 
             self._logger.info("SRIOV detected, forwarding mode of testpmd was changed from '%s' to '%s'",
                               self._testpmd_fwd_mode, 'io')
diff --git a/vsperf b/vsperf
index 6618bed5..bb0e199b 100755
--- a/vsperf
+++ b/vsperf
@@ -701,7 +701,7 @@ def main():
                         'installer': installer_name,
                         'pkg_list': pkg_list,
                         'db_url': opnfv_url}
-            if settings.getValue('VSWITCH').endswith('Vanilla'):
+            if str(settings.getValue('VSWITCH')).endswith('Vanilla'):
                 int_data['vanilla'] = True
             opnfvdashboard.results2opnfv_dashboard(results_path, int_data)