integration: Support of PVP and PVVP integration TCs

Integration TC support has been enhanced to support PVP
and PVVP scenarios. Definition of integration testcases
have been modified to use a sort of macros for repetitive
parts.
Additional improvements were introduced:
* instances of testcases are created only for testcases
selected for execution
* new TC definition options allow to define test specific
vswitch, VNF, traffic generator and test options
* tests filter applied on pattern specified by --tests
allows to define negative filter only; In that case
list of all tests is used as base for negative filter.
* traffic values defined within teststep passed to send_traffic
is merged with default values; This is essential for execution
of TCs with linux_bridge or SRIOV support. It also simplifies
integration TC definition
* typos removed

Change-Id: Icb734a7afd7e5154f27a8ff25615a39e01f58c27
JIRA: VSPERF-213
JIRA: VSPERF-216
Signed-off-by: Martin Klozik <martinx.klozik@intel.com>
Reviewed-by: Maryam Tahhan <maryam.tahhan@intel.com>
Reviewed-by: Al Morton <acmorton@att.com>
Reviewed-by: Christian Trautman <ctrautma@redhat.com>
Reviewed-by: Brian Castelli <brian.castelli@spirent.com>

9 files changed, 500 insertions, 183 deletions

diff --git a/conf/01_testcases.conf b/conf/01_testcases.conf
index f36172df..148171fd 100755
--- a/conf/01_testcases.conf
+++ b/conf/01_testcases.conf
@@ -100,6 +100,24 @@
 #                                  # value will be used
 #   "options": ""                  # Optional. Additional command line options
 #                                  # to be passed to the load generator.
+# "vSwitch" : "OvsVanilla"         # Defines vSwitch to be used for test execution.
+#                                  # It will override any VSWITCH option stated
+#                                  # in configuration files or value specified
+#                                  # on command line through --vswitch parameter.
+# "VNF" : "QemuVirtioNet"          # Defines VNF to be used for test execution.
+#                                  # It will override any VNF option stated
+#                                  # in configuration files or value specified
+#                                  # on command line through --vnf parameter.
+# "Trafficgen" : "Dummy"           # Defines traffic generator to be used for test
+#                                  # execution. It will override any VNF option
+#                                  # stated in configuration files or value
+#                                  # specified on command line through --trafficgen
+#                                  # parameter.
+# "Parameters" : "pkt_sizes=512"   # Defines list of test parameters used for test
+#                                  # execution. It will override any values defined
+#                                  # by TEST_PARAMS option stated in configuration
+#                                  # files or values specified on command line through
+#                                  # --test-params parameter.
 # "Test Modifier": [FrameMod|Other],
 # "Dependency": [Test_Case_Name |None],
 
diff --git a/conf/integration/01_testcases.conf b/conf/integration/01_testcases.conf
index fff148d4..e9257ae0 100644
--- a/conf/integration/01_testcases.conf
+++ b/conf/integration/01_testcases.conf
@@ -16,6 +16,13 @@
 # tunneling protocol for OP2P tests.
 SUPPORTED_TUNNELING_PROTO = ['vxlan', 'gre', 'geneve']
 
+#
+# Generic test configuration options are described at conf/01_testcases.conf
+#
+
+#
+# Options specific to integration testcases are described below:
+#
 # Required for OP2P tests
 # "Tunnel Type": ["vxlan"|"gre"|"geneve"]   # Tunnel Type defines tunneling protocol to use.
 #                                   # It can be overridden by cli option tunnel_type.
@@ -38,6 +45,92 @@ SUPPORTED_TUNNELING_PROTO = ['vxlan', 'gre', 'geneve']
 #                                   # Where i is a number of step (starts from 0)
 #                                   # and j is index of result returned by step i.
 
+#
+# Common TestSteps parts ("macros")
+#
+
+# P2P macros
+STEP_VSWITCH_P2P_FLOWS_INIT = [
+    ['vswitch', 'add_switch', 'int_br0'],           # STEP 0
+    ['vswitch', 'add_phy_port', 'int_br0'],         # STEP 1
+    ['vswitch', 'add_phy_port', 'int_br0'],         # STEP 2
+    ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[1][1]', 'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}],
+    ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[2][1]', 'actions': ['output:#STEP[1][1]'], 'idle_timeout': '0'}],
+]
+
+STEP_VSWITCH_P2P_FLOWS_FINIT = [
+    ['vswitch', 'dump_flows', 'int_br0'],
+    ['vswitch', 'del_flow', 'int_br0', {'in_port': '#STEP[1][1]'}],
+    ['vswitch', 'del_flow', 'int_br0', {'in_port': '#STEP[2][1]'}],
+    ['vswitch', 'del_port', 'int_br0', '#STEP[1][0]'],
+    ['vswitch', 'del_port', 'int_br0', '#STEP[2][0]'],
+    ['vswitch', 'del_switch', 'int_br0'],
+]
+
+# PVP and PVVP macros
+STEP_VSWITCH_PVP_INIT = [
+    ['vswitch', 'add_switch', 'int_br0'],           # STEP 0
+    ['vswitch', 'add_phy_port', 'int_br0'],         # STEP 1
+    ['vswitch', 'add_phy_port', 'int_br0'],         # STEP 2
+    ['vswitch', 'add_vport', 'int_br0'],            # STEP 3
+    ['vswitch', 'add_vport', 'int_br0'],            # STEP 4
+]
+
+STEP_VSWITCH_PVP_FINIT = [
+    ['vswitch', 'del_port', 'int_br0', '#STEP[1][0]'],
+    ['vswitch', 'del_port', 'int_br0', '#STEP[2][0]'],
+    ['vswitch', 'del_port', 'int_br0', '#STEP[3][0]'],
+    ['vswitch', 'del_port', 'int_br0', '#STEP[4][0]'],
+    ['vswitch', 'del_switch', 'int_br0'],
+]
+
+STEP_VSWITCH_PVP_FLOWS_INIT = STEP_VSWITCH_PVP_INIT + [
+    ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[1][1]', 'actions': ['output:#STEP[3][1]'], 'idle_timeout': '0'}],
+    ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[4][1]', 'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}],
+    ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[2][1]', 'actions': ['output:#STEP[4][1]'], 'idle_timeout': '0'}],
+    ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[3][1]', 'actions': ['output:#STEP[1][1]'], 'idle_timeout': '0'}],
+]
+
+STEP_VSWITCH_PVP_FLOWS_FINIT = [
+    ['vswitch', 'dump_flows', 'int_br0'],
+    ['vswitch', 'del_flow', 'int_br0', {'in_port': '#STEP[1][1]'}],
+    ['vswitch', 'del_flow', 'int_br0', {'in_port': '#STEP[4][1]'}],
+    ['vswitch', 'del_flow', 'int_br0', {'in_port': '#STEP[2][1]'}],
+    ['vswitch', 'del_flow', 'int_br0', {'in_port': '#STEP[3][1]'}],
+] + STEP_VSWITCH_PVP_FINIT
+
+STEP_VSWITCH_PVVP_INIT = STEP_VSWITCH_PVP_INIT + [
+    ['vswitch', 'add_vport', 'int_br0'],            # STEP 5
+    ['vswitch', 'add_vport', 'int_br0'],            # STEP 6
+]
+
+STEP_VSWITCH_PVVP_FINIT = [
+    ['vswitch', 'del_port', 'int_br0', '#STEP[5][0]'],
+    ['vswitch', 'del_port', 'int_br0', '#STEP[6][0]'],
+] + STEP_VSWITCH_PVP_FINIT
+
+STEP_VSWITCH_PVVP_FLOWS_INIT = STEP_VSWITCH_PVVP_INIT + [
+    ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[1][1]', 'actions': ['output:#STEP[3][1]'], 'idle_timeout': '0'}],
+    ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[4][1]', 'actions': ['output:#STEP[5][1]'], 'idle_timeout': '0'}],
+    ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[6][1]', 'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}],
+    ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[2][1]', 'actions': ['output:#STEP[6][1]'], 'idle_timeout': '0'}],
+    ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[5][1]', 'actions': ['output:#STEP[4][1]'], 'idle_timeout': '0'}],
+    ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[3][1]', 'actions': ['output:#STEP[1][1]'], 'idle_timeout': '0'}],
+]
+
+STEP_VSWITCH_PVVP_FLOWS_FINIT = [
+    ['vswitch', 'dump_flows', 'int_br0'],
+    ['vswitch', 'del_flow', 'int_br0', {'in_port': '#STEP[1][1]'}],
+    ['vswitch', 'del_flow', 'int_br0', {'in_port': '#STEP[4][1]'}],
+    ['vswitch', 'del_flow', 'int_br0', {'in_port': '#STEP[6][1]'}],
+    ['vswitch', 'del_flow', 'int_br0', {'in_port': '#STEP[2][1]'}],
+    ['vswitch', 'del_flow', 'int_br0', {'in_port': '#STEP[5][1]'}],
+    ['vswitch', 'del_flow', 'int_br0', {'in_port': '#STEP[3][1]'}],
+] + STEP_VSWITCH_PVVP_FINIT
+
+#
+# Definition of integration tests
+#
 INTEGRATION_TESTS = [
     {
         "Name": "overlay_p2p_tput",
@@ -162,76 +255,187 @@ INTEGRATION_TESTS = [
         "Name": "vswitch_add_del_flows",
         "Deployment": "clean",
         "Description": "vSwitch - add and delete flows",
-        "TestSteps": [
-                        ['vswitch', 'add_switch', 'int_br0'],
-                        ['vswitch', 'add_phy_port', 'int_br0'],
-                        ['vswitch', 'add_phy_port', 'int_br0'],
-                        ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[1][1]', 'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}],
-                        ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[2][1]', 'actions': ['output:#STEP[1][1]'], 'idle_timeout': '0'}],
-                        ['vswitch', 'dump_flows', 'int_br0'],
-                        ['vswitch', 'del_flow', 'int_br0', {'in_port': '#STEP[1][1]'}],
-                        ['vswitch', 'del_flow', 'int_br0', {'in_port': '#STEP[2][1]'}],
-                        ['vswitch', 'del_port', 'int_br0', '#STEP[1][0]'],
-                        ['vswitch', 'del_port', 'int_br0', '#STEP[2][0]'],
-                        ['vswitch', 'del_switch', 'int_br0'],
-                     ]
+        "TestSteps": STEP_VSWITCH_P2P_FLOWS_INIT +
+                     STEP_VSWITCH_P2P_FLOWS_FINIT
     },
     {
-        "Name": "vswitch_throughput",
+        "Name": "vswitch_p2p_tput",
         "Deployment": "clean",
         "Description": "vSwitch - configure switch and execute RFC2544 throughput test",
-        "TestSteps": [
-                        ['vswitch', 'add_switch', 'int_br0'],
-                        ['vswitch', 'add_phy_port', 'int_br0'],
-                        ['vswitch', 'add_phy_port', 'int_br0'],
-                        ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[1][1]', 'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}],
-                        ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[2][1]', 'actions': ['output:#STEP[1][1]'], 'idle_timeout': '0'}],
-                        ['trafficgen', 'send_traffic', {'traffic_type' : 'throughput', 'bidir' : True, 'frame_rate' : 100, 'multistream' : 0, 'stream_type' : 'L4'}],
-                        ['vswitch', 'dump_flows', 'int_br0'],
-                        ['vswitch', 'del_flow', 'int_br0', {'in_port': '#STEP[1][1]'}],
-                        ['vswitch', 'del_flow', 'int_br0', {'in_port': '#STEP[2][1]'}],
-                        ['vswitch', 'del_port', 'int_br0', '#STEP[1][0]'],
-                        ['vswitch', 'del_port', 'int_br0', '#STEP[2][0]'],
-                        ['vswitch', 'del_switch', 'int_br0'],
-                     ]
+        "TestSteps": STEP_VSWITCH_P2P_FLOWS_INIT +
+                     [
+                        ['trafficgen', 'send_traffic', {'traffic_type' : 'throughput', 'bidir' : True}],
+                     ] +
+                     STEP_VSWITCH_P2P_FLOWS_FINIT
     },
     {
-        "Name": "vswitch_back2back",
+        "Name": "vswitch_p2p_back2back",
         "Deployment": "clean",
         "Description": "vSwitch - configure switch and execute RFC2544 back2back test",
-        "TestSteps": [
-                        ['vswitch', 'add_switch', 'int_br0'],
-                        ['vswitch', 'add_phy_port', 'int_br0'],
-                        ['vswitch', 'add_phy_port', 'int_br0'],
-                        ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[1][1]', 'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}],
-                        ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[2][1]', 'actions': ['output:#STEP[1][1]'], 'idle_timeout': '0'}],
-                        ['trafficgen', 'send_traffic', {'traffic_type' : 'back2back', 'bidir' : True, 'frame_rate' : 100, 'multistream' : 0, 'stream_type' : 'L4'}],
-                        ['vswitch', 'dump_flows', 'int_br0'],
-                        ['vswitch', 'del_flow', 'int_br0', {'in_port': '#STEP[1][1]'}],
-                        ['vswitch', 'del_flow', 'int_br0', {'in_port': '#STEP[2][1]'}],
-                        ['vswitch', 'del_port', 'int_br0', '#STEP[1][0]'],
-                        ['vswitch', 'del_port', 'int_br0', '#STEP[2][0]'],
-                        ['vswitch', 'del_switch', 'int_br0'],
-                     ]
+        "TestSteps": STEP_VSWITCH_P2P_FLOWS_INIT +
+                     [
+                        ['trafficgen', 'send_traffic', {'traffic_type' : 'back2back', 'bidir' : True}],
+                     ] +
+                     STEP_VSWITCH_P2P_FLOWS_FINIT
     },
     {
-        "Name": "vswitch_continuous",
+        "Name": "vswitch_p2p_cont",
         "Deployment": "clean",
         "Description": "vSwitch - configure switch and execute continuous stream test",
-        "TestSteps": [
-                        ['vswitch', 'add_switch', 'int_br0'],
-                        ['vswitch', 'add_phy_port', 'int_br0'],
-                        ['vswitch', 'add_phy_port', 'int_br0'],
-                        ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[1][1]', 'actions': ['output:#STEP[2][1]'], 'idle_timeout': '0'}],
-                        ['vswitch', 'add_flow', 'int_br0', {'in_port': '#STEP[2][1]', 'actions': ['output:#STEP[1][1]'], 'idle_timeout': '0'}],
-                        ['trafficgen', 'send_traffic', {'traffic_type' : 'continuous', 'bidir' : True, 'frame_rate' : 100, 'multistream' : 0, 'stream_type' : 'L4'}],
-                        ['vswitch', 'dump_flows', 'int_br0'],
-                        ['vswitch', 'del_flow', 'int_br0', {'in_port': '#STEP[1][1]'}],
-                        ['vswitch', 'del_flow', 'int_br0', {'in_port': '#STEP[2][1]'}],
-                        ['vswitch', 'del_port', 'int_br0', '#STEP[1][0]'],
-                        ['vswitch', 'del_port', 'int_br0', '#STEP[2][0]'],
-                        ['vswitch', 'del_switch', 'int_br0'],
-                     ]
+        "TestSteps": STEP_VSWITCH_P2P_FLOWS_INIT +
+                     [
+                        ['trafficgen', 'send_traffic', {'traffic_type' : 'continuous', 'bidir' : True}],
+                     ] +
+                     STEP_VSWITCH_P2P_FLOWS_FINIT
+    },
+    {
+        "Name": "vswitch_pvp",
+        "Deployment": "clean",
+        "Description": "vSwitch - configure switch and one vnf",
+        "TestSteps": STEP_VSWITCH_PVP_INIT +
+                     [
+                        ['vnf', 'start'],
+                        ['vnf', 'stop'],
+                     ] +
+                     STEP_VSWITCH_PVP_FINIT
+    },
+    {
+        "Name": "vswitch_pvp_tput",
+        "Deployment": "clean",
+        "Description": "vSwitch - configure switch, vnf and execute RFC2544 throughput test",
+        "TestSteps": STEP_VSWITCH_PVP_FLOWS_INIT +
+                     [
+                        ['vnf', 'start'],
+                        ['trafficgen', 'send_traffic', {'traffic_type' : 'throughput', 'bidir' : True}],
+                        ['vnf', 'stop'],
+                     ] +
+                     STEP_VSWITCH_PVP_FLOWS_FINIT
+    },
+    {
+        "Name": "vswitch_pvp_back2back",
+        "Deployment": "clean",
+        "Description": "vSwitch - configure switch, vnf and execute RFC2544 back2back test",
+        "TestSteps": STEP_VSWITCH_PVP_FLOWS_INIT +
+                     [
+                        ['vnf', 'start'],
+                        ['trafficgen', 'send_traffic', {'traffic_type' : 'back2back', 'bidir' : True}],
+                        ['vnf', 'stop'],
+                     ] +
+                     STEP_VSWITCH_PVP_FLOWS_FINIT
+    },
+    {
+        "Name": "vswitch_pvp_cont",
+        "Deployment": "clean",
+        "Description": "vSwitch - configure switch, vnf and execute continuous stream test",
+        "TestSteps": STEP_VSWITCH_PVP_FLOWS_INIT +
+                     [
+                        ['vnf', 'start'],
+                        ['trafficgen', 'send_traffic', {'traffic_type' : 'continuous', 'bidir' : True}],
+                        ['vnf', 'stop'],
+                     ] +
+                     STEP_VSWITCH_PVP_FLOWS_FINIT
+    },
+    {
+        "Name": "vswitch_pvp_all",
+        "Deployment": "clean",
+        "Description": "vSwitch - configure switch, vnf and execute all test types",
+        "TestSteps": STEP_VSWITCH_PVP_FLOWS_INIT +
+                     [
+                        ['vnf', 'start'],
+                        ['trafficgen', 'send_traffic', {'traffic_type' : 'throughput', 'bidir' : True}],
+                        ['trafficgen', 'send_traffic', {'traffic_type' : 'back2back', 'bidir' : True}],
+                        ['trafficgen', 'send_traffic', {'traffic_type' : 'continuous', 'bidir' : True}],
+                        ['vnf', 'stop'],
+                     ] +
+                     STEP_VSWITCH_PVP_FLOWS_FINIT
+    },
+    {
+        "Name": "vswitch_pvvp",
+        "Deployment": "clean",
+        "Description": "vSwitch - configure switch and two vnfs",
+        "TestSteps": STEP_VSWITCH_PVVP_INIT +
+                     [
+                        ['vnf1', 'start'],
+                        ['vnf2', 'start'],
+                        ['vnf1', 'stop'],
+                        ['vnf2', 'stop'],
+                     ] +
+                     STEP_VSWITCH_PVVP_FINIT
+    },
+    {
+        "Name": "vswitch_pvvp_tput",
+        "Deployment": "clean",
+        "Description": "vSwitch - configure switch, two chained vnfs and execute RFC2544 throughput test",
+        "TestSteps": STEP_VSWITCH_PVVP_FLOWS_INIT +
+                     [
+                        ['vnf1', 'start'],
+                        ['vnf2', 'start'],
+                        ['trafficgen', 'send_traffic', {'traffic_type' : 'throughput', 'bidir' : True}],
+                        ['vnf1', 'stop'],
+                        ['vnf2', 'stop'],
+                     ] +
+                     STEP_VSWITCH_PVVP_FLOWS_FINIT
+    },
+    {
+        "Name": "vswitch_pvvp_back2back",
+        "Deployment": "clean",
+        "Description": "vSwitch - configure switch, two chained vnfs and execute RFC2544 back2back test",
+        "TestSteps": STEP_VSWITCH_PVVP_FLOWS_INIT +
+                     [
+                        ['vnf1', 'start'],
+                        ['vnf2', 'start'],
+                        ['trafficgen', 'send_traffic', {'traffic_type' : 'back2back', 'bidir' : True}],
+                        ['vnf1', 'stop'],
+                        ['vnf2', 'stop'],
+                     ] +
+                     STEP_VSWITCH_PVVP_FLOWS_FINIT
+    },
+    {
+        "Name": "vswitch_pvvp_cont",
+        "Deployment": "clean",
+        "Description": "vSwitch - configure switch, two chained vnfs and execute continuous stream test",
+        "TestSteps": STEP_VSWITCH_PVVP_FLOWS_INIT +
+                     [
+                        ['vnf1', 'start'],
+                        ['vnf2', 'start'],
+                        ['trafficgen', 'send_traffic', {'traffic_type' : 'continuous', 'bidir' : True}],
+                        ['vnf1', 'stop'],
+                        ['vnf2', 'stop'],
+                     ] +
+                     STEP_VSWITCH_PVVP_FLOWS_FINIT
+    },
+    {
+        "Name": "vswitch_pvvp_all",
+        "Deployment": "clean",
+        "Description": "vSwitch - configure switch, two chained vnfs and execute all test types",
+        "TestSteps": STEP_VSWITCH_PVVP_FLOWS_INIT +
+                     [
+                        ['vnf1', 'start'],
+                        ['vnf2', 'start'],
+                        ['trafficgen', 'send_traffic', {'traffic_type' : 'throughput', 'bidir' : True}],
+                        ['trafficgen', 'send_traffic', {'traffic_type' : 'back2back', 'bidir' : True}],
+                        ['trafficgen', 'send_traffic', {'traffic_type' : 'continuous', 'bidir' : True}],
+                        ['vnf1', 'stop'],
+                        ['vnf2', 'stop'],
+                     ] +
+                     STEP_VSWITCH_PVVP_FLOWS_FINIT
     },
 ]
 
+# Example of TC definition with exact vSwitch, VNF and TRAFFICGEN values.
+#    {
+#        "Name": "ovs_vanilla_linux_bridge_pvp_cont",
+#        "Deployment": "clean",
+#        "Description": "vSwitch - configure OVS Vanilla, QemuVirtioNet with linux bridge and execute continuous stream test",
+#        "vSwitch" : "OvsVanilla",
+#        "VNF" : "QemuVirtioNet",
+#        "Trafficgen": "IxNet",
+#        "Test Parameters": {"guest_loopback" : "linux_bridge"},
+#        "TestSteps": STEP_VSWITCH_PVP_FLOWS_INIT +
+#                     [
+#                        ['vnf', 'start'],
+#                        ['trafficgen', 'send_traffic', {'traffic_type' : 'continuous', 'bidir' : True}],
+#                        ['vnf', 'stop'],
+#                     ] +
+#                     STEP_VSWITCH_PVP_FLOWS_FINIT
+#    },
diff --git a/src/dpdk/dpdk.py b/src/dpdk/dpdk.py
index 36f1d055..30f228f7 100644
--- a/src/dpdk/dpdk.py
+++ b/src/dpdk/dpdk.py
@@ -30,7 +30,7 @@ from tools.module_manager import ModuleManager
 
 _LOGGER = logging.getLogger(__name__)
 RTE_PCI_TOOL = os.path.join(
-    settings.getValue('RTE_SDK'), 'tools', 'dpdk_nic_bind.py')
+    settings.getValue('RTE_SDK_USER'), 'tools', 'dpdk_nic_bind.py')
 
 _DPDK_MODULE_MANAGER = ModuleManager()
 
diff --git a/src/ovs/daemon.py b/src/ovs/daemon.py
index 09735600..089bc7a4 100644
--- a/src/ovs/daemon.py
+++ b/src/ovs/daemon.py
@@ -24,13 +24,6 @@ import pexpect
 from conf import settings
 from tools import tasks
 
-_OVS_VSWITCHD_BIN = os.path.join(
-    settings.getValue('OVS_DIR'), 'vswitchd', 'ovs-vswitchd')
-_OVSDB_TOOL_BIN = os.path.join(
-    settings.getValue('OVS_DIR'), 'ovsdb', 'ovsdb-tool')
-_OVSDB_SERVER_BIN = os.path.join(
-    settings.getValue('OVS_DIR'), 'ovsdb', 'ovsdb-server')
-
 _OVS_VAR_DIR = settings.getValue('OVS_VAR_DIR')
 _OVS_ETC_DIR = settings.getValue('OVS_ETC_DIR')
 
@@ -60,7 +53,9 @@ class VSwitchd(tasks.Process):
         self._timeout = timeout
         self._expect = expected_cmd
         vswitchd_args = vswitchd_args or []
-        self._cmd = ['sudo', '-E', _OVS_VSWITCHD_BIN] + vswitchd_args
+        ovs_vswitchd_bin = os.path.join(
+            settings.getValue('OVS_DIR'), 'vswitchd', 'ovs-vswitchd')
+        self._cmd = ['sudo', '-E', ovs_vswitchd_bin] + vswitchd_args
 
     # startup/shutdown
 
@@ -118,15 +113,20 @@ class VSwitchd(tasks.Process):
 
         :returns: None
         """
-        tasks.run_task(['sudo', _OVSDB_TOOL_BIN, 'create',
+        ovsdb_tool_bin = os.path.join(
+            settings.getValue('OVS_DIR'), 'ovsdb', 'ovsdb-tool')
+        tasks.run_task(['sudo', ovsdb_tool_bin, 'create',
                         os.path.join(_OVS_ETC_DIR, 'conf.db'),
                         os.path.join(settings.getValue('OVS_DIR'), 'vswitchd',
                                      'vswitch.ovsschema')],
                        self._logger,
                        'Creating ovsdb configuration database...')
 
+        ovsdb_server_bin = os.path.join(
+            settings.getValue('OVS_DIR'), 'ovsdb', 'ovsdb-server')
+
         tasks.run_background_task(
-            ['sudo', _OVSDB_SERVER_BIN,
+            ['sudo', ovsdb_server_bin,
              '--remote=punix:%s' % os.path.join(_OVS_VAR_DIR, 'db.sock'),
              '--remote=db:Open_vSwitch,Open_vSwitch,manager_options',
              '--pidfile=' + self._ovsdb_pidfile_path, '--overwrite-pidfile'],
diff --git a/testcases/integration.py b/testcases/integration.py
index 53ba17f4..9733c263 100644
--- a/testcases/integration.py
+++ b/testcases/integration.py
@@ -17,10 +17,12 @@
 import os
 import time
 import logging
+import copy
 
 from testcases import TestCase
 from conf import settings as S
 from collections import OrderedDict
+from core.loader import Loader
 
 CHECK_PREFIX = 'validate_'
 
@@ -39,7 +41,7 @@ class IntegrationTestCase(TestCase):
     def report_status(self, label, status):
         """ Log status of test step
         """
-        self._logger.debug("%s ... %s", label, 'OK' if status else 'FAILED')
+        self._logger.info("%s ... %s", label, 'OK' if status else 'FAILED')
 
     def run_initialize(self):
         """ Prepare test execution environment
@@ -104,6 +106,8 @@ class IntegrationTestCase(TestCase):
                         if not self.test:
                             self._traffic_ctl.send_traffic(self._traffic)
                         else:
+                            vnf_list = {}
+                            loader = Loader()
                             # execute test based on TestSteps definition
                             if self.test:
                                 step_result = [None] * len(self.test)
@@ -113,6 +117,18 @@ class IntegrationTestCase(TestCase):
                                         test_object = self._vswitch_ctl.get_vswitch()
                                     elif step[0] == 'trafficgen':
                                         test_object = self._traffic_ctl
+                                        # in case of send_traffic method, ensure that specified
+                                        # traffic values are merged with existing self._traffic
+                                        if step[1] == 'send_traffic':
+                                            tmp_traffic = copy.deepcopy(self._traffic)
+                                            tmp_traffic.update(step[2])
+                                            step[2] = tmp_traffic
+                                    elif step[0].startswith('vnf'):
+                                        if not step[0] in vnf_list:
+                                            # initialize new VM and copy data to its shared dir
+                                            vnf_list[step[0]] = loader.get_vnf_class()()
+                                            self._copy_fwd_tools_for_guest(len(vnf_list))
+                                        test_object = vnf_list[step[0]]
                                     else:
                                         self._logger.error("Unsupported test object %s", step[0])
                                         self._inttest = {'status' : False, 'details' : ' '.join(step)}
@@ -130,23 +146,32 @@ class IntegrationTestCase(TestCase):
                                             step_params = eval_step_params(step[2:], step_result)
                                             step_log = '{} {}'.format(' '.join(step[:2]), step_params)
                                             step_result[i] = test_method(*step_params)
-                                            self._logger.debug("Step {} '{}' results '{}'".format(
-                                                i, step_log, step_result[i]))
-                                            time.sleep(2)
+                                            self._logger.debug("Step %s '%s' results '%s'", i,
+                                                               step_log, step_result[i])
+                                            time.sleep(5)
                                             step_ok = test_method_check(step_result[i], *step_params)
                                         except AssertionError:
                                             self._inttest = {'status' : False, 'details' : step_log}
-                                            self._logger.error("Step {} raised assertion error".format(i))
+                                            self._logger.error("Step %s raised assertion error", i)
+                                            # stop vnfs in case of error
+                                            for vnf in vnf_list:
+                                                vnf_list[vnf].stop()
                                             break
                                         except IndexError:
                                             self._inttest = {'status' : False, 'details' : step_log}
-                                            self._logger.error("Step {} result index error {}".format(
-                                                i, ' '.join(step[2:])))
+                                            self._logger.error("Step %s result index error %s", i,
+                                                               ' '.join(step[2:]))
+                                            # stop vnfs in case of error
+                                            for vnf in vnf_list:
+                                                vnf_list[vnf].stop()
                                             break
 
                                     self.report_status("Step {} - '{}'".format(i, step_log), step_ok)
                                     if not step_ok:
                                         self._inttest = {'status' : False, 'details' : step_log}
+                                        # stop vnfs in case of error
+                                        for vnf in vnf_list:
+                                            vnf_list[vnf].stop()
                                         break
 
                     # dump vswitch flows before they are affected by VNF termination
diff --git a/testcases/testcase.py b/testcases/testcase.py
index 7c935792..f7908af9 100644
--- a/testcases/testcase.py
+++ b/testcases/testcase.py
@@ -27,6 +27,7 @@ from core.loader import Loader
 from core.results.results_constants import ResultsConstants
 from tools import tasks
 from tools import hugepages
+from tools import functions
 from tools.pkt_gen.trafficgen.trafficgenhelper import TRAFFIC_DEFAULTS
 from conf import settings as S
 from conf import get_test_param
@@ -52,6 +53,26 @@ class TestCase(object):
         self._loadgen = None
         self._output_file = None
         self._tc_results = None
+        self.guest_loopback = []
+        self._settings_original = {}
+        self._settings_paths_modified = False
+
+        self._update_settings('VSWITCH', cfg.get('vSwitch', S.getValue('VSWITCH')))
+        self._update_settings('VNF', cfg.get('VNF', S.getValue('VNF')))
+        self._update_settings('TRAFFICGEN', cfg.get('Trafficgen', S.getValue('TRAFFICGEN')))
+        self._update_settings('TEST_PARAMS', cfg.get('Parameters', S.getValue('TEST_PARAMS')))
+
+        # update global settings
+        guest_loopback = get_test_param('guest_loopback', None)
+        if guest_loopback:
+            self._update_settings('GUEST_LOOPBACK', [guest_loopback for dummy in S.getValue('GUEST_LOOPBACK')])
+
+        if 'VSWITCH' in self._settings_original or 'VNF' in self._settings_original:
+            self._settings_original.update({
+                'RTE_SDK' : S.getValue('RTE_SDK'),
+                'OVS_DIR' : S.getValue('OVS_DIR'),
+            })
+            functions.settings_update_paths()
 
         # set test parameters; CLI options take precedence to testcase settings
         self._logger = logging.getLogger(__name__)
@@ -82,18 +103,11 @@ class TestCase(object):
                 self._tunnel_type = get_test_param('tunnel_type',
                                                    self._tunnel_type)
 
-
         # identify guest loopback method, so it can be added into reports
-        self.guest_loopback = []
-        if self.deployment in ['pvp', 'pvvp']:
-            guest_loopback = get_test_param('guest_loopback', None)
-            if guest_loopback:
-                self.guest_loopback.append(guest_loopback)
-            else:
-                if self.deployment == 'pvp':
-                    self.guest_loopback.append(S.getValue('GUEST_LOOPBACK')[0])
-                else:
-                    self.guest_loopback = S.getValue('GUEST_LOOPBACK').copy()
+        if self.deployment == 'pvp':
+            self.guest_loopback.append(S.getValue('GUEST_LOOPBACK')[0])
+        else:
+            self.guest_loopback = S.getValue('GUEST_LOOPBACK').copy()
 
         # read configuration of streams; CLI parameter takes precedence to
         # testcase definition
@@ -127,6 +141,9 @@ class TestCase(object):
                               'pre_installed_flows' : pre_installed_flows,
                               'frame_rate': int(framerate)})
 
+        # Packet Forwarding mode
+        self._vswitch_none = 'none' == S.getValue('VSWITCH').strip().lower()
+
         # OVS Vanilla requires guest VM MAC address and IPs to work
         if 'linux_bridge' in self.guest_loopback:
             self._traffic['l2'].update({'srcmac': S.getValue('GUEST_NET2_MAC')[0],
@@ -134,20 +151,7 @@ class TestCase(object):
             self._traffic['l3'].update({'srcip': S.getValue('VANILLA_TGEN_PORT1_IP'),
                                         'dstip': S.getValue('VANILLA_TGEN_PORT2_IP')})
 
-        # Packet Forwarding mode
-        self._vswitch_none = 'none' == S.getValue('VSWITCH').strip().lower()
-
-    def run_initialize(self):
-        """ Prepare test execution environment
-        """
-        self._logger.debug(self.name)
-
-        # mount hugepages if needed
-        self._mount_hugepages()
-
-        # copy sources of l2 forwarding tools into VM shared dir if needed
-        self._copy_fwd_tools_for_guest()
-
+        # trafficgen configuration required for tests of tunneling protocols
         if self.deployment == "op2p":
             self._traffic['l2'].update({'srcmac':
                                         S.getValue('TRAFFICGEN_PORT1_MAC'),
@@ -171,7 +175,16 @@ class TestCase(object):
             else:
                 self._logger.debug("MAC addresses can not be read")
 
+    def run_initialize(self):
+        """ Prepare test execution environment
+        """
+        self._logger.debug(self.name)
+
+        # mount hugepages if needed
+        self._mount_hugepages()
 
+        # copy sources of l2 forwarding tools into VM shared dir if needed
+        self._copy_fwd_tools_for_all_guests()
 
         self._logger.debug("Controllers:")
         loader = Loader()
@@ -212,6 +225,9 @@ class TestCase(object):
         # umount hugepages if mounted
         self._umount_hugepages()
 
+        # restore original settings
+        S.load_from_dict(self._settings_original)
+
     def run_report(self):
         """ Report test results
         """
@@ -267,6 +283,19 @@ class TestCase(object):
         # report test results
         self.run_report()
 
+    def _update_settings(self, param, value):
+        """ Check value of given configuration parameter
+        In case that new value is different, then testcase
+        specific settings is updated and original value stored
+
+        :param param: Name of parameter inside settings
+        :param value: Disired parameter value
+        """
+        orig_value = S.getValue(param)
+        if orig_value != value:
+            self._settings_original[param] = orig_value
+            S.setValue(param, value)
+
     def _append_results(self, results):
         """
         Method appends mandatory Test Case results to list of dictionaries.
@@ -284,50 +313,55 @@ class TestCase(object):
                 item[ResultsConstants.SCAL_STREAM_COUNT] = self._traffic['multistream']
                 item[ResultsConstants.SCAL_STREAM_TYPE] = self._traffic['stream_type']
                 item[ResultsConstants.SCAL_PRE_INSTALLED_FLOWS] = self._traffic['pre_installed_flows']
-            if len(self.guest_loopback):
+            if self.deployment in ['pvp', 'pvvp'] and len(self.guest_loopback):
                 item[ResultsConstants.GUEST_LOOPBACK] = ' '.join(self.guest_loopback)
             if self._tunnel_type:
                 item[ResultsConstants.TUNNEL_TYPE] = self._tunnel_type
         return results
 
-    def _copy_fwd_tools_for_guest(self):
-        """Copy dpdk and l2fwd code to GUEST_SHARE_DIR[s] for use by guests.
+    def _copy_fwd_tools_for_all_guests(self):
+        """Copy dpdk and l2fwd code to GUEST_SHARE_DIR[s] based on selected deployment.
         """
-        counter = 0
-        # method is executed only for pvp and pvvp, so let's count number of 'v'
-        while counter < self.deployment.count('v'):
-            guest_dir = S.getValue('GUEST_SHARE_DIR')[counter]
-
-            # remove shared dir if it exists to avoid issues with file consistency
-            if os.path.exists(guest_dir):
-                tasks.run_task(['rm', '-f', '-r', guest_dir], self._logger,
-                               'Removing content of shared directory...', True)
-
-            # directory to share files between host and guest
-            os.makedirs(guest_dir)
-
-            # copy sources into shared dir only if neccessary
-            if 'testpmd' in self.guest_loopback or 'l2fwd' in self.guest_loopback:
-                try:
-                    # always use DPDK vhost user version inside VM, so results are not
-                    # affected by different testpmd behavior inside VM
-                    tasks.run_task(['rsync', '-a', '-r', '-l', r'--exclude="\.git"',
-                                    os.path.join(S.getValue('RTE_SDK_USER'), ''),
-                                    os.path.join(guest_dir, 'DPDK')],
-                                   self._logger,
-                                   'Copying DPDK to shared directory...',
-                                   True)
-                    tasks.run_task(['rsync', '-a', '-r', '-l',
-                                    os.path.join(S.getValue('ROOT_DIR'), 'src/l2fwd/'),
-                                    os.path.join(guest_dir, 'l2fwd')],
-                                   self._logger,
-                                   'Copying l2fwd to shared directory...',
-                                   True)
-                except subprocess.CalledProcessError:
-                    self._logger.error('Unable to copy DPDK and l2fwd to shared directory')
-
+        # data are copied only for pvp and pvvp, so let's count number of 'v'
+        counter = 1
+        while counter <= self.deployment.count('v'):
+            self._copy_fwd_tools_for_guest(counter)
             counter += 1
 
+    def _copy_fwd_tools_for_guest(self, index):
+        """Copy dpdk and l2fwd code to GUEST_SHARE_DIR of VM
+
+        :param index: Index of VM starting from 1 (i.e. 1st VM has index 1)
+        """
+        guest_dir = S.getValue('GUEST_SHARE_DIR')[index-1]
+
+        # remove shared dir if it exists to avoid issues with file consistency
+        if os.path.exists(guest_dir):
+            tasks.run_task(['rm', '-f', '-r', guest_dir], self._logger,
+                           'Removing content of shared directory...', True)
+
+        # directory to share files between host and guest
+        os.makedirs(guest_dir)
+
+        # copy sources into shared dir only if neccessary
+        if 'testpmd' in self.guest_loopback or 'l2fwd' in self.guest_loopback:
+            try:
+                tasks.run_task(['rsync', '-a', '-r', '-l', r'--exclude="\.git"',
+                                os.path.join(S.getValue('RTE_SDK'), ''),
+                                os.path.join(guest_dir, 'DPDK')],
+                               self._logger,
+                               'Copying DPDK to shared directory...',
+                               True)
+                tasks.run_task(['rsync', '-a', '-r', '-l',
+                                os.path.join(S.getValue('ROOT_DIR'), 'src/l2fwd/'),
+                                os.path.join(guest_dir, 'l2fwd')],
+                               self._logger,
+                               'Copying l2fwd to shared directory...',
+                               True)
+            except subprocess.CalledProcessError:
+                self._logger.error('Unable to copy DPDK and l2fwd to shared directory')
+
+
     def _mount_hugepages(self):
         """Mount hugepages if usage of DPDK or Qemu is detected
         """
@@ -335,7 +369,8 @@ class TestCase(object):
         if not self._hugepages_mounted and \
             (self.deployment.count('v') or \
              S.getValue('VSWITCH').lower().count('dpdk') or \
-             self._vswitch_none):
+             self._vswitch_none or \
+             self.test and 'vnf' in [step[0][0:3] for step in self.test]):
             hugepages.mount_hugepages()
             self._hugepages_mounted = True
 
diff --git a/tools/functions.py b/tools/functions.py
new file mode 100644
index 00000000..5079a9f0
--- /dev/null
+++ b/tools/functions.py
@@ -0,0 +1,43 @@
+# Copyright 2016 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Various helper functions
+"""
+
+from conf import settings
+
+#
+# Support functions
+#
+
+def settings_update_paths():
+    """ Configure paths to OVS and DPDK based on VSWITCH and VNF values
+    """
+    # set dpdk and ovs paths accorfing to VNF and VSWITCH
+    if settings.getValue('VSWITCH').endswith('Vanilla'):
+        # settings paths for Vanilla
+        settings.setValue('OVS_DIR', (settings.getValue('OVS_DIR_VANILLA')))
+    elif settings.getValue('VSWITCH').endswith('Vhost'):
+        if settings.getValue('VNF').endswith('Cuse'):
+            # settings paths for Cuse
+            settings.setValue('RTE_SDK', (settings.getValue('RTE_SDK_CUSE')))
+            settings.setValue('OVS_DIR', (settings.getValue('OVS_DIR_CUSE')))
+        else:
+            # settings paths for VhostUser
+            settings.setValue('RTE_SDK', (settings.getValue('RTE_SDK_USER')))
+            settings.setValue('OVS_DIR', (settings.getValue('OVS_DIR_USER')))
+    else:
+        # default - set to VHOST USER but can be changed during enhancement
+        settings.setValue('RTE_SDK', (settings.getValue('RTE_SDK_USER')))
+        settings.setValue('OVS_DIR', (settings.getValue('OVS_DIR_USER')))
diff --git a/vnfs/vnf/vnf.py b/vnfs/vnf/vnf.py
index 483faf38..3dae2733 100644
--- a/vnfs/vnf/vnf.py
+++ b/vnfs/vnf/vnf.py
@@ -122,6 +122,19 @@ class IVnf(tasks.Process):
         self.execute(cmd)
         self.wait(prompt=prompt, timeout=timeout)
 
+    def validate_start(self, dummy_result):
+        """ Validate call of VNF start()
+        """
+        if self._child and self._child.isalive():
+            return True
+        else:
+            return False
+
+    def validate_stop(self, result):
+        """ Validate call of fVNF stop()
+        """
+        return not self.validate_start(result)
+
     @staticmethod
     def reset_vnf_counter():
         """
diff --git a/vsperf b/vsperf
index 57d68990..98bc7db0 100755
--- a/vsperf
+++ b/vsperf
@@ -40,6 +40,7 @@ from testcases import PerformanceTestCase
 from testcases import IntegrationTestCase
 from tools import tasks
 from tools import networkcard
+from tools import functions
 from tools.pkt_gen import trafficgen
 from tools.opnfvdashboard import opnfvdashboard
 from tools.pkt_gen.trafficgen.trafficgenhelper import TRAFFIC_DEFAULTS
@@ -157,7 +158,8 @@ def parse_arguments():
     group.add_argument('-d', '--test-dir', help='directory containing tests')
     group.add_argument('-t', '--tests', help='Comma-separated list of terms \
             indicating tests to run. e.g. "RFC2544,!p2p" - run all tests whose\
-            name contains RFC2544 less those containing "p2p"')
+            name contains RFC2544 less those containing "p2p"; "!back2back" - \
+            run all tests except those containing back2back')
     group.add_argument('--verbosity', choices=list_logging_levels(),
                        help='debug level')
     group.add_argument('--integration', action='store_true', help='execute integration tests')
@@ -244,7 +246,11 @@ def apply_filter(tests, tc_filter):
         e.g. '' - empty string selects all tests.
     :return: A list of the selected Tests.
     """
-    result = []
+    # if negative filter is first we have to start with full list of tests
+    if tc_filter.strip()[0] == '!':
+        result = tests
+    else:
+        result = []
     if tc_filter is None:
         tc_filter = ""
 
@@ -252,11 +258,11 @@ def apply_filter(tests, tc_filter):
         if not term or term[0] != '!':
             # Add matching tests from 'tests' into results
             result.extend([test for test in tests \
-                if test.name.lower().find(term) >= 0])
+                if test['Name'].lower().find(term) >= 0])
         else:
             # Term begins with '!' so we remove matching tests
             result = [test for test in result \
-                if test.name.lower().find(term[1:]) < 0]
+                if test['Name'].lower().find(term[1:]) < 0]
 
     return result
 
@@ -496,26 +502,8 @@ def main():
     # than both a settings file and environment variables
     settings.load_from_dict(args)
 
-    vswitch_none = False
     # set dpdk and ovs paths accorfing to VNF and VSWITCH
-    if settings.getValue('VSWITCH').endswith('Vanilla'):
-        # settings paths for Vanilla
-        settings.setValue('OVS_DIR', (settings.getValue('OVS_DIR_VANILLA')))
-    elif settings.getValue('VSWITCH').endswith('Vhost'):
-        if settings.getValue('VNF').endswith('Cuse'):
-            # settings paths for Cuse
-            settings.setValue('RTE_SDK', (settings.getValue('RTE_SDK_CUSE')))
-            settings.setValue('OVS_DIR', (settings.getValue('OVS_DIR_CUSE')))
-        else:
-            # settings paths for VhostUser
-            settings.setValue('RTE_SDK', (settings.getValue('RTE_SDK_USER')))
-            settings.setValue('OVS_DIR', (settings.getValue('OVS_DIR_USER')))
-    else:
-        # default - set to VHOST USER but can be changed during enhancement
-        settings.setValue('RTE_SDK', (settings.getValue('RTE_SDK_USER')))
-        settings.setValue('OVS_DIR', (settings.getValue('OVS_DIR_USER')))
-        if 'none' == settings.getValue('VSWITCH').strip().lower():
-            vswitch_none = True
+    functions.settings_update_paths()
 
     # if required, handle list-* operations
     handle_list_options(args)
@@ -641,46 +629,37 @@ def main():
         else:
             testcases = settings.getValue('PERFORMANCE_TESTS')
 
-        all_tests = []
-        for cfg in testcases:
-            try:
-                if args['integration']:
-                    all_tests.append(IntegrationTestCase(cfg))
-                else:
-                    all_tests.append(PerformanceTestCase(cfg))
-            except (Exception) as _:
-                _LOGGER.exception("Failed to create test: %s",
-                                  cfg.get('Name', '<Name not set>'))
-                vsperf_finalize()
-                raise
-
-        # select requested tests
         if args['exact_test_name']:
             exact_names = args['exact_test_name']
             # positional args => exact matches only
-            selected_tests = [test for test in all_tests if test.name in exact_names]
+            selected_tests = [test for test in testcases if test['Name'] in exact_names]
         elif args['tests']:
             # --tests => apply filter to select requested tests
-            selected_tests = apply_filter(all_tests, args['tests'])
+            selected_tests = apply_filter(testcases, args['tests'])
         else:
             # Default - run all tests
-            selected_tests = all_tests
+            selected_tests = testcases
 
-        if not selected_tests:
-            _LOGGER.error("No tests matched --test option or positional args. Done.")
+        if not len(selected_tests):
+            _LOGGER.error("No tests matched --tests option or positional args. Done.")
             vsperf_finalize()
             sys.exit(1)
 
         # run tests
         suite = unittest.TestSuite()
-        for test in selected_tests:
+        for cfg in selected_tests:
+            test_name = cfg.get('Name', '<Name not set>')
             try:
+                if args['integration']:
+                    test = IntegrationTestCase(cfg)
+                else:
+                    test = PerformanceTestCase(cfg)
                 test.run()
                 suite.addTest(MockTestCase('', True, test.name))
             #pylint: disable=broad-except
             except (Exception) as ex:
-                _LOGGER.exception("Failed to run test: %s", test.name)
-                suite.addTest(MockTestCase(str(ex), False, test.name))
+                _LOGGER.exception("Failed to run test: %s", test_name)
+                suite.addTest(MockTestCase(str(ex), False, test_name))
                 _LOGGER.info("Continuing with next test...")
 
         # generate final rst report with results of all executed TCs