teststeps: Generic support of step driven tests

In the past, step driven testcases were supported
only by integration testcases. This patch adds generic
support of TestSteps for both integration and performance
testcases. Step driven test were improved to support
modification of existing deployment. As part of
the patch a refactoring of traffic controllers
were performed. Traffic controllers were modified
to support trafficgen-off and trafficgen-pause
modes in all possible ways of trafficgen invocation.

JIRA: VSPERF-362

Change-Id: Ic8b7a9b0e7165f0a15a52279ed0f0952da9fedb8
Signed-off-by: Martin Klozik <martinx.klozik@intel.com>
Reviewed-by: Maryam Tahhan <maryam.tahhan@intel.com>
Reviewed-by: Al Morton <acmorton@att.com>
Reviewed-by: Christian Trautman <ctrautma@redhat.com>
Reviewed-by: Bill Michalowski <bmichalo@redhat.com>
Reviewed-by: Antonio Fischetti <antonio.fischetti@intel.com>
Reviewed-by: Sridhar K. N. Rao <sridhar.rao@spirent.com>

6 files changed, 181 insertions, 196 deletions

diff --git a/core/__init__.py b/core/__init__.py
index e39ec88e..d16401ea 100644
--- a/core/__init__.py
+++ b/core/__init__.py
@@ -15,5 +15,5 @@
 """Core structural interfaces and their implementations
 """
 import core.component_factory
-from core.traffic_controller import (ITrafficController)
+from core.traffic_controller import (TrafficController)
 from core.vswitch_controller import (IVswitchController)
diff --git a/core/component_factory.py b/core/component_factory.py
index ef7ba86f..236a61ed 100644
--- a/core/component_factory.py
+++ b/core/component_factory.py
@@ -46,7 +46,7 @@ def create_traffic(traffic_type, trafficgen_class):
 
     :param traffic_type: Name of traffic type
     :param trafficgen_class: Reference to traffic generator class to be used.
-    :return: A new ITrafficController
+    :return: A new TrafficController
     """
     if traffic_type.lower().startswith('rfc2889'):
         return TrafficControllerRFC2889(trafficgen_class)
@@ -87,7 +87,7 @@ def create_vswitch(deployment_scenario, vswitch_class, traffic,
         raise RuntimeError("Unknown deployment scenario '{}'.".format(deployment_scenario))
 
 
-def create_vnf(deployment_scenario, vnf_class):
+def create_vnf(deployment_scenario, vnf_class, extra_vnfs):
     """Return a new VnfController for the deployment_scenario.
 
     The returned controller is configured with the given VNF class.
@@ -96,9 +96,13 @@ def create_vnf(deployment_scenario, vnf_class):
 
     :param deployment_scenario: The deployment scenario name
     :param vswitch_class: Reference to vSwitch class to be used.
+    :param extra_vnfs: The number of VNFs not involved in given
+        deployment scenario. It will be used to correctly expand
+        configuration values and initialize shared dirs. This parameter
+        is used in case, that additional VNFs are executed by TestSteps.
     :return: VnfController for the deployment_scenario
     """
-    return VnfController(deployment_scenario, vnf_class)
+    return VnfController(deployment_scenario, vnf_class, extra_vnfs)
 
 def create_collector(collector_class, result_dir, test_name):
     """Return a new Collector of the given class
diff --git a/core/traffic_controller.py b/core/traffic_controller.py
index 428e91f8..a6c1bd8d 100644
--- a/core/traffic_controller.py
+++ b/core/traffic_controller.py
@@ -1,4 +1,4 @@
-# Copyright 2015 Intel Corporation.
+# Copyright 2015-2016 Intel Corporation.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,17 +12,105 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Interface to traffic controllers
+"""Base class for traffic controllers
 """
 
-class ITrafficController(object):
-    """Abstract class which defines a traffic controller object
+import logging
+import os
+import time
+
+from core.results.results_constants import ResultsConstants
+from conf import settings
+from conf import get_test_param
+
+class TrafficController(object):
+    """Base class which defines a common functionality for all traffic
+       controller classes.
 
     Used to setup and control a traffic generator for a particular deployment
     scenario.
     """
+    def __init__(self, traffic_gen_class):
+        """Initialization common for all types of traffic controllers
+
+        :param traffic_gen_class: The traffic generator class to be used.
+        """
+        self._type = None
+        self._logger = logging.getLogger(__name__)
+        self._logger.debug("__init__")
+        self._traffic_gen_class = traffic_gen_class()
+        self._traffic_started = False
+        self._traffic_started_call_count = 0
+        self._duration = int(get_test_param('duration', 30))
+        self._lossrate = float(get_test_param('lossrate', 0.0))
+        self._mode = settings.getValue('mode').lower()
+        self._results = []
+
+        # If set, comma separated packet_sizes value from --test_params
+        # on cli takes precedence over value in settings file.
+        self._packet_sizes = None
+        packet_sizes_cli = get_test_param('pkt_sizes')
+        if packet_sizes_cli:
+            self._packet_sizes = [int(x.strip())
+                                  for x in packet_sizes_cli.split(',')]
+        else:
+            self._packet_sizes = settings.getValue('TRAFFICGEN_PKT_SIZES')
+
+    def __enter__(self):
+        """Call initialisation function.
+        """
+        self._traffic_gen_class.connect()
+
+    def __exit__(self, type_, value, traceback):
+        """Stop traffic, clean up.
+        """
+        if self._traffic_started:
+            self.stop_traffic()
+
+    def _append_results(self, result_dict, packet_size):
+        """Adds common values to traffic generator results.
+
+        :param result_dict: Dictionary containing results from trafficgen
+        :param packet_size: Packet size value.
+
+        :returns: dictionary of results with additional entries.
+        """
+
+        ret_value = result_dict
 
-    def send_traffic(self, traffic):
+        ret_value[ResultsConstants.TYPE] = self._type
+        ret_value[ResultsConstants.PACKET_SIZE] = str(packet_size)
+
+        return ret_value
+
+    def traffic_required(self):
+        """Checks selected '--mode' of traffic generator and performs
+           its specific handling.
+
+        :returns: True - in case that traffic generator should be executed
+                  False - if traffic generation is not required
+        """
+        if self._mode == 'trafficgen-off':
+            time.sleep(2)
+            self._logger.debug("All is set. Please run traffic generator manually.")
+            input(os.linesep + "Press Enter to terminate vswitchperf..." +
+                  os.linesep + os.linesep)
+            return False
+        elif self._mode == 'trafficgen-pause':
+            time.sleep(2)
+            while True:
+                choice = input(os.linesep + 'Transmission paused, should'
+                               ' transmission be resumed? [y/n]' + os.linesep).lower()
+                if choice in ('yes', 'y', 'ye'):
+                    return True
+                elif choice in ('no', 'n'):
+                    self._logger.info("Traffic transmission will be skipped.")
+                    return False
+                else:
+                    print("Please respond with 'yes', 'y', 'no' or 'n' ", end='')
+        return True
+
+    def send_traffic(self, dummy_traffic):
         """Triggers traffic to be sent from the traffic generator.
 
         This is a blocking function.
@@ -33,7 +121,7 @@ class ITrafficController(object):
             "The TrafficController does not implement",
             "the \"send_traffic\" function.")
 
-    def send_traffic_async(self, traffic, function):
+    def send_traffic_async(self, dummy_traffic, dummy_function):
         """Triggers traffic to be sent  asynchronously.
 
         This is not a blocking function.
@@ -55,6 +143,33 @@ class ITrafficController(object):
     def stop_traffic(self):
         """Kills traffic being sent from the traffic generator.
         """
-        raise NotImplementedError(
-            "The TrafficController does not implement",
-            "the \"stop_traffic\" function.")
+        self._logger.debug("stop_traffic()")
+
+    def print_results(self):
+        """IResult interface implementation.
+        """
+        counter = 0
+        for item in self._results:
+            logging.info("Record: " + str(counter))
+            counter += 1
+            for(key, value) in list(item.items()):
+                logging.info("         Key: " + str(key) +
+                             ", Value: " + str(value))
+
+    def get_results(self):
+        """IResult interface implementation.
+        """
+        return self._results
+
+    def validate_send_traffic(self, dummy_result, dummy_traffic):
+        """Verify that send traffic has succeeded
+        """
+        if len(self._results):
+            if 'b2b_frames' in self._results[-1]:
+                return float(self._results[-1]['b2b_frames']) > 0
+            elif 'throughput_rx_fps' in self._results[-1]:
+                return float(self._results[-1]['throughput_rx_fps']) > 0
+            else:
+                return True
+        else:
+            return False
diff --git a/core/traffic_controller_rfc2544.py b/core/traffic_controller_rfc2544.py
index af09deff..874c3ae7 100644
--- a/core/traffic_controller_rfc2544.py
+++ b/core/traffic_controller_rfc2544.py
@@ -1,4 +1,4 @@
-# Copyright 2015 Intel Corporation.
+# Copyright 2015-2016 Intel Corporation.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,16 +13,12 @@
 # limitations under the License.
 """RFC2544 Traffic Controller implementation.
 """
-import logging
-
-from core.traffic_controller import ITrafficController
-from core.results.results_constants import ResultsConstants
+from core.traffic_controller import TrafficController
 from core.results.results import IResults
-from conf import settings
 from conf import get_test_param
 
 
-class TrafficControllerRFC2544(ITrafficController, IResults):
+class TrafficControllerRFC2544(TrafficController, IResults):
     """Traffic controller for RFC2544 traffic
 
     Used to setup and control a traffic generator for an RFC2544 deployment
@@ -34,60 +30,15 @@ class TrafficControllerRFC2544(ITrafficController, IResults):
 
         :param traffic_gen_class: The traffic generator class to be used.
         """
-        self._logger = logging.getLogger(__name__)
-        self._logger.debug("__init__")
-        self._traffic_gen_class = traffic_gen_class()
-        self._traffic_started = False
-        self._traffic_started_call_count = 0
+        super(TrafficControllerRFC2544, self).__init__(traffic_gen_class)
+        self._type = 'rfc2544'
         self._tests = int(get_test_param('rfc2544_tests', 1))
-        self._duration = int(get_test_param('duration', 30))
-        self._lossrate = float(get_test_param('lossrate', 0.0))
-        self._results = []
-
-        # If set, comma separated packet_sizes value from --test_params
-        # on cli takes precedence over value in settings file.
-        self._packet_sizes = None
-        packet_sizes_cli = get_test_param('pkt_sizes')
-        if packet_sizes_cli:
-            self._packet_sizes = [int(x.strip())
-                                  for x in packet_sizes_cli.split(',')]
-        else:
-            self._packet_sizes = settings.getValue('TRAFFICGEN_PKT_SIZES')
-
-    def __enter__(self):
-        """Call initialisation function.
-        """
-        self._traffic_gen_class.connect()
-
-    def __exit__(self, type_, value, traceback):
-        """Stop traffic, clean up.
-        """
-        if self._traffic_started:
-            self.stop_traffic()
-
-    @staticmethod
-    def _append_results(result_dict, packet_size):
-        """Adds common values to traffic generator results.
-
-        :param result_dict: Dictionary containing results from trafficgen
-        :param packet_size: Packet size value.
-
-        :returns: dictionary of results with additional entries.
-        """
-
-        ret_value = result_dict
-
-        # TODO Old TOIT controller had knowledge about scenario beeing
-        # executed, should new controller also fill Configuration & ID,
-        # or this should be passed to TestCase?
-        ret_value[ResultsConstants.TYPE] = 'rfc2544'
-        ret_value[ResultsConstants.PACKET_SIZE] = str(packet_size)
-
-        return ret_value
 
     def send_traffic(self, traffic):
-        """See ITrafficController for description
+        """See TrafficController for description
         """
+        if not self.traffic_required():
+            return
         self._logger.debug('send_traffic with ' +
                            str(self._traffic_gen_class))
 
@@ -109,13 +60,14 @@ class TrafficControllerRFC2544(ITrafficController, IResults):
                 result = self._traffic_gen_class.send_rfc2544_throughput(
                     traffic, tests=self._tests, duration=self._duration, lossrate=self._lossrate)
 
-            result = TrafficControllerRFC2544._append_results(result,
-                                                              packet_size)
+            result = self._append_results(result, packet_size)
             self._results.append(result)
 
     def send_traffic_async(self, traffic, function):
-        """See ITrafficController for description
+        """See TrafficController for description
         """
+        if not self.traffic_required():
+            return
         self._logger.debug('send_traffic_async with ' +
                            str(self._traffic_gen_class))
 
@@ -131,40 +83,6 @@ class TrafficControllerRFC2544(ITrafficController, IResults):
             else:
                 function['function']()
             result = self._traffic_gen_class.wait_rfc2544_throughput()
-            result = TrafficControllerRFC2544._append_results(result,
-                                                              packet_size)
+            result = self._append_results(result, packet_size)
             self._results.append(result)
 
-    def stop_traffic(self):
-        """Kills traffic being sent from the traffic generator.
-        """
-        self._logger.debug("stop_traffic()")
-
-    def print_results(self):
-        """IResult interface implementation.
-        """
-        counter = 0
-        for item in self._results:
-            logging.info("Record: " + str(counter))
-            counter += 1
-            for(key, value) in list(item.items()):
-                logging.info("         Key: " + str(key) +
-                             ", Value: " + str(value))
-
-    def get_results(self):
-        """IResult interface implementation.
-        """
-        return self._results
-
-    def validate_send_traffic(self, dummy_result, dummy_traffic):
-        """Verify that send traffic has succeeded
-        """
-        if len(self._results):
-            if 'b2b_frames' in self._results[-1]:
-                return float(self._results[-1]['b2b_frames']) > 0
-            elif 'throughput_rx_fps' in self._results[-1]:
-                return float(self._results[-1]['throughput_rx_fps']) > 0
-            else:
-                return True
-        else:
-            return False
diff --git a/core/traffic_controller_rfc2889.py b/core/traffic_controller_rfc2889.py
index a97a47d3..a2e12e6d 100644
--- a/core/traffic_controller_rfc2889.py
+++ b/core/traffic_controller_rfc2889.py
@@ -1,15 +1,24 @@
+# Copyright 2016 Spirent Communications, Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 """RFC2889 Traffic Controller implementation.
 """
-import logging
-
-from core.traffic_controller import ITrafficController
-from core.results.results_constants import ResultsConstants
+from core.traffic_controller import TrafficController
 from core.results.results import IResults
-from conf import settings
 from conf import get_test_param
 
 
-class TrafficControllerRFC2889(ITrafficController, IResults):
+class TrafficControllerRFC2889(TrafficController, IResults):
     """Traffic controller for RFC2889 traffic
 
     Used to setup and control a traffic generator for an RFC2889 deployment
@@ -21,56 +30,15 @@ class TrafficControllerRFC2889(ITrafficController, IResults):
 
         :param traffic_gen_class: The traffic generator class to be used.
         """
-        self._logger = logging.getLogger(__name__)
-        self._logger.debug("__init__")
-        self._traffic_gen_class = traffic_gen_class()
-        self._traffic_started = False
-        self._traffic_started_call_count = 0
+        super(TrafficControllerRFC2889, self).__init__(traffic_gen_class)
+        self._type = 'rfc2889'
         self._trials = int(get_test_param('rfc2889_trials', 1))
-        self._duration = int(get_test_param('duration', 30))
-        self._results = []
-
-        # If set, comma separated packet_sizes value from --test_params
-        # on cli takes precedence over value in settings file.
-        self._packet_sizes = None
-        packet_sizes_cli = get_test_param('pkt_sizes')
-        if packet_sizes_cli:
-            self._packet_sizes = [int(x.strip())
-                                  for x in packet_sizes_cli.split(',')]
-        else:
-            self._packet_sizes = settings.getValue('TRAFFICGEN_PKT_SIZES')
-
-    def __enter__(self):
-        """Call initialisation function.
-        """
-        self._traffic_gen_class.connect()
-
-    def __exit__(self, type_, value, traceback):
-        """Stop traffic, clean up.
-        """
-        if self._traffic_started:
-            self.stop_traffic()
-
-    @staticmethod
-    def _append_results(result_dict, packet_size):
-        """Adds common values to traffic generator results.
-
-        :param result_dict: Dictionary containing results from trafficgen
-        :param packet_size: Packet size value.
-
-        :returns: dictionary of results with additional entries.
-        """
-
-        ret_value = result_dict
-
-        ret_value[ResultsConstants.TYPE] = 'rfc2889'
-        ret_value[ResultsConstants.PACKET_SIZE] = str(packet_size)
-
-        return ret_value
 
     def send_traffic(self, traffic):
-        """See ITrafficController for description
+        """See TrafficController for description
         """
+        if not self.traffic_required():
+            return
         self._logger.debug('send_traffic with ' +
                            str(self._traffic_gen_class))
 
@@ -92,13 +60,14 @@ class TrafficControllerRFC2889(ITrafficController, IResults):
                 result = self._traffic_gen_class.send_rfc2889_forwarding(
                     traffic, tests=self._trials, duration=self._duration)
 
-            result = TrafficControllerRFC2889._append_results(result,
-                                                              packet_size)
+            result = self._append_results(result, packet_size)
             self._results.append(result)
 
     def send_traffic_async(self, traffic, function):
-        """See ITrafficController for description
+        """See TrafficController for description
         """
+        if not self.traffic_required():
+            return
         self._logger.debug('send_traffic_async with ' +
                            str(self._traffic_gen_class))
 
@@ -115,32 +84,6 @@ class TrafficControllerRFC2889(ITrafficController, IResults):
                 function['function']()
             result = self._traffic_gen_class.wait_rfc2889_forwarding(
                         traffic, trials=self._trials, duration=self._duration)
-            result = TrafficControllerRFC2889._append_results(result,
-                                                              packet_size)
+            result = self._append_results(result, packet_size)
             self._results.append(result)
 
-    def stop_traffic(self):
-        """Kills traffic being sent from the traffic generator.
-        """
-        self._logger.debug("stop_traffic()")
-
-    def print_results(self):
-        """IResult interface implementation.
-        """
-        counter = 0
-        for item in self._results:
-            logging.info("Record: " + str(counter))
-            counter += 1
-            for(key, value) in list(item.items()):
-                logging.info("         Key: " + str(key) +
-                             ", Value: " + str(value))
-
-    def get_results(self):
-        """IResult interface implementation.
-        """
-        return self._results
-
-    def validate_send_traffic(self, result, traffic):
-        """Verify that send traffic has succeeded
-        """
-        return True
diff --git a/core/vnf_controller.py b/core/vnf_controller.py
index 3e472f04..937cd5cc 100644
--- a/core/vnf_controller.py
+++ b/core/vnf_controller.py
@@ -31,10 +31,14 @@ class VnfController(object):
         _vnfs: A list of vnfs controlled by the controller.
     """
 
-    def __init__(self, deployment, vnf_class):
+    def __init__(self, deployment, vnf_class, extra_vnfs):
         """Sets up the VNF infrastructure based on deployment scenario
 
         :param vnf_class: The VNF class to be used.
+        :param extra_vnfs: The number of VNFs not involved in given
+            deployment scenario. It will be used to correctly expand
+            configuration values and initialize shared dirs. This parameter
+            is used in case, that additional VNFs are executed by TestSteps.
         """
         # reset VNF ID counter for each testcase
         IVnf.reset_vnf_counter()
@@ -57,9 +61,9 @@ class VnfController(object):
             # without VNFs like p2p
             vm_number = 0
 
-        if vm_number:
-            self._logger.debug('Check configuration for %s guests.', vm_number)
-            settings.check_vm_settings(vm_number)
+        if vm_number + extra_vnfs > 0:
+            self._logger.debug('Check configuration for %s guests.', vm_number + extra_vnfs)
+            settings.check_vm_settings(vm_number + extra_vnfs)
             # enforce that GUEST_NIC_NR is 1 or even number of NICs
             updated = False
             nics_nr = settings.getValue('GUEST_NICS_NR')
@@ -73,10 +77,11 @@ class VnfController(object):
                                      'was updated to GUEST_NICS_NR = %s',
                                      settings.getValue('GUEST_NICS_NR'))
 
+        if vm_number:
             self._vnfs = [vnf_class() for _ in range(vm_number)]
 
-        self._logger.debug('__init__ ' + str(len(self._vnfs)) +
-                           ' VNF[s] with ' + ' '.join(map(str, self._vnfs)))
+            self._logger.debug('__init__ ' + str(len(self._vnfs)) +
+                               ' VNF[s] with ' + ' '.join(map(str, self._vnfs)))
 
     def get_vnfs(self):
         """Returns a list of vnfs controlled by this controller.