teststeps: Generic support of step driven tests

In the past, step driven testcases were supported
only by integration testcases. This patch adds generic
support of TestSteps for both integration and performance
testcases. Step driven test were improved to support
modification of existing deployment. As part of
the patch a refactoring of traffic controllers
were performed. Traffic controllers were modified
to support trafficgen-off and trafficgen-pause
modes in all possible ways of trafficgen invocation.

JIRA: VSPERF-362

Change-Id: Ic8b7a9b0e7165f0a15a52279ed0f0952da9fedb8
Signed-off-by: Martin Klozik <martinx.klozik@intel.com>
Reviewed-by: Maryam Tahhan <maryam.tahhan@intel.com>
Reviewed-by: Al Morton <acmorton@att.com>
Reviewed-by: Christian Trautman <ctrautma@redhat.com>
Reviewed-by: Bill Michalowski <bmichalo@redhat.com>
Reviewed-by: Antonio Fischetti <antonio.fischetti@intel.com>
Reviewed-by: Sridhar K. N. Rao <sridhar.rao@spirent.com>

3 files changed, 200 insertions, 218 deletions

diff --git a/testcases/integration.py b/testcases/integration.py
index bec38624..4b9dacfd 100644
--- a/testcases/integration.py
+++ b/testcases/integration.py
@@ -28,9 +28,6 @@ from tools import veth
 from tools.teststepstools import TestStepsTools
 from core.loader import Loader
 
-CHECK_PREFIX = 'validate_'
-
-
 class IntegrationTestCase(TestCase):
     """IntegrationTestCase class
     """
@@ -41,193 +38,20 @@ class IntegrationTestCase(TestCase):
         self._type = 'integration'
         super(IntegrationTestCase, self).__init__(cfg)
         self._logger = logging.getLogger(__name__)
-        self._inttest = None
-
-    def report_status(self, label, status):
-        """ Log status of test step
-        """
-        self._logger.info("%s ... %s", label, 'OK' if status else 'FAILED')
-
-    def run_initialize(self):
-        """ Prepare test execution environment
-        """
-        super(IntegrationTestCase, self).run_initialize()
-        self._inttest = {'status' : True, 'details' : ''}
-
-    def run(self):
-        """Run the test
-
-        All setup and teardown through controllers is included.
-        """
-        def eval_step_params(params, step_result):
-            """ Evaluates referrences to results from previous steps
-            """
-            def eval_param(param, STEP):
-                # pylint: disable=invalid-name
-                """ Helper function
-                """
-                if isinstance(param, str):
-                    # evaluate every #STEP reference inside parameter itself
-                    macros = re.findall(r'#STEP\[[\w\[\]\-\'\"]+\]', param)
-                    if macros:
-                        for macro in macros:
-                            # pylint: disable=eval-used
-                            tmp_val = str(eval(macro[1:]))
-                            param = param.replace(macro, tmp_val)
-                    return param
-                elif isinstance(param, list) or isinstance(param, tuple):
-                    tmp_list = []
-                    for item in param:
-                        tmp_list.append(eval_param(item, STEP))
-                    return tmp_list
-                elif isinstance(param, dict):
-                    tmp_dict = {}
-                    for (key, value) in param.items():
-                        tmp_dict[key] = eval_param(value, STEP)
-                    return tmp_dict
-                else:
-                    return param
-
-            eval_params = []
-            # evaluate all parameters if needed
-            for param in params:
-                eval_params.append(eval_param(param, step_result))
-            return eval_params
-
-        # prepare test execution environment
-        self.run_initialize()
-
-        try:
-            with self._vswitch_ctl, self._loadgen:
-                with self._vnf_ctl, self._collector:
-                    if not self._vswitch_none:
-                        self._add_flows()
-
-                    # run traffic generator if requested, otherwise wait for manual termination
-                    if S.getValue('mode') == 'trafficgen-off':
-                        time.sleep(2)
-                        self._logger.debug("All is set. Please run traffic generator manually.")
-                        input(os.linesep + "Press Enter to terminate vswitchperf..." + os.linesep + os.linesep)
-                    else:
-                        with self._traffic_ctl:
-                            if not self.test:
-                                self._traffic_ctl.send_traffic(self._traffic)
-                            else:
-                                vnf_list = {}
-                                loader = Loader()
-                                # execute test based on TestSteps definition
-                                if self.test:
-                                    # initialize list with results
-                                    step_result = [None] * len(self.test)
-
-                                    # count how many VNFs are involved in the test
-                                    for step in self.test:
-                                        if step[0].startswith('vnf'):
-                                            vnf_list[step[0]] = None
-
-                                    # check/expand GUEST configuration and copy data to shares
-                                    if len(vnf_list):
-                                        S.check_vm_settings(len(vnf_list))
-                                        self._copy_fwd_tools_for_all_guests(len(vnf_list))
-
-                                    # run test step by step...
-                                    for i, step in enumerate(self.test):
-                                        step_ok = False
-                                        if step[0] == 'vswitch':
-                                            test_object = self._vswitch_ctl.get_vswitch()
-                                        elif step[0] == 'namespace':
-                                            test_object = namespace
-                                        elif step[0] == 'veth':
-                                            test_object = veth
-                                        elif step[0] == 'settings':
-                                            test_object = S
-                                        elif step[0] == 'tools':
-                                            test_object = TestStepsTools()
-                                            step[1] = step[1].title()
-                                        elif step[0] == 'trafficgen':
-                                            test_object = self._traffic_ctl
-                                            # in case of send_traffic method, ensure that specified
-                                            # traffic values are merged with existing self._traffic
-                                            if step[1] == 'send_traffic':
-                                                tmp_traffic = copy.deepcopy(self._traffic)
-                                                tmp_traffic.update(step[2])
-                                                step[2] = tmp_traffic
-                                        elif step[0].startswith('vnf'):
-                                            if not vnf_list[step[0]]:
-                                                # initialize new VM
-                                                vnf_list[step[0]] = loader.get_vnf_class()()
-                                            test_object = vnf_list[step[0]]
-                                        elif step[0] == 'wait':
-                                            input(os.linesep + "Step {}: Press Enter to continue with "
-                                                  "the next step...".format(i) + os.linesep + os.linesep)
-                                            continue
-                                        else:
-                                            self._logger.error("Unsupported test object %s", step[0])
-                                            self._inttest = {'status' : False, 'details' : ' '.join(step)}
-                                            self.report_status("Step '{}'".format(' '.join(step)),
-                                                               self._inttest['status'])
-                                            break
-
-                                        test_method = getattr(test_object, step[1])
-                                        test_method_check = getattr(test_object, CHECK_PREFIX + step[1])
-
-                                        step_params = []
-                                        if test_method and test_method_check and \
-                                            callable(test_method) and callable(test_method_check):
-
-                                            try:
-                                                # eval parameters, but use only valid step_results
-                                                # to support negative indexes
-                                                step_params = eval_step_params(step[2:], step_result[:i])
-                                                step_log = '{} {}'.format(' '.join(step[:2]), step_params)
-                                                step_result[i] = test_method(*step_params)
-                                                self._logger.debug("Step %s '%s' results '%s'", i,
-                                                                   step_log, step_result[i])
-                                                time.sleep(5)
-                                                step_ok = test_method_check(step_result[i], *step_params)
-                                            except AssertionError:
-                                                self._inttest = {'status' : False, 'details' : step_log}
-                                                self._logger.error("Step %s raised assertion error", i)
-                                                # stop vnfs in case of error
-                                                for vnf in vnf_list:
-                                                    vnf_list[vnf].stop()
-                                                break
-                                            except IndexError:
-                                                self._inttest = {'status' : False, 'details' : step_log}
-                                                self._logger.error("Step %s result index error %s", i,
-                                                                   ' '.join(step[2:]))
-                                                # stop vnfs in case of error
-                                                for vnf in vnf_list:
-                                                    vnf_list[vnf].stop()
-                                                break
-
-                                        self.report_status("Step {} - '{}'".format(i, step_log), step_ok)
-                                        if not step_ok:
-                                            self._inttest = {'status' : False, 'details' : step_log}
-                                            # stop vnfs in case of error
-                                            for vnf in vnf_list:
-                                                vnf_list[vnf].stop()
-                                            break
-
-                        # dump vswitch flows before they are affected by VNF termination
-                        if not self._vswitch_none:
-                            self._vswitch_ctl.dump_vswitch_flows()
-        finally:
-            # tear down test execution environment and log results
-            self.run_finalize()
-
-        # report test results
-        self.run_report()
+        # enforce check of step result for step driven testcases
+        self._step_check = True
 
     def run_report(self):
         """ Report test results
         """
         if self.test:
             results = OrderedDict()
-            results['status'] = 'OK' if self._inttest['status'] else 'FAILED'
-            results['details'] = self._inttest['details']
+            results['status'] = 'OK' if self._step_status['status'] else 'FAILED'
+            results['details'] = self._step_status['details']
             TestCase.write_result_to_file([results], self._output_file)
-            self.report_status("Test '{}'".format(self.name), self._inttest['status'])
+            self.step_report_status("Test '{}'".format(self.name), self._step_status['status'])
             # inform vsperf about testcase failure
-            if not self._inttest['status']:
+            if not self._step_status['status']:
                 raise Exception
+        else:
+            super(IntegrationTestCase, self).run_report()
diff --git a/testcases/performance.py b/testcases/performance.py
index a4769a28..240d04a9 100644
--- a/testcases/performance.py
+++ b/testcases/performance.py
@@ -34,5 +34,5 @@ class PerformanceTestCase(TestCase):
 
     def run_report(self):
         super(PerformanceTestCase, self).run_report()
-        if S.getValue('mode') != 'trafficgen-off':
+        if self._tc_results:
             report.generate(self._output_file, self._tc_results, self._collector.get_results(), self._type)
diff --git a/testcases/testcase.py b/testcases/testcase.py
index 7f22c18f..00164ea2 100644
--- a/testcases/testcase.py
+++ b/testcases/testcase.py
@@ -34,6 +34,7 @@ from tools import hugepages
 from tools import functions
 from tools.pkt_gen.trafficgen.trafficgenhelper import TRAFFIC_DEFAULTS
 
+CHECK_PREFIX = 'validate_'
 
 class TestCase(object):
     """TestCase base class
@@ -60,6 +61,11 @@ class TestCase(object):
         self._settings_original = {}
         self._settings_paths_modified = False
         self._testcast_run_time = None
+        # initialization of step driven specific members
+        self._step_check = False    # by default don't check result for step driven testcases
+        self._step_vnf_list = {}
+        self._step_result = []
+        self._step_status = None
 
         # store all GUEST_ specific settings to keep original values before their expansion
         for key in S.__dict__:
@@ -170,6 +176,12 @@ class TestCase(object):
             else:
                 self._logger.debug("MAC addresses can not be read")
 
+        # count how many VNFs are involved in TestSteps
+        if self.test:
+            for step in self.test:
+                if step[0].startswith('vnf'):
+                    self._step_vnf_list[step[0]] = None
+
     def run_initialize(self):
         """ Prepare test execution environment
         """
@@ -186,26 +198,31 @@ class TestCase(object):
 
         self._vnf_ctl = component_factory.create_vnf(
             self.deployment,
-            loader.get_vnf_class())
+            loader.get_vnf_class(),
+            len(self._step_vnf_list))
 
         # verify enough hugepages are free to run the testcase
         if not self._check_for_enough_hugepages():
             raise RuntimeError('Not enough hugepages free to run test.')
 
         # perform guest related handling
-        if self._vnf_ctl.get_vnfs_number():
+        tmp_vm_count = self._vnf_ctl.get_vnfs_number() + len(self._step_vnf_list)
+        if tmp_vm_count:
             # copy sources of l2 forwarding tools into VM shared dir if needed
-            self._copy_fwd_tools_for_all_guests(self._vnf_ctl.get_vnfs_number())
+            self._copy_fwd_tools_for_all_guests(tmp_vm_count)
 
             # in case of multi VM in parallel, set the number of streams to the number of VMs
             if self.deployment.startswith('pvpv'):
                 # for each VM NIC pair we need an unique stream
                 streams = 0
-                for vm_nic in S.getValue('GUEST_NICS_NR')[:self._vnf_ctl.get_vnfs_number()]:
+                for vm_nic in S.getValue('GUEST_NICS_NR')[:tmp_vm_count]:
                     streams += int(vm_nic / 2) if vm_nic > 1 else 1
                 self._logger.debug("VMs with parallel connection were detected. "
                                    "Thus Number of streams was set to %s", streams)
-                self._traffic.update({'multistream': streams})
+                # update streams if needed; In case of additional VNFs deployed by TestSteps
+                # user can define a proper stream count manually
+                if 'multistream' not in self._traffic or self._traffic['multistream'] < streams:
+                    self._traffic.update({'multistream': streams})
 
             # OVS Vanilla requires guest VM MAC address and IPs to work
             if 'linux_bridge' in S.getValue('GUEST_LOOPBACK'):
@@ -235,11 +252,16 @@ class TestCase(object):
         self._output_file = os.path.join(self._results_dir, "result_" + self.name +
                                          "_" + self.deployment + ".csv")
 
+        self._step_status = {'status' : True, 'details' : ''}
+
         self._logger.debug("Setup:")
 
     def run_finalize(self):
         """ Tear down test execution environment and record test results
         """
+        # Stop all VNFs started by TestSteps in case that something went wrong
+        self.step_stop_vnfs()
+
         # umount hugepages if mounted
         self._umount_hugepages()
 
@@ -272,11 +294,12 @@ class TestCase(object):
         self._logger.debug("self._collector Results:")
         self._collector.print_results()
 
-        if S.getValue('mode') != 'trafficgen-off':
+        results = self._traffic_ctl.get_results()
+        if results:
             self._logger.debug("Traffic Results:")
             self._traffic_ctl.print_results()
 
-            self._tc_results = self._append_results(self._traffic_ctl.get_results())
+            self._tc_results = self._append_results(results)
             TestCase.write_result_to_file(self._tc_results, self._output_file)
 
     def run(self):
@@ -287,36 +310,31 @@ class TestCase(object):
         # prepare test execution environment
         self.run_initialize()
 
-        with self._vswitch_ctl, self._loadgen:
-            with self._vnf_ctl, self._collector:
-                if not self._vswitch_none:
-                    self._add_flows()
+        try:
+            with self._vswitch_ctl, self._loadgen:
+                with self._vnf_ctl, self._collector:
+                    if not self._vswitch_none:
+                        self._add_flows()
 
-                # run traffic generator if requested, otherwise wait for manual termination
-                if S.getValue('mode') == 'trafficgen-off':
-                    time.sleep(2)
-                    self._logger.debug("All is set. Please run traffic generator manually.")
-                    input(os.linesep + "Press Enter to terminate vswitchperf..." + os.linesep + os.linesep)
-                else:
-                    if S.getValue('mode') == 'trafficgen-pause':
-                        time.sleep(2)
-                        true_vals = ('yes', 'y', 'ye', None)
-                        while True:
-                            choice = input(os.linesep + 'Transmission paused, should'
-                                           ' transmission be resumed? ' + os.linesep).lower()
-                            if not choice or choice not in true_vals:
-                                print('Please respond with \'yes\' or \'y\' ', end='')
-                            else:
-                                break
                     with self._traffic_ctl:
-                        self._traffic_ctl.send_traffic(self._traffic)
+                        # execute test based on TestSteps definition if needed...
+                        if self.step_run():
+                            # ...and continue with traffic generation, but keep
+                            # in mind, that clean deployment does not configure
+                            # OVS nor executes the traffic
+                            if self.deployment != 'clean':
+                                self._traffic_ctl.send_traffic(self._traffic)
 
-                    # dump vswitch flows before they are affected by VNF termination
-                    if not self._vswitch_none:
-                        self._vswitch_ctl.dump_vswitch_flows()
+                        # dump vswitch flows before they are affected by VNF termination
+                        if not self._vswitch_none:
+                            self._vswitch_ctl.dump_vswitch_flows()
 
-        # tear down test execution environment and log results
-        self.run_finalize()
+                    # garbage collection for case that TestSteps modify existing deployment
+                    self.step_stop_vnfs()
+
+        finally:
+            # tear down test execution environment and log results
+            self.run_finalize()
 
         self._testcase_run_time = time.strftime("%H:%M:%S",
                                   time.gmtime(time.time() - self._testcase_start_time))
@@ -635,3 +653,143 @@ class TestCase(object):
             vswitch.add_flow(bridge, flow)
         else:
             pass
+
+
+    #
+    # TestSteps realted methods
+    #
+    def step_report_status(self, label, status):
+        """ Log status of test step
+        """
+        self._logger.info("%s ... %s", label, 'OK' if status else 'FAILED')
+
+    def step_stop_vnfs(self):
+        """ Stop all VNFs started by TestSteps
+        """
+        for vnf in self._step_vnf_list:
+            self._step_vnf_list[vnf].stop()
+
+    @staticmethod
+    def step_eval_param(param, STEP):
+        # pylint: disable=invalid-name
+        """ Helper function for #STEP macro evaluation
+        """
+        if isinstance(param, str):
+            # evaluate every #STEP reference inside parameter itself
+            macros = re.findall(r'#STEP\[[\w\[\]\-\'\"]+\]', param)
+            if macros:
+                for macro in macros:
+                    # pylint: disable=eval-used
+                    tmp_val = str(eval(macro[1:]))
+                    param = param.replace(macro, tmp_val)
+            return param
+        elif isinstance(param, list) or isinstance(param, tuple):
+            tmp_list = []
+            for item in param:
+                tmp_list.append(TestCase.step_eval_param(item, STEP))
+            return tmp_list
+        elif isinstance(param, dict):
+            tmp_dict = {}
+            for (key, value) in param.items():
+                tmp_dict[key] = TestCase.step_eval_param(value, STEP)
+            return tmp_dict
+        else:
+            return param
+
+    @staticmethod
+    def step_eval_params(params, step_result):
+        """ Evaluates referrences to results from previous steps
+        """
+        eval_params = []
+        # evaluate all parameters if needed
+        for param in params:
+            eval_params.append(TestCase.step_eval_param(param, step_result))
+        return eval_params
+
+    def step_run(self):
+        """ Execute actions specified by TestSteps list
+
+        :return: False if any error was detected
+                 True otherwise
+        """
+        # anything to do?
+        if not self.test:
+            return True
+
+        # required for VNFs initialization
+        loader = Loader()
+        # initialize list with results
+        self._step_result = [None] * len(self.test)
+
+        # run test step by step...
+        for i, step in enumerate(self.test):
+            step_ok = not self._step_check
+            if step[0] == 'vswitch':
+                test_object = self._vswitch_ctl.get_vswitch()
+            elif step[0] == 'namespace':
+                test_object = namespace
+            elif step[0] == 'veth':
+                test_object = veth
+            elif step[0] == 'settings':
+                test_object = S
+            elif step[0] == 'tools':
+                test_object = TestStepsTools()
+                step[1] = step[1].title()
+            elif step[0] == 'trafficgen':
+                test_object = self._traffic_ctl
+                # in case of send_traffic or send_traffic_async methods, ensure
+                # that specified traffic values are merged with existing self._traffic
+                if step[1].startswith('send_traffic'):
+                    tmp_traffic = copy.deepcopy(self._traffic)
+                    tmp_traffic.update(step[2])
+                    step[2] = tmp_traffic
+            elif step[0].startswith('vnf'):
+                if not self._step_vnf_list[step[0]]:
+                    # initialize new VM
+                    self._step_vnf_list[step[0]] = loader.get_vnf_class()()
+                test_object = self._step_vnf_list[step[0]]
+            elif step[0] == 'wait':
+                input(os.linesep + "Step {}: Press Enter to continue with "
+                      "the next step...".format(i) + os.linesep + os.linesep)
+                continue
+            else:
+                self._logger.error("Unsupported test object %s", step[0])
+                self._step_status = {'status' : False, 'details' : ' '.join(step)}
+                self.step_report_status("Step '{}'".format(' '.join(step)),
+                                        self._step_status['status'])
+                return False
+
+            test_method = getattr(test_object, step[1])
+            if self._step_check:
+                test_method_check = getattr(test_object, CHECK_PREFIX + step[1])
+            else:
+                test_method_check = None
+
+            step_params = []
+            try:
+                # eval parameters, but use only valid step_results
+                # to support negative indexes
+                step_params = TestCase.step_eval_params(step[2:], self._step_result[:i])
+                step_log = '{} {}'.format(' '.join(step[:2]), step_params)
+                self._logger.debug("Step %s '%s' start", i, step_log)
+                self._step_result[i] = test_method(*step_params)
+                self._logger.debug("Step %s '%s' results '%s'", i,
+                                   step_log, self._step_result[i])
+                time.sleep(5)
+                if self._step_check:
+                    step_ok = test_method_check(self._step_result[i], *step_params)
+            except (AssertionError, AttributeError, IndexError) as ex:
+                step_ok = False
+                self._logger.error("Step %s raised %s", i, type(ex).__name__)
+
+            if self._step_check:
+                self.step_report_status("Step {} - '{}'".format(i, step_log), step_ok)
+
+            if not step_ok:
+                self._step_status = {'status' : False, 'details' : step_log}
+                # Stop all VNFs started by TestSteps
+                self.step_stop_vnfs()
+                return False
+
+        # all steps processed without any issue
+        return True