teststeps: Improvements and bugfixing of teststeps

This patch introduces several improvements and small bugfixes
of teststeps. These changes were identified during implementation
of OVS/DPDK regression tests.

Patch content:
* teststeps: step aliases were implemented
* teststeps: improved filtering by regex for any step, which returns string
  or list of stings; filter will process all lines
* teststeps: support for log object
* teststeps: support for trafficgen get_results call
* teststeps: configurable suppression of step validation
* trafficgen: remove old results before traffic is executed
* trafficgen: support for flow control on/off (IxNet)
* trafficgen: support for configurable learning frames (IxNet)
* trafficgen: support for runtime changes of TRAFFICGEN_PKT_SIZES, _DURATION
  and _LOSSRATE
* vnf: flush pexpect output of previous commands
* vnf: use execute_and_wait() to ensure correct cmds order
* vnf: dpdk vHost User interface name set according to its type,
  e.g. dpdkvhostuserclient
* vswitch: support for OVS restart
* decap: simplify configuration of tunneling decapsulation tests
* settings: values of all configuration options are restored after TC execution
* modified formatting of test description used by --list
* testcase name and description is logged before its execution
* small bugfixes

JIRA: VSPERF-539

Change-Id: I550ba0d897ece89abd3f33d6d66f545c4d863e7b
Signed-off-by: Martin Klozik <martinx.klozik@intel.com>
Reviewed-by: Al Morton <acmorton@att.com>
Reviewed-by: Christian Trautman <ctrautma@redhat.com>
Reviewed-by: Sridhar Rao <sridhar.rao@spirent.com>
Reviewed-by: Trevor Cooper <trevor.cooper@intel.com>

1 files changed, 74 insertions, 40 deletions

diff --git a/testcases/testcase.py b/testcases/testcase.py
index 01b3a975..1b8bf7dc 100644
--- a/testcases/testcase.py
+++ b/testcases/testcase.py
@@ -68,7 +68,6 @@ class TestCase(object):
         self._loadgen = None
         self._output_file = None
         self._tc_results = None
-        self._settings_original = {}
         self._settings_paths_modified = False
         self._testcast_run_time = None
         self._versions = []
@@ -76,21 +75,18 @@ class TestCase(object):
         self._step_check = False    # by default don't check result for step driven testcases
         self._step_vnf_list = {}
         self._step_result = []
+        self._step_result_mapping = {}
         self._step_status = None
+        self._step_send_traffic = False # indication if send_traffic was called within test steps
         self._testcase_run_time = None
 
-        # store all GUEST_ specific settings to keep original values before their expansion
-        for key in S.__dict__:
-            if key.startswith('GUEST_'):
-                self._settings_original[key] = S.getValue(key)
-
-        self._update_settings('VSWITCH', cfg.get('vSwitch', S.getValue('VSWITCH')))
-        self._update_settings('VNF', cfg.get('VNF', S.getValue('VNF')))
-        self._update_settings('TRAFFICGEN', cfg.get('Trafficgen', S.getValue('TRAFFICGEN')))
+        S.setValue('VSWITCH', cfg.get('vSwitch', S.getValue('VSWITCH')))
+        S.setValue('VNF', cfg.get('VNF', S.getValue('VNF')))
+        S.setValue('TRAFFICGEN', cfg.get('Trafficgen', S.getValue('TRAFFICGEN')))
         test_params = copy.deepcopy(S.getValue('TEST_PARAMS'))
         tc_test_params = cfg.get('Parameters', S.getValue('TEST_PARAMS'))
         test_params = merge_spec(test_params, tc_test_params)
-        self._update_settings('TEST_PARAMS', test_params)
+        S.setValue('TEST_PARAMS', test_params)
         S.check_test_params()
 
         # override all redefined GUEST_ values to have them expanded correctly
@@ -109,6 +105,15 @@ class TestCase(object):
         self.desc = cfg.get('Description', 'No description given.')
         self.test = cfg.get('TestSteps', None)
 
+        # log testcase name and details
+        tmp_desc = functions.format_description(self.desc, 50)
+        self._logger.info('############################################################')
+        self._logger.info('# Test:    %s', self.name)
+        self._logger.info('# Details: %s', tmp_desc[0])
+        for i in range(1, len(tmp_desc)):
+            self._logger.info('#          %s', tmp_desc[i])
+        self._logger.info('############################################################')
+
         bidirectional = S.getValue('TRAFFIC')['bidir']
         if not isinstance(S.getValue('TRAFFIC')['bidir'], str):
             raise TypeError(
@@ -167,6 +172,7 @@ class TestCase(object):
                 self._traffic['l2'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L2')
                 self._traffic['l3'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L3')
                 self._traffic['l4'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L4')
+                self._traffic['l2']['dstmac'] = S.getValue('NICS')[1]['mac']
         elif len(S.getValue('NICS')) and \
              (S.getValue('NICS')[0]['type'] == 'vf' or
               S.getValue('NICS')[1]['type'] == 'vf'):
@@ -186,8 +192,6 @@ class TestCase(object):
     def run_initialize(self):
         """ Prepare test execution environment
         """
-        self._logger.debug(self.name)
-
         # mount hugepages if needed
         self._mount_hugepages()
 
@@ -343,7 +347,7 @@ class TestCase(object):
                             # ...and continue with traffic generation, but keep
                             # in mind, that clean deployment does not configure
                             # OVS nor executes the traffic
-                            if self.deployment != 'clean':
+                            if self.deployment != 'clean' and not self._step_send_traffic:
                                 self._traffic_ctl.send_traffic(self._traffic)
 
                         # dump vswitch flows before they are affected by VNF termination
@@ -365,23 +369,6 @@ class TestCase(object):
         # report test results
         self.run_report()
 
-        # restore original settings
-        for key in self._settings_original:
-            S.setValue(key, self._settings_original[key])
-
-    def _update_settings(self, param, value):
-        """ Check value of given configuration parameter
-        In case that new value is different, then testcase
-        specific settings is updated and original value stored
-
-        :param param: Name of parameter inside settings
-        :param value: Disired parameter value
-        """
-        orig_value = S.getValue(param)
-        if orig_value != value:
-            self._settings_original[param] = copy.deepcopy(orig_value)
-            S.setValue(param, value)
-
     def _append_results(self, results):
         """
         Method appends mandatory Test Case results to list of dictionaries.
@@ -689,18 +676,22 @@ class TestCase(object):
             if self._step_vnf_list[vnf]:
                 self._step_vnf_list[vnf].stop()
 
-    def step_eval_param(self, param, STEP):
-        # pylint: disable=invalid-name
+    def step_eval_param(self, param, step_result):
         """ Helper function for #STEP macro evaluation
         """
         if isinstance(param, str):
             # evaluate every #STEP reference inside parameter itself
-            macros = re.findall(r'#STEP\[[\w\[\]\-\'\"]+\]', param)
+            macros = re.findall(r'(#STEP\[([\w\-:]+)\]((\[[\w\-\'\"]+\])*))', param)
+
             if macros:
                 for macro in macros:
+                    if macro[1] in self._step_result_mapping:
+                        key = self._step_result_mapping[macro[1]]
+                    else:
+                        key = macro[1]
                     # pylint: disable=eval-used
-                    tmp_val = str(eval(macro[1:]))
-                    param = param.replace(macro, tmp_val)
+                    tmp_val = str(eval('step_result[{}]{}'.format(key, macro[2])))
+                    param = param.replace(macro[0], tmp_val)
 
             # evaluate references to vsperf configuration options
             macros = re.findall(r'\$(([\w\-]+)(\[[\w\[\]\-\'\"]+\])*)', param)
@@ -718,12 +709,12 @@ class TestCase(object):
         elif isinstance(param, list) or isinstance(param, tuple):
             tmp_list = []
             for item in param:
-                tmp_list.append(self.step_eval_param(item, STEP))
+                tmp_list.append(self.step_eval_param(item, step_result))
             return tmp_list
         elif isinstance(param, dict):
             tmp_dict = {}
             for (key, value) in param.items():
-                tmp_dict[key] = self.step_eval_param(value, STEP)
+                tmp_dict[key] = self.step_eval_param(value, step_result)
             return tmp_dict
         else:
             return param
@@ -737,6 +728,7 @@ class TestCase(object):
             eval_params.append(self.step_eval_param(param, step_result))
         return eval_params
 
+    # pylint: disable=too-many-locals, too-many-branches, too-many-statements
     def step_run(self):
         """ Execute actions specified by TestSteps list
 
@@ -758,6 +750,30 @@ class TestCase(object):
         # run test step by step...
         for i, step in enumerate(self.test):
             step_ok = not self._step_check
+            step_check = self._step_check
+            regex = None
+            # configure step result mapping if step alias/label is detected
+            if step[0].startswith('#'):
+                key = step[0][1:]
+                if key.isdigit():
+                    raise RuntimeError('Step alias can\'t be an integer value {}'.format(key))
+                if key in self._step_result_mapping:
+                    raise RuntimeError('Step alias {} has been used already for step '
+                                       '{}'.format(key, self._step_result_mapping[key]))
+                self._step_result_mapping[step[0][1:]] = i
+                step = step[1:]
+
+            # store regex filter if it is specified
+            if isinstance(step[-1], str) and step[-1].startswith('|'):
+                # evalute macros and variables used in regex
+                regex = self.step_eval_params([step[-1][1:]], self._step_result[:i])[0]
+                step = step[:-1]
+
+            # check if step verification should be suppressed
+            if step[0].startswith('!'):
+                step_check = False
+                step_ok = True
+                step[0] = step[0][1:]
             if step[0] == 'vswitch':
                 test_object = self._vswitch_ctl.get_vswitch()
             elif step[0] == 'namespace':
@@ -777,6 +793,9 @@ class TestCase(object):
                     tmp_traffic = copy.deepcopy(self._traffic)
                     tmp_traffic.update(step[2])
                     step[2] = tmp_traffic
+                    # store indication that traffic has been sent
+                    # so it is not sent again after the execution of teststeps
+                    self._step_send_traffic = True
             elif step[0].startswith('vnf'):
                 if not self._step_vnf_list[step[0]]:
                     # initialize new VM
@@ -790,6 +809,15 @@ class TestCase(object):
                 self._logger.debug("Sleep %s seconds", step[1])
                 time.sleep(int(step[1]))
                 continue
+            elif step[0] == 'log':
+                test_object = self._logger
+                # there isn't a need for validation of log entry
+                step_check = False
+                step_ok = True
+            elif step[0] == 'pdb':
+                import pdb
+                pdb.set_trace()
+                continue
             else:
                 self._logger.error("Unsupported test object %s", step[0])
                 self._step_status = {'status' : False, 'details' : ' '.join(step)}
@@ -798,7 +826,7 @@ class TestCase(object):
                 return False
 
             test_method = getattr(test_object, step[1])
-            if self._step_check:
+            if step_check:
                 test_method_check = getattr(test_object, CHECK_PREFIX + step[1])
             else:
                 test_method_check = None
@@ -809,18 +837,24 @@ class TestCase(object):
                 # to support negative indexes
                 step_params = self.step_eval_params(step[2:], self._step_result[:i])
                 step_log = '{} {}'.format(' '.join(step[:2]), step_params)
+                step_log += ' filter "{}"'.format(regex) if regex else ''
                 self._logger.debug("Step %s '%s' start", i, step_log)
                 self._step_result[i] = test_method(*step_params)
+                if regex:
+                    # apply regex to step output
+                    self._step_result[i] = functions.filter_output(
+                        self._step_result[i], regex)
+
                 self._logger.debug("Step %s '%s' results '%s'", i,
                                    step_log, self._step_result[i])
                 time.sleep(S.getValue('TEST_STEP_DELAY'))
-                if self._step_check:
+                if step_check:
                     step_ok = test_method_check(self._step_result[i], *step_params)
             except (AssertionError, AttributeError, IndexError) as ex:
                 step_ok = False
                 self._logger.error("Step %s raised %s", i, type(ex).__name__)
 
-            if self._step_check:
+            if step_check:
                 self.step_report_status("Step {} - '{}'".format(i, step_log), step_ok)
 
             if not step_ok: