aboutsummaryrefslogtreecommitdiffstats
path: root/testcases
diff options
context:
space:
mode:
authorMartin Klozik <martinx.klozik@intel.com>2017-08-31 15:01:18 +0200
committerMartin Klozik <martinx.klozik@intel.com>2017-11-03 08:36:29 +0000
commitb1534957e463b5e34957a8d48ce5c6b0552ffbb4 (patch)
tree10985b181d62cffb4ea36355de66dc8ea4edbf8a /testcases
parent87f6e48ca1b17361955f0d31551b0c6360028688 (diff)
teststeps: Improvements and bugfixing of teststeps
This patch introduces several improvements and small bugfixes of teststeps. These changes were identified during implementation of OVS/DPDK regression tests. Patch content: * teststeps: step aliases were implemented * teststeps: improved filtering by regex for any step, which returns string or list of stings; filter will process all lines * teststeps: support for log object * teststeps: support for trafficgen get_results call * teststeps: configurable suppression of step validation * trafficgen: remove old results before traffic is executed * trafficgen: support for flow control on/off (IxNet) * trafficgen: support for configurable learning frames (IxNet) * trafficgen: support for runtime changes of TRAFFICGEN_PKT_SIZES, _DURATION and _LOSSRATE * vnf: flush pexpect output of previous commands * vnf: use execute_and_wait() to ensure correct cmds order * vnf: dpdk vHost User interface name set according to its type, e.g. dpdkvhostuserclient * vswitch: support for OVS restart * decap: simplify configuration of tunneling decapsulation tests * settings: values of all configuration options are restored after TC execution * modified formatting of test description used by --list * testcase name and description is logged before its execution * small bugfixes JIRA: VSPERF-539 Change-Id: I550ba0d897ece89abd3f33d6d66f545c4d863e7b Signed-off-by: Martin Klozik <martinx.klozik@intel.com> Reviewed-by: Al Morton <acmorton@att.com> Reviewed-by: Christian Trautman <ctrautma@redhat.com> Reviewed-by: Sridhar Rao <sridhar.rao@spirent.com> Reviewed-by: Trevor Cooper <trevor.cooper@intel.com>
Diffstat (limited to 'testcases')
-rw-r--r--testcases/testcase.py114
1 files changed, 74 insertions, 40 deletions
diff --git a/testcases/testcase.py b/testcases/testcase.py
index 01b3a975..1b8bf7dc 100644
--- a/testcases/testcase.py
+++ b/testcases/testcase.py
@@ -68,7 +68,6 @@ class TestCase(object):
self._loadgen = None
self._output_file = None
self._tc_results = None
- self._settings_original = {}
self._settings_paths_modified = False
self._testcast_run_time = None
self._versions = []
@@ -76,21 +75,18 @@ class TestCase(object):
self._step_check = False # by default don't check result for step driven testcases
self._step_vnf_list = {}
self._step_result = []
+ self._step_result_mapping = {}
self._step_status = None
+ self._step_send_traffic = False # indication if send_traffic was called within test steps
self._testcase_run_time = None
- # store all GUEST_ specific settings to keep original values before their expansion
- for key in S.__dict__:
- if key.startswith('GUEST_'):
- self._settings_original[key] = S.getValue(key)
-
- self._update_settings('VSWITCH', cfg.get('vSwitch', S.getValue('VSWITCH')))
- self._update_settings('VNF', cfg.get('VNF', S.getValue('VNF')))
- self._update_settings('TRAFFICGEN', cfg.get('Trafficgen', S.getValue('TRAFFICGEN')))
+ S.setValue('VSWITCH', cfg.get('vSwitch', S.getValue('VSWITCH')))
+ S.setValue('VNF', cfg.get('VNF', S.getValue('VNF')))
+ S.setValue('TRAFFICGEN', cfg.get('Trafficgen', S.getValue('TRAFFICGEN')))
test_params = copy.deepcopy(S.getValue('TEST_PARAMS'))
tc_test_params = cfg.get('Parameters', S.getValue('TEST_PARAMS'))
test_params = merge_spec(test_params, tc_test_params)
- self._update_settings('TEST_PARAMS', test_params)
+ S.setValue('TEST_PARAMS', test_params)
S.check_test_params()
# override all redefined GUEST_ values to have them expanded correctly
@@ -109,6 +105,15 @@ class TestCase(object):
self.desc = cfg.get('Description', 'No description given.')
self.test = cfg.get('TestSteps', None)
+ # log testcase name and details
+ tmp_desc = functions.format_description(self.desc, 50)
+ self._logger.info('############################################################')
+ self._logger.info('# Test: %s', self.name)
+ self._logger.info('# Details: %s', tmp_desc[0])
+ for i in range(1, len(tmp_desc)):
+ self._logger.info('# %s', tmp_desc[i])
+ self._logger.info('############################################################')
+
bidirectional = S.getValue('TRAFFIC')['bidir']
if not isinstance(S.getValue('TRAFFIC')['bidir'], str):
raise TypeError(
@@ -167,6 +172,7 @@ class TestCase(object):
self._traffic['l2'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L2')
self._traffic['l3'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L3')
self._traffic['l4'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L4')
+ self._traffic['l2']['dstmac'] = S.getValue('NICS')[1]['mac']
elif len(S.getValue('NICS')) and \
(S.getValue('NICS')[0]['type'] == 'vf' or
S.getValue('NICS')[1]['type'] == 'vf'):
@@ -186,8 +192,6 @@ class TestCase(object):
def run_initialize(self):
""" Prepare test execution environment
"""
- self._logger.debug(self.name)
-
# mount hugepages if needed
self._mount_hugepages()
@@ -343,7 +347,7 @@ class TestCase(object):
# ...and continue with traffic generation, but keep
# in mind, that clean deployment does not configure
# OVS nor executes the traffic
- if self.deployment != 'clean':
+ if self.deployment != 'clean' and not self._step_send_traffic:
self._traffic_ctl.send_traffic(self._traffic)
# dump vswitch flows before they are affected by VNF termination
@@ -365,23 +369,6 @@ class TestCase(object):
# report test results
self.run_report()
- # restore original settings
- for key in self._settings_original:
- S.setValue(key, self._settings_original[key])
-
- def _update_settings(self, param, value):
- """ Check value of given configuration parameter
- In case that new value is different, then testcase
- specific settings is updated and original value stored
-
- :param param: Name of parameter inside settings
- :param value: Disired parameter value
- """
- orig_value = S.getValue(param)
- if orig_value != value:
- self._settings_original[param] = copy.deepcopy(orig_value)
- S.setValue(param, value)
-
def _append_results(self, results):
"""
Method appends mandatory Test Case results to list of dictionaries.
@@ -689,18 +676,22 @@ class TestCase(object):
if self._step_vnf_list[vnf]:
self._step_vnf_list[vnf].stop()
- def step_eval_param(self, param, STEP):
- # pylint: disable=invalid-name
+ def step_eval_param(self, param, step_result):
""" Helper function for #STEP macro evaluation
"""
if isinstance(param, str):
# evaluate every #STEP reference inside parameter itself
- macros = re.findall(r'#STEP\[[\w\[\]\-\'\"]+\]', param)
+ macros = re.findall(r'(#STEP\[([\w\-:]+)\]((\[[\w\-\'\"]+\])*))', param)
+
if macros:
for macro in macros:
+ if macro[1] in self._step_result_mapping:
+ key = self._step_result_mapping[macro[1]]
+ else:
+ key = macro[1]
# pylint: disable=eval-used
- tmp_val = str(eval(macro[1:]))
- param = param.replace(macro, tmp_val)
+ tmp_val = str(eval('step_result[{}]{}'.format(key, macro[2])))
+ param = param.replace(macro[0], tmp_val)
# evaluate references to vsperf configuration options
macros = re.findall(r'\$(([\w\-]+)(\[[\w\[\]\-\'\"]+\])*)', param)
@@ -718,12 +709,12 @@ class TestCase(object):
elif isinstance(param, list) or isinstance(param, tuple):
tmp_list = []
for item in param:
- tmp_list.append(self.step_eval_param(item, STEP))
+ tmp_list.append(self.step_eval_param(item, step_result))
return tmp_list
elif isinstance(param, dict):
tmp_dict = {}
for (key, value) in param.items():
- tmp_dict[key] = self.step_eval_param(value, STEP)
+ tmp_dict[key] = self.step_eval_param(value, step_result)
return tmp_dict
else:
return param
@@ -737,6 +728,7 @@ class TestCase(object):
eval_params.append(self.step_eval_param(param, step_result))
return eval_params
+ # pylint: disable=too-many-locals, too-many-branches, too-many-statements
def step_run(self):
""" Execute actions specified by TestSteps list
@@ -758,6 +750,30 @@ class TestCase(object):
# run test step by step...
for i, step in enumerate(self.test):
step_ok = not self._step_check
+ step_check = self._step_check
+ regex = None
+ # configure step result mapping if step alias/label is detected
+ if step[0].startswith('#'):
+ key = step[0][1:]
+ if key.isdigit():
+ raise RuntimeError('Step alias can\'t be an integer value {}'.format(key))
+ if key in self._step_result_mapping:
+ raise RuntimeError('Step alias {} has been used already for step '
+ '{}'.format(key, self._step_result_mapping[key]))
+ self._step_result_mapping[step[0][1:]] = i
+ step = step[1:]
+
+ # store regex filter if it is specified
+ if isinstance(step[-1], str) and step[-1].startswith('|'):
+ # evalute macros and variables used in regex
+ regex = self.step_eval_params([step[-1][1:]], self._step_result[:i])[0]
+ step = step[:-1]
+
+ # check if step verification should be suppressed
+ if step[0].startswith('!'):
+ step_check = False
+ step_ok = True
+ step[0] = step[0][1:]
if step[0] == 'vswitch':
test_object = self._vswitch_ctl.get_vswitch()
elif step[0] == 'namespace':
@@ -777,6 +793,9 @@ class TestCase(object):
tmp_traffic = copy.deepcopy(self._traffic)
tmp_traffic.update(step[2])
step[2] = tmp_traffic
+ # store indication that traffic has been sent
+ # so it is not sent again after the execution of teststeps
+ self._step_send_traffic = True
elif step[0].startswith('vnf'):
if not self._step_vnf_list[step[0]]:
# initialize new VM
@@ -790,6 +809,15 @@ class TestCase(object):
self._logger.debug("Sleep %s seconds", step[1])
time.sleep(int(step[1]))
continue
+ elif step[0] == 'log':
+ test_object = self._logger
+ # there isn't a need for validation of log entry
+ step_check = False
+ step_ok = True
+ elif step[0] == 'pdb':
+ import pdb
+ pdb.set_trace()
+ continue
else:
self._logger.error("Unsupported test object %s", step[0])
self._step_status = {'status' : False, 'details' : ' '.join(step)}
@@ -798,7 +826,7 @@ class TestCase(object):
return False
test_method = getattr(test_object, step[1])
- if self._step_check:
+ if step_check:
test_method_check = getattr(test_object, CHECK_PREFIX + step[1])
else:
test_method_check = None
@@ -809,18 +837,24 @@ class TestCase(object):
# to support negative indexes
step_params = self.step_eval_params(step[2:], self._step_result[:i])
step_log = '{} {}'.format(' '.join(step[:2]), step_params)
+ step_log += ' filter "{}"'.format(regex) if regex else ''
self._logger.debug("Step %s '%s' start", i, step_log)
self._step_result[i] = test_method(*step_params)
+ if regex:
+ # apply regex to step output
+ self._step_result[i] = functions.filter_output(
+ self._step_result[i], regex)
+
self._logger.debug("Step %s '%s' results '%s'", i,
step_log, self._step_result[i])
time.sleep(S.getValue('TEST_STEP_DELAY'))
- if self._step_check:
+ if step_check:
step_ok = test_method_check(self._step_result[i], *step_params)
except (AssertionError, AttributeError, IndexError) as ex:
step_ok = False
self._logger.error("Step %s raised %s", i, type(ex).__name__)
- if self._step_check:
+ if step_check:
self.step_report_status("Step {} - '{}'".format(i, step_log), step_ok)
if not step_ok: