tests: Improvement of step driven testcases

A set of improvements was introduced to enhance step driven
testcases capabilities.

Details:
* delay among test steps is configurable by TEST_STEP_DELAY
  parameter
* step driven tool function exec was renamed to exec_shell
* new step driven tool function exec_python was introduced to execute
  a python code
* new step driven object sleep was introduced to pause test execution
  for defined number of seconds.
* fixed bug in settings.validate_getValue() to correctly validate
  access of parameters modified by TEST_PARAMS
* new #PARAM() macro was introduced to allow references among
  configuration parameters
* multistream support has been added into ixnetrfc2544v2.tcl,
  which is used for tunneling protocols test (op2p deployment)
* fixed bug in op2p deployment to list interfaces and flows
  from both bridges involved in the test
* test report updated to state exact rfcxxxx type of traffic
  type, e.g. rfc2544_continuous
* test report of step driven testcases was updated to contain
  measured values from traffic generator in CSV report
* method for ovs flow comparison was modified to normalize
  IPv4 CIDR network addr (e.g. 10.0.0.5/8 => 10.0.0.0/8)

JIRA: VSPERF-512

Change-Id: Ib4f38dcdfbf3820dd766b25520da0ad0c81f3293
Signed-off-by: Martin Klozik <martinx.klozik@intel.com>
Reviewed-by: Al Morton <acmorton@att.com>
Reviewed-by: Christian Trautman <ctrautma@redhat.com>
Reviewed-by: Sridhar Rao <sridhar.rao@spirent.com>
Reviewed-by: Trevor Cooper <trevor.cooper@intel.com>
Reviewed-by: Ciara Loftus <ciara.loftus@intel.com>

2 files changed, 40 insertions, 10 deletions

diff --git a/testcases/integration.py b/testcases/integration.py
index f2a5fecf..f87a8ee2 100644
--- a/testcases/integration.py
+++ b/testcases/integration.py
@@ -1,4 +1,4 @@
-# Copyright 2015-2016 Intel Corporation.
+# Copyright 2015-2017 Intel Corporation.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -36,10 +36,13 @@ class IntegrationTestCase(TestCase):
         """ Report test results
         """
         if self.test:
-            results = OrderedDict()
-            results['status'] = 'OK' if self._step_status['status'] else 'FAILED'
-            results['details'] = self._step_status['details']
-            TestCase.write_result_to_file([results], self._output_file)
+            tmp_results = OrderedDict()
+            tmp_results['status'] = 'OK' if self._step_status['status'] else 'FAILED'
+            tmp_results['details'] = self._step_status['details']
+            self._tc_results = [tmp_results]
+
+            super(IntegrationTestCase, self).run_report()
+
             self.step_report_status("Test '{}'".format(self.name), self._step_status['status'])
             # inform vsperf about testcase failure
             if not self._step_status['status']:
diff --git a/testcases/testcase.py b/testcases/testcase.py
index 75ed1a5c..1537186f 100644
--- a/testcases/testcase.py
+++ b/testcases/testcase.py
@@ -143,7 +143,7 @@ class TestCase(object):
         self._traffic = functions.check_traffic(self._traffic)
 
         # Packet Forwarding mode
-        self._vswitch_none = S.getValue('VSWITCH').strip().lower() == 'none'
+        self._vswitch_none = str(S.getValue('VSWITCH')).strip().lower() == 'none'
 
         # trafficgen configuration required for tests of tunneling protocols
         if self.deployment == "op2p":
@@ -289,8 +289,31 @@ class TestCase(object):
             self._logger.debug("Traffic Results:")
             self._traffic_ctl.print_results()
 
+        if self._tc_results is None:
             self._tc_results = self._append_results(results)
-            TestCase.write_result_to_file(self._tc_results, self._output_file)
+        else:
+            # integration step driven tests have their status and possible
+            # failure details stored inside self._tc_results
+            results = self._append_results(results)
+            if len(self._tc_results) < len(results):
+                if len(self._tc_results) > 1:
+                    raise RuntimeError('Testcase results do not match:'
+                                       'results: {}\n'
+                                       'trafficgen results: {}\n',
+                                       self._tc_results,
+                                       results)
+                else:
+                    tmp_results = copy.deepcopy(self._tc_results[0])
+                    self._tc_results = []
+                    for res in results:
+                        tmp_res = copy.deepcopy(tmp_results)
+                        tmp_res.update(res)
+                        self._tc_results.append(tmp_res)
+            else:
+                for i, result in enumerate(results):
+                    self._tc_results[i].update(result)
+
+        TestCase.write_result_to_file(self._tc_results, self._output_file)
 
     def run(self):
         """Run the test
@@ -440,7 +463,7 @@ class TestCase(object):
         # hugepages are needed by DPDK and Qemu
         if not self._hugepages_mounted and \
             (self.deployment.count('v') or \
-             S.getValue('VSWITCH').lower().count('dpdk') or \
+             str(S.getValue('VSWITCH')).lower().count('dpdk') or \
              self._vswitch_none or \
              self.test and 'vnf' in [step[0][0:3] for step in self.test]):
             hugepages.mount_hugepages()
@@ -467,7 +490,7 @@ class TestCase(object):
         # get hugepage amounts for each socket on dpdk
         sock0_mem, sock1_mem = 0, 0
 
-        if S.getValue('VSWITCH').lower().count('dpdk'):
+        if str(S.getValue('VSWITCH')).lower().count('dpdk'):
             sock_mem = S.getValue('DPDK_SOCKET_MEM')
             sock0_mem, sock1_mem = (int(sock_mem[0]) * 1024 / hugepage_size,
                                     int(sock_mem[1]) * 1024 / hugepage_size)
@@ -739,6 +762,10 @@ class TestCase(object):
                 input(os.linesep + "Step {}: Press Enter to continue with "
                       "the next step...".format(i) + os.linesep + os.linesep)
                 continue
+            elif step[0] == 'sleep':
+                self._logger.debug("Sleep %s seconds", step[1])
+                time.sleep(int(step[1]))
+                continue
             else:
                 self._logger.error("Unsupported test object %s", step[0])
                 self._step_status = {'status' : False, 'details' : ' '.join(step)}
@@ -762,7 +789,7 @@ class TestCase(object):
                 self._step_result[i] = test_method(*step_params)
                 self._logger.debug("Step %s '%s' results '%s'", i,
                                    step_log, self._step_result[i])
-                time.sleep(5)
+                time.sleep(S.getValue('TEST_STEP_DELAY'))
                 if self._step_check:
                     step_ok = test_method_check(self._step_result[i], *step_params)
             except (AssertionError, AttributeError, IndexError) as ex: