teststeps: Improvements and bugfixing of teststeps

This patch introduces several improvements and small bugfixes
of teststeps. These changes were identified during implementation
of OVS/DPDK regression tests.

Patch content:
* teststeps: step aliases were implemented
* teststeps: improved filtering by regex for any step, which returns string
  or list of stings; filter will process all lines
* teststeps: support for log object
* teststeps: support for trafficgen get_results call
* teststeps: configurable suppression of step validation
* trafficgen: remove old results before traffic is executed
* trafficgen: support for flow control on/off (IxNet)
* trafficgen: support for configurable learning frames (IxNet)
* trafficgen: support for runtime changes of TRAFFICGEN_PKT_SIZES, _DURATION
  and _LOSSRATE
* vnf: flush pexpect output of previous commands
* vnf: use execute_and_wait() to ensure correct cmds order
* vnf: dpdk vHost User interface name set according to its type,
  e.g. dpdkvhostuserclient
* vswitch: support for OVS restart
* decap: simplify configuration of tunneling decapsulation tests
* settings: values of all configuration options are restored after TC execution
* modified formatting of test description used by --list
* testcase name and description is logged before its execution
* small bugfixes

JIRA: VSPERF-539

Change-Id: I550ba0d897ece89abd3f33d6d66f545c4d863e7b
Signed-off-by: Martin Klozik <martinx.klozik@intel.com>
Reviewed-by: Al Morton <acmorton@att.com>
Reviewed-by: Christian Trautman <ctrautma@redhat.com>
Reviewed-by: Sridhar Rao <sridhar.rao@spirent.com>
Reviewed-by: Trevor Cooper <trevor.cooper@intel.com>

3 files changed, 92 insertions, 73 deletions

diff --git a/vnfs/qemu/qemu.py b/vnfs/qemu/qemu.py
index 30f073ee..8e3d44de 100644
--- a/vnfs/qemu/qemu.py
+++ b/vnfs/qemu/qemu.py
@@ -146,13 +146,14 @@ class IVnfQemu(IVnf):
         """
         if self.is_running():
             try:
-                # exit testpmd if needed
-                if self._guest_loopback == 'testpmd':
-                    self.execute_and_wait('stop', 120, "Done")
-                    self.execute_and_wait('quit', 120, "[bB]ye")
+                if self._login_active:
+                    # exit testpmd if needed
+                    if self._guest_loopback == 'testpmd':
+                        self.execute_and_wait('stop', 120, "Done")
+                        self.execute_and_wait('quit', 120, "[bB]ye")
 
-                # turn off VM
-                self.execute_and_wait('poweroff', 120, "Power down")
+                    # turn off VM
+                    self.execute_and_wait('poweroff', 120, "Power down")
 
             except pexpect.TIMEOUT:
                 self.kill()
@@ -169,7 +170,7 @@ class IVnfQemu(IVnf):
 
     # helper functions
 
-    def _login(self, timeout=120):
+    def login(self, timeout=120):
         """
         Login to QEMU instance.
 
@@ -178,8 +179,11 @@ class IVnfQemu(IVnf):
 
         :param timeout: Timeout to wait for login to complete.
 
-        :returns: None
+        :returns: True if login is active
         """
+        if self._login_active:
+            return self._login_active
+
         # if no timeout was set, we likely started QEMU without waiting for it
         # to boot. This being the case, we best check that it has finished
         # first.
@@ -191,21 +195,8 @@ class IVnfQemu(IVnf):
         self._child.sendline(S.getValue('GUEST_PASSWORD')[self._number])
 
         self._expect_process(S.getValue('GUEST_PROMPT')[self._number], timeout=5)
-
-    def send_and_pass(self, cmd, timeout=30):
-        """
-        Send ``cmd`` and wait ``timeout`` seconds for it to pass.
-
-        :param cmd: Command to send to guest.
-        :param timeout: Time to wait for prompt before checking return code.
-
-        :returns: None
-        """
-        self.execute(cmd)
-        self.wait(S.getValue('GUEST_PROMPT')[self._number], timeout=timeout)
-        self.execute('echo $?')
-        self._child.expect('^0$', timeout=1)  # expect a 0
-        self.wait(S.getValue('GUEST_PROMPT')[self._number], timeout=timeout)
+        self._login_active = True
+        return self._login_active
 
     def _affinitize(self):
         """
@@ -277,29 +268,31 @@ class IVnfQemu(IVnf):
         """
         Configure VM to run VNF, e.g. port forwarding application based on the configuration
         """
+        if self._guest_loopback == 'buildin':
+            return
+
+        self.login()
+
         if self._guest_loopback == 'testpmd':
-            self._login()
             self._configure_testpmd()
         elif self._guest_loopback == 'l2fwd':
-            self._login()
             self._configure_l2fwd()
         elif self._guest_loopback == 'linux_bridge':
-            self._login()
             self._configure_linux_bridge()
-        elif self._guest_loopback != 'buildin':
-            self._logger.error('Unsupported guest loopback method "%s" was specified. Option'
-                               ' "buildin" will be used as a fallback.', self._guest_loopback)
+        elif self._guest_loopback != 'clean':
+            raise RuntimeError('Unsupported guest loopback method "%s" was specified.',
+                               self._guest_loopback)
 
     def wait(self, prompt=None, timeout=30):
         if prompt is None:
             prompt = S.getValue('GUEST_PROMPT')[self._number]
-        super(IVnfQemu, self).wait(prompt=prompt, timeout=timeout)
+        return super(IVnfQemu, self).wait(prompt=prompt, timeout=timeout)
 
     def execute_and_wait(self, cmd, timeout=30, prompt=None):
         if prompt is None:
             prompt = S.getValue('GUEST_PROMPT')[self._number]
-        super(IVnfQemu, self).execute_and_wait(cmd, timeout=timeout,
-                                               prompt=prompt)
+        return super(IVnfQemu, self).execute_and_wait(cmd, timeout=timeout,
+                                                      prompt=prompt)
 
     def _modify_dpdk_makefile(self):
         """
@@ -393,7 +386,7 @@ class IVnfQemu(IVnf):
                 'VSWITCH_JUMBO_FRAMES_SIZE'))
 
         self.execute_and_wait('./testpmd {}'.format(testpmd_params), 60, "Done")
-        self.execute('set fwd ' + self._testpmd_fwd_mode, 1)
+        self.execute_and_wait('set fwd ' + self._testpmd_fwd_mode, 20, 'testpmd>')
         self.execute_and_wait('start', 20, 'testpmd>')
 
     def _configure_l2fwd(self):
@@ -407,12 +400,12 @@ class IVnfQemu(IVnf):
 
         # configure all interfaces
         for nic in self._nics:
-            self.execute('ip addr add ' +
-                         nic['ip'] + ' dev ' + nic['device'])
+            self.execute_and_wait('ip addr add ' +
+                                  nic['ip'] + ' dev ' + nic['device'])
             if S.getValue('VSWITCH_JUMBO_FRAMES_ENABLED'):
-                self.execute('ifconfig {} mtu {}'.format(
+                self.execute_and_wait('ifconfig {} mtu {}'.format(
                     nic['device'], S.getValue('VSWITCH_JUMBO_FRAMES_SIZE')))
-            self.execute('ip link set dev ' + nic['device'] + ' up')
+            self.execute_and_wait('ip link set dev ' + nic['device'] + ' up')
 
         # build and configure system for l2fwd
         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
@@ -437,43 +430,42 @@ class IVnfQemu(IVnf):
         self._configure_disable_firewall()
 
         # configure linux bridge
-        self.execute('brctl addbr br0')
+        self.execute_and_wait('brctl addbr br0')
 
         # add all NICs into the bridge
         for nic in self._nics:
-            self.execute('ip addr add ' +
-                         nic['ip'] + ' dev ' + nic['device'])
+            self.execute_and_wait('ip addr add ' + nic['ip'] + ' dev ' + nic['device'])
             if S.getValue('VSWITCH_JUMBO_FRAMES_ENABLED'):
-                self.execute('ifconfig {} mtu {}'.format(
+                self.execute_and_wait('ifconfig {} mtu {}'.format(
                     nic['device'], S.getValue('VSWITCH_JUMBO_FRAMES_SIZE')))
-            self.execute('ip link set dev ' + nic['device'] + ' up')
-            self.execute('brctl addif br0 ' + nic['device'])
+            self.execute_and_wait('ip link set dev ' + nic['device'] + ' up')
+            self.execute_and_wait('brctl addif br0 ' + nic['device'])
 
-        self.execute('ip addr add ' +
-                     S.getValue('GUEST_BRIDGE_IP')[self._number] +
-                     ' dev br0')
-        self.execute('ip link set dev br0 up')
+        self.execute_and_wait('ip addr add {} dev br0'.format(
+            S.getValue('GUEST_BRIDGE_IP')[self._number]))
+        self.execute_and_wait('ip link set dev br0 up')
 
         # Add the arp entries for the IXIA ports and the bridge you are using.
         # Use command line values if provided.
         trafficgen_mac = S.getValue('VANILLA_TGEN_PORT1_MAC')
         trafficgen_ip = S.getValue('VANILLA_TGEN_PORT1_IP')
 
-        self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
+        self.execute_and_wait('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 
         trafficgen_mac = S.getValue('VANILLA_TGEN_PORT2_MAC')
         trafficgen_ip = S.getValue('VANILLA_TGEN_PORT2_IP')
 
-        self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
+        self.execute_and_wait('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 
         # Enable forwarding
-        self.execute('sysctl -w net.ipv4.ip_forward=1')
+        self.execute_and_wait('sysctl -w net.ipv4.ip_forward=1')
 
         # Controls source route verification
         # 0 means no source validation
-        self.execute('sysctl -w net.ipv4.conf.all.rp_filter=0')
+        self.execute_and_wait('sysctl -w net.ipv4.conf.all.rp_filter=0')
         for nic in self._nics:
-            self.execute('sysctl -w net.ipv4.conf.' + nic['device'] + '.rp_filter=0')
+            self.execute_and_wait('sysctl -w net.ipv4.conf.' + nic['device'] +
+                                  '.rp_filter=0')
 
     def _bind_dpdk_driver(self, driver, pci_slots):
         """
diff --git a/vnfs/qemu/qemu_dpdk_vhost_user.py b/vnfs/qemu/qemu_dpdk_vhost_user.py
index 93147838..b53af6fa 100644
--- a/vnfs/qemu/qemu_dpdk_vhost_user.py
+++ b/vnfs/qemu/qemu_dpdk_vhost_user.py
@@ -68,11 +68,14 @@ class QemuDpdkVhostUser(IVnfQemu):
             else:
                 vhost_folder = S.getValue('TOOLS')['ovs_var_tmp']
 
-            nic_mode = '' if S.getValue('VSWITCH_VHOSTUSER_SERVER_MODE') else ',server'
+            if S.getValue('VSWITCH_VHOSTUSER_SERVER_MODE'):
+                nic_name = 'dpdkvhostuser' + ifi
+            else:
+                nic_name = 'dpdkvhostuserclient' + ifi + ',server'
+
             self._cmd += ['-chardev',
                           'socket,id=char' + ifi +
-                          ',path=' + vhost_folder +
-                          'dpdkvhostuser' + ifi + nic_mode,
+                          ',path=' + vhost_folder + nic_name,
                           '-netdev',
                           'type=vhost-user,id=' + net +
                           ',chardev=char' + ifi + ',vhostforce' + queue_str,
diff --git a/vnfs/vnf/vnf.py b/vnfs/vnf/vnf.py
index 759bdd66..5ac2ada3 100644
--- a/vnfs/vnf/vnf.py
+++ b/vnfs/vnf/vnf.py
@@ -17,6 +17,7 @@ Interface for VNF.
 """
 
 import time
+import pexpect
 from tools import tasks
 
 class IVnf(tasks.Process):
@@ -40,14 +41,7 @@ class IVnf(tasks.Process):
                            self._number + 1, self._number)
         IVnf._number_vnfs = IVnf._number_vnfs + 1
         self._log_prefix = 'vnf_%d_cmd : ' % self._number
-
-    def start(self):
-        """
-        Starts VNF instance.
-
-        This is a blocking function
-        """
-        super(IVnf, self).start()
+        self._login_active = False
 
     def stop(self):
         """
@@ -60,6 +54,18 @@ class IVnf(tasks.Process):
             # sporadic reboot of host. (caused by hugepages or DPDK ports)
             super(IVnf, self).kill(signal='-9', sleep=10)
 
+    def login(self, dummy_timeout=120):
+        """
+        Login to GUEST instance.
+
+        This can be used after booting the machine
+
+        :param timeout: Timeout to wait for login to complete.
+
+        :returns: True if login is active
+        """
+        raise NotImplementedError()
+
     def execute(self, cmd, delay=0):
         """
         execute ``cmd`` with given ``delay``.
@@ -77,6 +83,14 @@ class IVnf(tasks.Process):
         :returns: None.
         """
         self._logger.debug('%s%s', self._log_prefix, cmd)
+
+        # ensure that output from previous commands is flushed
+        try:
+            while not self._child.expect(r'.+', timeout=0.1):
+                pass
+        except (pexpect.TIMEOUT, pexpect.EOF):
+            pass
+
         self._child.sendline(cmd)
         time.sleep(delay)
 
@@ -95,10 +109,10 @@ class IVnf(tasks.Process):
                         Please note that this value can be floating
                         point which allows to pass milliseconds.
 
-        :returns: True if result_cmd has been detected before
-                  timeout has been reached, False otherwise.
+        :returns: output of executed command
         """
         self._child.expect(prompt, timeout=timeout)
+        return self._child.before
 
     def execute_and_wait(self, cmd, timeout=30, prompt=''):
         """
@@ -119,27 +133,37 @@ class IVnf(tasks.Process):
                              ``prompt`` passed in __init__
                              method will be used.
 
-        :returns: True if end of execution has been detected
-                  before timeout has been reached, False otherwise.
+        :returns: output of executed command
         """
         self.execute(cmd)
-        self.wait(prompt=prompt, timeout=timeout)
+        return self.wait(prompt=prompt, timeout=timeout)
 
-    # pylint: disable=simplifiable-if-statement
-    def validate_start(self, dummy_result):
+    def validate_start(self, dummyresult):
         """ Validate call of VNF start()
         """
         if self._child and self._child.isalive():
             return True
-        else:
-            return False
+
+        return False
 
     def validate_stop(self, result):
-        """ Validate call of fVNF stop()
+        """ Validate call of VNF stop()
         """
         return not self.validate_start(result)
 
     @staticmethod
+    def validate_execute_and_wait(result, dummy_cmd, dummy_timeout=30, dummy_prompt=''):
+        """ Validate command execution within VNF
+        """
+        return len(result) > 0
+
+    @staticmethod
+    def validate_login(result):
+        """ Validate successful login into guest
+        """
+        return result
+
+    @staticmethod
     def reset_vnf_counter():
         """
         Reset internal VNF counters