connections: Introduction of generic API

Redesign of vSwitch and vSwitch controller classes, to use generic
connection methods for configuration of vSwitch. This API
is more generic and vSwitch agnostic, thus deployment scenarios like
P2P, PVP, PVVP (i.e. PVVPx) can be used for all (currently)
supported vSwitches. Usage of new API will simplify an introduction
of new vSwitches in the future.

This patchset introduces following changes:
* OVS: implementation of add_, del_, dump_ connection(s) and their
  validation methods
* VPP: bidir parameter removed - it is up to the deployment scenario
  implementation to take care about bidirectional connections
* P2P and PXP controllers were updated to use connection methods
  instead of flow related methods. Thus standard TCs will support
  both OVS and VPP. NOTE, PVPV is not supported for VPP (yet?).
* refactoring of vSwitch interfaces and inherited classes
* VPP step driven TCs were replaced by standard TCs with appropriate
  deployment scenarios. This is for backward compatibility with
  TC reporting. Once reporting of VPP TC results into results DB will be
  modified, this TCs can be removed.
* OVS routing tables support was generalized to support P2P and
  PXP deployments and step driven TCs. Usage of OVS routing tables
  is now configurable (turned off by default) for better comparison of
  results among various vSwitches.
* Multistream pre_installed_flows feature was generalized to
  support P2P and PXP deployments and step driven TCs.
* IxNet: TRAFFIC['l4']['dstport'] will be used as a start value for port
  iteration if L4 multistream feature is enabled.
* OVS: default flow template is now configurable via OVS_FLOW_TEMPLATE
* OVS: support of TRAFFIC['flow_type']='ip' was generalized to work
  with connection methods (i.e. with P2P and PXP deployments and
  step driven TCs)
* integration TCs: modification of integration TCs and their macros to
  utilize new generic connection based API
* CI: list of TCs for VERIFY & MERGE jobs was changed to run
  the same generic tests for both OVS & VPP
* documentation update
* small fixes and improvements

JIRA: VSPERF-579

Change-Id: If4e6e6037929eab9f16c2bbcb8a0fb30e5d6f9b0
Signed-off-by: Martin Klozik <martin.klozik@tieto.com>
Reviewed-by: Richard Elias <richard.elias@tieto.com>
Reviewed-by: Al Morton <acmorton@att.com>
Reviewed-by: Christian Trautman <ctrautma@redhat.com>
Reviewed-by: Sridhar Rao <sridhar.rao@spirent.com>

1 files changed, 42 insertions, 94 deletions

diff --git a/core/vswitch_controller_pxp.py b/core/vswitch_controller_pxp.py
index e3c208a3..d36ecdba 100644
--- a/core/vswitch_controller_pxp.py
+++ b/core/vswitch_controller_pxp.py
@@ -1,4 +1,4 @@
-# Copyright 2016 Intel Corporation.
+# Copyright 2016-2018 Intel Corporation., Tieto
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,34 +14,18 @@
 
 """VSwitch controller for multi VM scenarios with serial or parallel connection
 """
-
-import logging
 import netaddr
 
 from core.vswitch_controller import IVswitchController
-from vswitches.utils import add_ports_to_flow
 from conf import settings
 
-_FLOW_TEMPLATE = {
-    'idle_timeout': '0'
-}
-
-_PROTO_TCP = 6
-_PROTO_UDP = 17
-
 class VswitchControllerPXP(IVswitchController):
     """VSwitch controller for PXP deployment scenario.
     """
     def __init__(self, deployment, vswitch_class, traffic):
-        """Initializes up the prerequisites for the PXP deployment scenario.
-
-        :vswitch_class: the vSwitch class to be used.
-        :deployment: the deployment scenario to configure
-        :traffic: dictionary with detailed traffic definition
+        """See IVswitchController for general description
         """
-        self._logger = logging.getLogger(__name__)
-        self._vswitch_class = vswitch_class
-        self._vswitch = vswitch_class()
+        super().__init__(deployment, vswitch_class, traffic)
         self._pxp_topology = 'parallel' if deployment.startswith('pvpv') else 'serial'
         if deployment == 'pvp':
             self._pxp_vm_count = 1
@@ -55,9 +39,7 @@ class VswitchControllerPXP(IVswitchController):
 
         self._deployment_scenario = deployment
 
-        self._traffic = traffic.copy()
         self._bidir = True if self._traffic['bidir'] == 'True' else False
-        self._logger.debug('Creation using %s', str(self._vswitch_class))
         self._bridge = settings.getValue('VSWITCH_BRIDGE_NAME')
 
     def setup(self):
@@ -71,8 +53,8 @@ class VswitchControllerPXP(IVswitchController):
             self._vswitch.add_switch(self._bridge)
 
             # create physical ports
-            (_, phy1_number) = self._vswitch.add_phy_port(self._bridge)
-            (_, phy2_number) = self._vswitch.add_phy_port(self._bridge)
+            (phy1, _) = self._vswitch.add_phy_port(self._bridge)
+            (phy2, _) = self._vswitch.add_phy_port(self._bridge)
 
             # create VM ports
             # initialize vport array to requested number of VMs
@@ -86,54 +68,42 @@ class VswitchControllerPXP(IVswitchController):
                 self._logger.debug('Create %s vports for %s. VM with index %s',
                                    nics_nr, vmindex + 1, vmindex)
                 for _ in range(nics_nr):
-                    (_, vport) = self._vswitch.add_vport(self._bridge)
+                    (vport, _) = self._vswitch.add_vport(self._bridge)
                     vm_ports[vmindex].append(vport)
 
-            self._vswitch.del_flow(self._bridge)
-
-            # configure flows according to the TC definition
+            # configure connections according to the TC definition
             if self._pxp_topology == 'serial':
-                flow = _FLOW_TEMPLATE.copy()
-                if self._traffic['flow_type'] == 'IP':
-                    flow.update({'dl_type':'0x0800',
-                                 'nw_src':self._traffic['l3']['srcip'],
-                                 'nw_dst':self._traffic['l3']['dstip']})
+                # NOTE: all traffic from VMs is sent to other ports directly
+                # without applying traffic options to avoid issues with MAC swapping
+                # and upper layer mods performed inside guests
 
-                # insert flows for phy ports first
+                # insert connections for phy ports first
                 # from 1st PHY to 1st vport of 1st VM
-                self._add_flow(flow,
-                               phy1_number,
-                               vm_ports[0][0],
-                               self._bidir)
+                self._vswitch.add_connection(self._bridge, phy1, vm_ports[0][0], self._traffic)
+                self._vswitch.add_connection(self._bridge, vm_ports[0][0], phy1)
                 # from last vport of last VM to 2nd phy
-                self._add_flow(flow,
-                               vm_ports[self._pxp_vm_count-1][-1],
-                               phy2_number,
-                               self._bidir)
+                self._vswitch.add_connection(self._bridge, vm_ports[self._pxp_vm_count-1][-1], phy2)
+                self._vswitch.add_connection(self._bridge, phy2, vm_ports[self._pxp_vm_count-1][-1], self._traffic)
 
                 # add serial connections among VMs and VM NICs pairs if needed
                 # in case of multiple NICs pairs per VM, the pairs are chained
-                # first, before flow to the next VM is created
+                # first, before connection to the next VM is created
                 for vmindex in range(self._pxp_vm_count):
                     # connect VMs NICs pairs in case of 4 and more NICs per VM
                     connections = [(vm_ports[vmindex][2*(x+1)-1],
                                     vm_ports[vmindex][2*(x+1)])
                                    for x in range(int(len(vm_ports[vmindex])/2)-1)]
                     for connection in connections:
-                        self._add_flow(flow,
-                                       connection[0],
-                                       connection[1],
-                                       self._bidir)
+                        self._vswitch.add_connection(self._bridge, connection[0], connection[1])
+                        self._vswitch.add_connection(self._bridge, connection[1], connection[0])
                     # connect last NICs to the next VM if there is any
                     if self._pxp_vm_count > vmindex + 1:
-                        self._add_flow(flow,
-                                       vm_ports[vmindex][-1],
-                                       vm_ports[vmindex+1][0],
-                                       self._bidir)
+                        self._vswitch.add_connection(self._bridge, vm_ports[vmindex][-1], vm_ports[vmindex+1][0])
+                        self._vswitch.add_connection(self._bridge, vm_ports[vmindex+1][0], vm_ports[vmindex][-1])
             else:
-                proto = _PROTO_TCP if self._traffic['l3']['proto'].lower() == 'tcp' else _PROTO_UDP
-                dst_mac_value = netaddr.EUI(self._traffic['l2']['dstmac']).value
-                dst_ip_value = netaddr.IPAddress(self._traffic['l3']['dstip']).value
+                mac_value = netaddr.EUI(self._traffic['l2']['dstmac']).value
+                ip_value = netaddr.IPAddress(self._traffic['l3']['dstip']).value
+                port_value = self._traffic['l4']['dstport']
                 # initialize stream index; every NIC pair of every VM uses unique stream
                 stream = 0
                 for vmindex in range(self._pxp_vm_count):
@@ -146,31 +116,33 @@ class VswitchControllerPXP(IVswitchController):
                         port_pairs = [(vm_ports[vmindex][0], vm_ports[vmindex][0])]
 
                     for port_pair in port_pairs:
-                        flow_p = _FLOW_TEMPLATE.copy()
-                        flow_v = _FLOW_TEMPLATE.copy()
-
-                        # update flow based on trafficgen settings
+                        # override traffic options to ensure, that traffic is
+                        # dispatched among VMs connected in parallel
+                        options = {'multistream':1,
+                                   'stream_type':self._traffic['stream_type'],
+                                   'pre_installed_flows':'Yes'}
+                        # update connection based on trafficgen settings
                         if self._traffic['stream_type'] == 'L2':
-                            tmp_mac = netaddr.EUI(dst_mac_value + stream)
+                            tmp_mac = netaddr.EUI(mac_value + stream)
                             tmp_mac.dialect = netaddr.mac_unix_expanded
-                            flow_p.update({'dl_dst':tmp_mac})
+                            options.update({'l2':{'dstmac':tmp_mac}})
                         elif self._traffic['stream_type'] == 'L3':
-                            tmp_ip = netaddr.IPAddress(dst_ip_value + stream)
-                            flow_p.update({'dl_type':'0x0800', 'nw_dst':tmp_ip})
+                            tmp_ip = netaddr.IPAddress(ip_value + stream)
+                            options.update({'l3':{'dstip':tmp_ip}})
                         elif self._traffic['stream_type'] == 'L4':
-                            flow_p.update({'dl_type':'0x0800', 'nw_proto':proto, 'tp_dst':stream})
+                            options.update({'l3':{'proto':self._traffic['l3']['proto']}})
+                            options.update({'l4':{'dstport':(port_value + stream) % 65536}})
                         else:
                             raise RuntimeError('Unknown stream_type {}'.format(self._traffic['stream_type']))
 
-                        # insert flow to dispatch traffic from physical ports
+                        # insert connection to dispatch traffic from physical ports
                         # to VMs based on stream type; all traffic from VMs is
                         # sent to physical ports to avoid issues with MAC swapping
                         # and upper layer mods performed inside guests
-                        self._add_flow(flow_p, phy1_number, port_pair[0])
-                        self._add_flow(flow_v, port_pair[1], phy2_number)
-                        if self._bidir:
-                            self._add_flow(flow_p, phy2_number, port_pair[1])
-                            self._add_flow(flow_v, port_pair[0], phy1_number)
+                        self._vswitch.add_connection(self._bridge, phy1, port_pair[0], options)
+                        self._vswitch.add_connection(self._bridge, port_pair[1], phy2)
+                        self._vswitch.add_connection(self._bridge, phy2, port_pair[1], options)
+                        self._vswitch.add_connection(self._bridge, port_pair[0], phy1)
 
                         # every NIC pair needs its own unique traffic stream
                         stream += 1
@@ -185,37 +157,13 @@ class VswitchControllerPXP(IVswitchController):
         self._logger.debug('Stop using %s', str(self._vswitch_class))
         self._vswitch.stop()
 
-    def _add_flow(self, flow, port1, port2, reverse_flow=False):
-        """ Helper method to insert flow into the vSwitch
-        """
-        self._vswitch.add_flow(self._bridge,
-                               add_ports_to_flow(flow,
-                                                 port1,
-                                                 port2))
-        if reverse_flow:
-            self._vswitch.add_flow(self._bridge,
-                                   add_ports_to_flow(flow,
-                                                     port2,
-                                                     port1))
-
-    def __enter__(self):
-        self.setup()
-
-    def __exit__(self, type_, value, traceback):
-        self.stop()
-
-    def get_vswitch(self):
-        """See IVswitchController for description
-        """
-        return self._vswitch
-
     def get_ports_info(self):
         """See IVswitchController for description
         """
         self._logger.debug('get_ports_info  using %s', str(self._vswitch_class))
         return self._vswitch.get_ports(self._bridge)
 
-    def dump_vswitch_flows(self):
+    def dump_vswitch_connections(self):
         """See IVswitchController for description
         """
-        self._vswitch.dump_flows(self._bridge)
+        self._vswitch.dump_connections(self._bridge)