Add L3 traffic management with Neutron routers

Change-Id: Ic9bff87e0d78652de28b3a756f9ebc342983cfbb
Signed-off-by: fmenguy <francoisregis.menguy@orange.com>

16 files changed, 641 insertions, 81 deletions

diff --git a/docs/development/design/traffic_desc.rst b/docs/development/design/traffic_desc.rst
index cd80a1c..bbd31a6 100644
--- a/docs/development/design/traffic_desc.rst
+++ b/docs/development/design/traffic_desc.rst
@@ -37,6 +37,7 @@ The destination MAC address is based on the configuration and can be:
   or when using a loopback cable
 - the dest MAC as specified by the configuration file (EXT chain no ARP)
 - the dest MAC as discovered by ARP (EXT chain)
+- the router MAC as discovered from Neutron API (PVPL3 chain)
 - the VM MAC as dicovered from Neutron API (PVP, PVVP chains)
 
 NFVbench does not currently range on the MAC addresses.
diff --git a/docs/testing/user/userguide/advanced.rst b/docs/testing/user/userguide/advanced.rst
index 1a6e999..e49cfab 100644
--- a/docs/testing/user/userguide/advanced.rst
+++ b/docs/testing/user/userguide/advanced.rst
@@ -217,7 +217,7 @@ For example to run NFVbench with 3 PVP chains:
 
 It is not necessary to specify the service chain type (-sc) because PVP is set as default. The PVP service chains will have 3 VMs in 3 chains with this configuration.
 If ``-sc PVVP`` is specified instead, there would be 6 VMs in 3 chains as this service chain has 2 VMs per chain.
-Both **single run** or **NDR/PDR** can be run as multichain. Runnin multichain is a scenario closer to a real life situation than runs with a single chain.
+Both **single run** or **NDR/PDR** can be run as multichain. Running multichain is a scenario closer to a real life situation than runs with a single chain.
 
 
 Multiflow
diff --git a/docs/testing/user/userguide/images/nfvbench-pvpl3.png b/docs/testing/user/userguide/images/nfvbench-pvpl3.png
new file mode 100644
index 0000000..d583724
--- /dev/null
+++ b/docs/testing/user/userguide/images/nfvbench-pvpl3.png
diff --git a/docs/testing/user/userguide/index.rst b/docs/testing/user/userguide/index.rst
index e83912f..84c79b0 100644
--- a/docs/testing/user/userguide/index.rst
+++ b/docs/testing/user/userguide/index.rst
@@ -24,6 +24,7 @@ Table of Content
    installation
    examples
    advanced
+   pvpl3
    extchains
    fluentd
    sriov
diff --git a/docs/testing/user/userguide/pvpl3.rst b/docs/testing/user/userguide/pvpl3.rst
new file mode 100644
index 0000000..12f1d86
--- /dev/null
+++ b/docs/testing/user/userguide/pvpl3.rst
@@ -0,0 +1,66 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International License.
+.. SPDX-License-Identifier: CC-BY-4.0
+.. (c) Cisco Systems, Inc
+
+
+PVP L3 Router Internal Chain
+--------------
+
+NFVbench can measure the performance of 1 L3 service chain that are setup by NFVbench (VMs, routers and networks).
+
+PVP L3 router chain is made of 1 VNF (in vpp mode) and has exactly 2 end network interfaces (left and right internal network interfaces) that are connected to 2 neutron routers with 2 edge networks (left and right edge networks).
+The PVP L3 router service chain can route L3 packets properly between the left and right networks.
+
+To run NFVbench on such PVP L3 router service chain:
+
+- explicitly tell NFVbench to use PVP service chain with L3 router option by adding ``-l3`` or ``--l3-router`` to NFVbench CLI options or ``l3_router: true`` in config
+- explicitly tell NFVbench to use VPP forwarder with ``vm_forwarder: vpp`` in config
+- specify the 2 end point networks (networks between NFVBench and neutron routers) of your environment in ``internal_networks`` inside the config file.
+    - The two networks specified will be created if not existing in Neutron and will be used as the end point networks by NFVbench ('lyon' and 'bordeaux' in the diagram below)
+- specify the 2 edge networks (networks between neutron routers and loopback VM) of your environment in ``edge_networks`` inside the config file.
+    - The two networks specified will be created if not existing in Neutron and will be used as the router gateway networks by NFVbench ('paris' and 'marseille' in the diagram below)
+- specify the router gateway IPs for the PVPL3 router service chain (1.2.0.1 and 2.2.0.1)
+- specify the traffic generator gateway IPs for the PVPL3 router service chain (1.2.0.254 and 2.2.0.254 in diagram below)
+- specify the packet source and destination IPs for the virtual devices that are simulated (10.0.0.0/8 and 20.0.0.0/8)
+
+
+.. image:: images/nfvbench-pvpl3.png
+
+nfvbench configuration file:
+
+.. code-block:: bash
+
+    vm_forwarder: vpp
+
+    traffic_generator:
+        ip_addrs: ['10.0.0.0/8', '20.0.0.0/8']
+        tg_gateway_ip_addrs: ['1.2.0.254', '2.2.0.254']
+        gateway_ip_addrs: ['1.2.0.1', '2.2.0.1']
+
+    internal_networks:
+        left:
+            name: 'lyon'
+            cidr: '1.2.0.0/24'
+            gateway: '1.2.0.1'
+        right:
+            name: 'bordeaux'
+            cidr: '2.2.0.0/24'
+            gateway: '2.2.0.1'
+
+    edge_networks:
+        left:
+            name: 'paris'
+            cidr: '1.1.0.0/24'
+            gateway: '1.1.0.1'
+        right:
+            name: 'marseille'
+            cidr: '2.1.0.0/24'
+            gateway: '2.1.0.1'
+
+Upon start, NFVbench will:
+- first retrieve the properties of the left and right networks using Neutron APIs,
+- extract the underlying network ID (typically VLAN segmentation ID),
+- generate packets with the proper VLAN ID and measure traffic.
+
+
+Please note: ``l3_router`` option is also compatible with external routers. In this case NFVBench will use ``EXT`` chain.
\ No newline at end of file
diff --git a/docs/testing/user/userguide/readme.rst b/docs/testing/user/userguide/readme.rst
index acd4763..48c8b02 100644
--- a/docs/testing/user/userguide/readme.rst
+++ b/docs/testing/user/userguide/readme.rst
@@ -175,6 +175,8 @@ P2P (Physical interface to Physical interface - no VM) can be supported using th
 
 V2V (VM to VM) is not supported but PVVP provides a more complete (and more realistic) alternative.
 
+PVP chain with L3 routers in the path can be supported using PVP chain with L3 forwarding mode (l3_router option). See PVP L3 Router Internal Chain section for more details.
+
 
 Supported Neutron Network Plugins and vswitches
 -----------------------------------------------
diff --git a/nfvbench/cfg.default.yaml b/nfvbench/cfg.default.yaml
index b2b9f49..0d6edd8 100755
--- a/nfvbench/cfg.default.yaml
+++ b/nfvbench/cfg.default.yaml
@@ -162,6 +162,7 @@ traffic_generator:
     #                       chain count consecutive IP addresses spaced by tg_gateway_ip_addrs_step will be used
     # `tg_gateway_ip_addrs__step`: step for generating traffic generator gateway sequences. default is 0.0.0.1
     tg_gateway_ip_addrs: ['1.1.0.100', '2.2.0.100']
+    tg_gateway_ip_cidrs: ['1.1.0.0/24','2.2.0.0/24']
     tg_gateway_ip_addrs_step: 0.0.0.1
     # `gateway_ip_addrs`: base IPs of VNF router gateways (left and right), quantity used depends on chain count
     #                     must correspond to the public IP on the left and right networks
@@ -465,6 +466,40 @@ external_networks:
     left:
     right:
 
+# PVP with L3 router in the packet path only.
+# Only use when l3_router option is True (see l3_router)
+# Prefix names of edge networks which will be used to send traffic via traffic generator.
+# If a network with given name already exists it will be reused.
+# Otherwise a new edge network will be created with that name, subnet and CIDR.
+#
+# gateway can be set in case of L3 traffic with edge networks - refer to edge_networks
+#
+# segmentation_id can be set to enforce a specific VLAN id - by default (empty) the VLAN id
+#                 will be assigned by Neutron.
+#                 Must be unique for each network
+# physical_network can be set to pick a specific phsyical network - by default (empty) the
+#                   default physical network will be picked
+#
+edge_networks:
+    left:
+        name: 'nfvbench-net2'
+        router_name: 'router_left'
+        subnet: 'nfvbench-subnet2'
+        cidr: '192.168.3.0/24'
+        gateway:
+        network_type:
+        segmentation_id:
+        physical_network:
+    right:
+        name: 'nfvbench-net3'
+        router_name: 'router_right'
+        subnet: 'nfvbench-subnet3'
+        cidr: '192.168.4.0/24'
+        gateway:
+        network_type:
+        segmentation_id:
+        physical_network:
+
 # Use 'true' to enable VXLAN encapsulation support and sent by the traffic generator
 # When this option enabled internal networks 'network type' parameter value should be 'vxlan'
 vxlan: false
@@ -525,6 +560,11 @@ traffic:
 # Can be overriden by --no-traffic
 no_traffic: false
 
+# Use an L3 router in the packet path. This option if set will create or reuse an openstack neutron
+# router (PVP, PVVP) or reuse an existing L3 router (EXT) to route traffic to the destination VM.
+# Can be overriden by --l3-router
+l3_router: false
+
 # Test configuration
 
 # The rate pps for traffic going in reverse direction in case of unidirectional flow. Default to 1.
diff --git a/nfvbench/chain_router.py b/nfvbench/chain_router.py
new file mode 100644
index 0000000..9372716
--- /dev/null
+++ b/nfvbench/chain_router.py
@@ -0,0 +1,186 @@
+#!/usr/bin/env python
+# Copyright 2018 Cisco Systems, Inc.  All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+# This module takes care of chaining routers
+#
+"""NFVBENCH CHAIN DISCOVERY/STAGING.
+
+This module takes care of staging/discovering resources that are participating in a
+L3 benchmarking session: routers, networks, ports, routes.
+If a resource is discovered with the same name, it will be reused.
+Otherwise it will be created.
+
+Once created/discovered, instances are checked to be in the active state (ready to pass traffic)
+Configuration parameters that will influence how these resources are staged/related:
+- openstack or no openstack
+- chain type
+- number of chains
+- number of VNF in each chain (PVP, PVVP)
+- SRIOV and middle port SRIOV for port types
+- whether networks are shared across chains or not
+
+There is not traffic generation involved in this module.
+"""
+import time
+
+from netaddr import IPAddress
+from netaddr import IPNetwork
+
+from log import LOG
+
+
+class ChainException(Exception):
+    """Exception while operating the chains."""
+
+    pass
+
+
+class ChainRouter(object):
+    """Could be a shared router across all chains or a chain private router."""
+
+    def __init__(self, manager, name, subnets, routes):
+        """Create a router for given chain."""
+        self.manager = manager
+        self.subnets = subnets
+        self.routes = routes
+        self.name = name
+        self.ports = [None, None]
+        self.reuse = False
+        self.router = None
+        try:
+            self._setup()
+        except Exception:
+            LOG.error("Error creating router %s", self.name)
+            self.delete()
+            raise
+
+    def _setup(self):
+        # Lookup if there is a matching router with same name
+        routers = self.manager.neutron_client.list_routers(name=self.name)
+
+        if routers['routers']:
+            router = routers['routers'][0]
+            # a router of same name already exists, we need to verify it has the same
+            # characteristics
+            if self.subnets:
+                for subnet in self.subnets:
+                    if not self.get_router_interface(router['id'], subnet.network['subnets'][0]):
+                        raise ChainException("Mismatch of 'subnet_id' for reused "
+                                             "router '{router}'.Router has no subnet id '{sub_id}'."
+                                             .format(router=self.name,
+                                                     sub_id=subnet.network['subnets'][0]))
+                interfaces = self.manager.neutron_client.list_ports(device_id=router['id'])['ports']
+                for interface in interfaces:
+                    if self.is_ip_in_network(
+                            interface['fixed_ips'][0]['ip_address'],
+                            self.manager.config.traffic_generator.tg_gateway_ip_cidrs[0]) \
+                        or self.is_ip_in_network(
+                            interface['fixed_ips'][0]['ip_address'],
+                            self.manager.config.traffic_generator.tg_gateway_ip_cidrs[1]):
+                        self.ports[0] = interface
+                    else:
+                        self.ports[1] = interface
+            if self.routes:
+                for route in self.routes:
+                    if route not in router['routes']:
+                        LOG.info("Mismatch of 'router' for reused router '%s'."
+                                 "Router has no existing route destination '%s', "
+                                 "and nexthop '%s'.", self.name,
+                                 route['destination'],
+                                 route['nexthop'])
+                        LOG.info("New route added to router %s for reused ", self.name)
+                        body = {
+                            'router': {
+                                'routes': self.routes
+                            }
+                        }
+                        self.manager.neutron_client.update_router(router['id'], body)
+
+            LOG.info('Reusing existing router: %s', self.name)
+            self.reuse = True
+            self.router = router
+            return
+
+        body = {
+            'router': {
+                'name': self.name,
+                'admin_state_up': True
+            }
+        }
+        router = self.manager.neutron_client.create_router(body)['router']
+        router_id = router['id']
+
+        if self.subnets:
+            for subnet in self.subnets:
+                router_interface = {'subnet_id': subnet.network['subnets'][0]}
+                self.manager.neutron_client.add_interface_router(router_id, router_interface)
+            interfaces = self.manager.neutron_client.list_ports(device_id=router_id)['ports']
+            for interface in interfaces:
+                itf = interface['fixed_ips'][0]['ip_address']
+                cidr0 = self.manager.config.traffic_generator.tg_gateway_ip_cidrs[0]
+                cidr1 = self.manager.config.traffic_generator.tg_gateway_ip_cidrs[1]
+                if self.is_ip_in_network(itf, cidr0) or self.is_ip_in_network(itf, cidr1):
+                    self.ports[0] = interface
+                else:
+                    self.ports[1] = interface
+
+        if self.routes:
+            body = {
+                'router': {
+                    'routes': self.routes
+                }
+            }
+            self.manager.neutron_client.update_router(router_id, body)
+
+        LOG.info('Created router: %s.', self.name)
+        self.router = self.manager.neutron_client.show_router(router_id)
+
+    def get_uuid(self):
+        """
+        Extract UUID of this router.
+
+        :return: UUID of this router
+        """
+        return self.router['id']
+
+    def get_router_interface(self, router_id, subnet_id):
+        interfaces = self.manager.neutron_client.list_ports(device_id=router_id)['ports']
+        matching_interface = None
+        for interface in interfaces:
+            if interface['fixed_ips'][0]['subnet_id'] == subnet_id:
+                matching_interface = interface
+        return matching_interface
+
+    def is_ip_in_network(self, interface_ip, cidr):
+        return IPAddress(interface_ip) in IPNetwork(cidr)
+
+    def delete(self):
+        """Delete this router."""
+        if not self.reuse and self.router:
+            retry = 0
+            while retry < self.manager.config.generic_retry_count:
+                try:
+                    self.manager.neutron_client.delete_router(self.router['id'])
+                    LOG.info("Deleted router: %s", self.name)
+                    return
+                except Exception:
+                    retry += 1
+                    LOG.info('Error deleting router %s (retry %d/%d)...',
+                             self.name,
+                             retry,
+                             self.manager.config.generic_retry_count)
+                    time.sleep(self.manager.config.generic_poll_sec)
+            LOG.error('Unable to delete router: %s', self.name)
diff --git a/nfvbench/chain_runner.py b/nfvbench/chain_runner.py
index 627e9ea..833373c 100644
--- a/nfvbench/chain_runner.py
+++ b/nfvbench/chain_runner.py
@@ -78,7 +78,7 @@ class ChainRunner(object):
         # Note that in the case of EXT+ARP+VxLAN, the dest MACs need to be loaded
         # because ARP only operates on the dest VTEP IP not on the VM dest MAC
         if not config.l2_loopback and \
-           (config.service_chain != ChainType.EXT or config.no_arp or config.vxlan):
+                (config.service_chain != ChainType.EXT or config.no_arp or config.vxlan):
             gen_config.set_dest_macs(0, self.chain_manager.get_dest_macs(0))
             gen_config.set_dest_macs(1, self.chain_manager.get_dest_macs(1))
 
@@ -104,8 +104,8 @@ class ChainRunner(object):
         self.traffic_client.setup()
         if not self.config.no_traffic:
             # ARP is needed for EXT chain or VxLAN overlay unless disabled explicitly
-            if (self.config.service_chain == ChainType.EXT or self.config.vxlan) and \
-               not self.config.no_arp:
+            if (self.config.service_chain == ChainType.EXT or
+                    self.config.vxlan or self.config.l3_router) and not self.config.no_arp:
                 self.traffic_client.ensure_arp_successful()
             self.traffic_client.ensure_end_to_end()
 
diff --git a/nfvbench/chaining.py b/nfvbench/chaining.py
index 898e9ea..3350299 100644
--- a/nfvbench/chaining.py
+++ b/nfvbench/chaining.py
@@ -54,13 +54,16 @@ from neutronclient.neutron import client as neutronclient
 from novaclient.client import Client
 
 from attrdict import AttrDict
+from chain_router import ChainRouter
 import compute
 from log import LOG
 from specs import ChainType
-
 # Left and right index for network and port lists
 LEFT = 0
 RIGHT = 1
+# L3 traffic edge networks are at the end of networks list
+EDGE_LEFT = -2
+EDGE_RIGHT = -1
 # Name of the VM config file
 NFVBENCH_CFG_FILENAME = 'nfvbenchvm.conf'
 # full pathame of the VM config in the VM
@@ -76,6 +79,7 @@ class ChainException(Exception):
 
     pass
 
+
 class NetworkEncaps(object):
     """Network encapsulation."""
 
@@ -174,6 +178,10 @@ class ChainVnfPort(object):
         """Get the MAC address for this port."""
         return self.port['mac_address']
 
+    def get_ip(self):
+        """Get the IP address for this port."""
+        return self.port['fixed_ips'][0]['ip_address']
+
     def delete(self):
         """Delete this port instance."""
         if self.reuse or not self.port:
@@ -222,6 +230,8 @@ class ChainNetwork(object):
         self.reuse = False
         self.network = None
         self.vlan = None
+        if manager.config.l3_router and hasattr(network_config, 'router_name'):
+            self.router_name = network_config.router_name
         try:
             self._setup(network_config, lookup_only)
         except Exception:
@@ -294,7 +304,7 @@ class ChainNetwork(object):
                 'network': {
                     'name': self.name,
                     'admin_state_up': True
-                    }
+                }
             }
             if network_config.network_type:
                 body['network']['provider:network_type'] = network_config.network_type
@@ -388,6 +398,7 @@ class ChainVnf(object):
         # For example if 7 idle interfaces are requested, the corresp. ports will be
         # at index 2 to 8
         self.ports = []
+        self.routers = []
         self.status = None
         self.instance = None
         self.reuse = False
@@ -397,7 +408,10 @@ class ChainVnf(object):
         try:
             # the vnf_id is conveniently also the starting index in networks
             # for the left and right networks associated to this VNF
-            self._setup(networks[vnf_id:vnf_id + 2])
+            if self.manager.config.l3_router:
+                self._setup(networks[vnf_id:vnf_id + 4])
+            else:
+                self._setup(networks[vnf_id:vnf_id + 2])
         except Exception:
             LOG.error("Error creating VNF %s", self.name)
             self.delete()
@@ -406,22 +420,52 @@ class ChainVnf(object):
     def _get_vm_config(self, remote_mac_pair):
         config = self.manager.config
         devices = self.manager.generator_config.devices
+
+        if config.l3_router:
+            tg_gateway1_ip = self.routers[LEFT].ports[1]['fixed_ips'][0][
+                'ip_address']  # router edge ip left
+            tg_gateway2_ip = self.routers[RIGHT].ports[1]['fixed_ips'][0][
+                'ip_address']  # router edge ip right
+            tg_mac1 = self.routers[LEFT].ports[1]['mac_address']  # router edge mac left
+            tg_mac2 = self.routers[RIGHT].ports[1]['mac_address']  # router edge mac right
+            # edge cidr mask left
+            vnf_gateway1_cidr = \
+                self.ports[LEFT].get_ip() + self.manager.config.edge_networks.left.cidr[-3:]
+            # edge cidr mask right
+            vnf_gateway2_cidr = \
+                self.ports[RIGHT].get_ip() + self.manager.config.edge_networks.right.cidr[-3:]
+            if config.vm_forwarder != 'vpp':
+                raise ChainException(
+                    'L3 router mode imply to set VPP as VM forwarder.'
+                    'Please update your config file with: vm_forwarder: vpp')
+        else:
+            tg_gateway1_ip = devices[LEFT].tg_gateway_ip_addrs
+            tg_gateway2_ip = devices[RIGHT].tg_gateway_ip_addrs
+            tg_mac1 = remote_mac_pair[0]
+            tg_mac2 = remote_mac_pair[1]
+
+            g1cidr = devices[LEFT].get_gw_ip(
+                self.chain.chain_id) + self.manager.config.internal_networks.left.cidr[-3:]
+            g2cidr = devices[RIGHT].get_gw_ip(
+                self.chain.chain_id) + self.manager.config.internal_networks.right.cidr[-3:]
+
+            vnf_gateway1_cidr = g1cidr
+            vnf_gateway2_cidr = g2cidr
+
         with open(BOOT_SCRIPT_PATHNAME, 'r') as boot_script:
             content = boot_script.read()
-        g1cidr = devices[LEFT].get_gw_ip(self.chain.chain_id) + '/8'
-        g2cidr = devices[RIGHT].get_gw_ip(self.chain.chain_id) + '/8'
         vm_config = {
             'forwarder': config.vm_forwarder,
             'intf_mac1': self.ports[LEFT].get_mac(),
             'intf_mac2': self.ports[RIGHT].get_mac(),
-            'tg_gateway1_ip': devices[LEFT].tg_gateway_ip_addrs,
-            'tg_gateway2_ip': devices[RIGHT].tg_gateway_ip_addrs,
+            'tg_gateway1_ip': tg_gateway1_ip,
+            'tg_gateway2_ip': tg_gateway2_ip,
             'tg_net1': devices[LEFT].ip_addrs,
             'tg_net2': devices[RIGHT].ip_addrs,
-            'vnf_gateway1_cidr': g1cidr,
-            'vnf_gateway2_cidr': g2cidr,
-            'tg_mac1': remote_mac_pair[0],
-            'tg_mac2': remote_mac_pair[1],
+            'vnf_gateway1_cidr': vnf_gateway1_cidr,
+            'vnf_gateway2_cidr': vnf_gateway2_cidr,
+            'tg_mac1': tg_mac1,
+            'tg_mac2': tg_mac2,
             'vif_mq_size': config.vif_multiqueue_size
         }
         return content.format(**vm_config)
@@ -505,21 +549,27 @@ class ChainVnf(object):
         # Check if we can reuse an instance with same name
         for instance in self.manager.existing_instances:
             if instance.name == self.name:
+                instance_left = LEFT
+                instance_right = RIGHT
+                # In case of L3 traffic instance use edge networks
+                if self.manager.config.l3_router:
+                    instance_left = EDGE_LEFT
+                    instance_right = EDGE_RIGHT
                 # Verify that other instance characteristics match
                 if instance.flavor['id'] != flavor_id:
                     self._reuse_exception('Flavor mismatch')
                 if instance.status != "ACTIVE":
                     self._reuse_exception('Matching instance is not in ACTIVE state')
                 # The 2 networks for this instance must also be reused
-                if not networks[LEFT].reuse:
-                    self._reuse_exception('network %s is new' % networks[LEFT].name)
-                if not networks[RIGHT].reuse:
-                    self._reuse_exception('network %s is new' % networks[RIGHT].name)
+                if not networks[instance_left].reuse:
+                    self._reuse_exception('network %s is new' % networks[instance_left].name)
+                if not networks[instance_right].reuse:
+                    self._reuse_exception('network %s is new' % networks[instance_right].name)
                 # instance.networks have the network names as keys:
                 # {'nfvbench-rnet0': ['192.168.2.10'], 'nfvbench-lnet0': ['192.168.1.8']}
-                if networks[LEFT].name not in instance.networks:
+                if networks[instance_left].name not in instance.networks:
                     self._reuse_exception('Left network mismatch')
-                if networks[RIGHT].name not in instance.networks:
+                if networks[instance_right].name not in instance.networks:
                     self._reuse_exception('Right network mismatch')
 
                 self.reuse = True
@@ -527,16 +577,51 @@ class ChainVnf(object):
                 LOG.info('Reusing existing instance %s on %s',
                          self.name, self.get_hypervisor_name())
         # create or reuse/discover 2 ports per instance
-        self.ports = [ChainVnfPort(self.name + '-' + str(index),
-                                   self,
-                                   networks[index],
-                                   self._get_vnic_type(index)) for index in [0, 1]]
+        if self.manager.config.l3_router:
+            self.ports = [ChainVnfPort(self.name + '-' + str(index),
+                                       self,
+                                       networks[index + 2],
+                                       self._get_vnic_type(index)) for index in [0, 1]]
+        else:
+            self.ports = [ChainVnfPort(self.name + '-' + str(index),
+                                       self,
+                                       networks[index],
+                                       self._get_vnic_type(index)) for index in [0, 1]]
 
         # create idle networks and ports only if instance is not reused
         # if reused, we do not care about idle networks/ports
         if not self.reuse:
             self._get_idle_networks_ports()
 
+        # Create neutron routers for L3 traffic use case
+        if self.manager.config.l3_router and self.manager.openstack:
+            internal_nets = networks[:2]
+            if self.manager.config.service_chain == ChainType.PVP:
+                edge_nets = networks[2:]
+            else:
+                edge_nets = networks[3:]
+            subnets_left = [internal_nets[0], edge_nets[0]]
+            routes_left = [{'destination': self.manager.config.traffic_generator.ip_addrs[0],
+                            'nexthop': self.manager.config.traffic_generator.tg_gateway_ip_addrs[
+                                0]},
+                           {'destination': self.manager.config.traffic_generator.ip_addrs[1],
+                            'nexthop': self.ports[0].get_ip()}]
+            self.routers.append(
+                ChainRouter(self.manager, edge_nets[0].router_name, subnets_left, routes_left))
+            subnets_right = [internal_nets[1], edge_nets[1]]
+            routes_right = [{'destination': self.manager.config.traffic_generator.ip_addrs[0],
+                             'nexthop': self.ports[1].get_ip()},
+                            {'destination': self.manager.config.traffic_generator.ip_addrs[1],
+                             'nexthop': self.manager.config.traffic_generator.tg_gateway_ip_addrs[
+                                 1]}]
+            self.routers.append(
+                ChainRouter(self.manager, edge_nets[1].router_name, subnets_right, routes_right))
+            # Overload gateway_ips property with router ip address for ARP and traffic calls
+            self.manager.generator_config.devices[LEFT].set_gw_ip(
+                self.routers[LEFT].ports[0]['fixed_ips'][0]['ip_address'])  # router edge ip left)
+            self.manager.generator_config.devices[RIGHT].set_gw_ip(
+                self.routers[RIGHT].ports[0]['fixed_ips'][0]['ip_address'])  # router edge ip right)
+
         # if no reuse, actual vm creation is deferred after all ports in the chain are created
         # since we need to know the next mac in a multi-vnf chain
 
@@ -659,6 +744,7 @@ class ChainVnf(object):
             for network in self.idle_networks:
                 network.delete()
 
+
 class Chain(object):
     """A class to manage a single chain.
 
@@ -720,7 +806,8 @@ class Chain(object):
 
     def get_length(self):
         """Get the number of VNF in the chain."""
-        return len(self.networks) - 1
+        # Take into account 2 edge networks for routers
+        return len(self.networks) - 3 if self.manager.config.l3_router else len(self.networks) - 1
 
     def _get_remote_mac_pairs(self):
         """Get the list of remote mac pairs for every VNF in the chain.
@@ -1156,6 +1243,10 @@ class ChainManager(object):
                 net_cfg = [int_nets.left, int_nets.right]
             else:
                 net_cfg = [int_nets.left, int_nets.middle, int_nets.right]
+            if self.config.l3_router:
+                edge_nets = self.config.edge_networks
+                net_cfg.append(edge_nets.left)
+                net_cfg.append(edge_nets.right)
         networks = []
         try:
             for cfg in net_cfg:
diff --git a/nfvbench/cleanup.py b/nfvbench/cleanup.py
index 6b13f69..fc85b5d 100644
--- a/nfvbench/cleanup.py
+++ b/nfvbench/cleanup.py
@@ -25,6 +25,7 @@ from tabulate import tabulate
 import credentials as credentials
 from log import LOG
 
+
 class ComputeCleaner(object):
     """A cleaner for compute resources."""
 
@@ -45,30 +46,42 @@ class ComputeCleaner(object):
     def get_resource_list(self):
         return [["Instance", server.name, server.id] for server in self.servers]
 
-    def clean(self):
-        if self.servers:
-            for server in self.servers:
-                try:
-                    LOG.info('Deleting instance %s...', server.name)
-                    self.nova_client.servers.delete(server.id)
-                except Exception:
-                    LOG.exception("Instance %s deletion failed", server.name)
-            LOG.info('    Waiting for %d instances to be fully deleted...', len(self.servers))
-            retry_count = 15 + len(self.servers) * 5
-            while True:
-                retry_count -= 1
-                self.servers = [server for server in self.servers if self.instance_exists(server)]
-                if not self.servers:
-                    break
+    def get_cleaner_code(self):
+        return "instances"
 
-                if retry_count:
-                    LOG.info('    %d yet to be deleted by Nova, retries left=%d...',
-                             len(self.servers), retry_count)
-                    time.sleep(2)
-                else:
-                    LOG.warning('    instance deletion verification time-out: %d still not deleted',
-                                len(self.servers))
-                    break
+    def clean_needed(self, clean_options):
+        if clean_options is None:
+            return True
+        code = self.get_cleaner_code()
+        return code[0] in clean_options
+
+    def clean(self, clean_options):
+        if self.clean_needed(clean_options):
+            if self.servers:
+                for server in self.servers:
+                    try:
+                        LOG.info('Deleting instance %s...', server.name)
+                        self.nova_client.servers.delete(server.id)
+                    except Exception:
+                        LOG.exception("Instance %s deletion failed", server.name)
+                LOG.info('    Waiting for %d instances to be fully deleted...', len(self.servers))
+                retry_count = 15 + len(self.servers) * 5
+                while True:
+                    retry_count -= 1
+                    self.servers = [server for server in self.servers if
+                                    self.instance_exists(server)]
+                    if not self.servers:
+                        break
+
+                    if retry_count:
+                        LOG.info('    %d yet to be deleted by Nova, retries left=%d...',
+                                 len(self.servers), retry_count)
+                        time.sleep(2)
+                    else:
+                        LOG.warning(
+                            '    instance deletion verification time-out: %d still not deleted',
+                            len(self.servers))
+                        break
 
 
 class NetworkCleaner(object):
@@ -99,21 +112,103 @@ class NetworkCleaner(object):
         res_list.extend([["Port", port['name'], port['id']] for port in self.ports])
         return res_list
 
-    def clean(self):
-        for port in self.ports:
-            LOG.info("Deleting port %s...", port['id'])
-            try:
-                self.neutron_client.delete_port(port['id'])
-            except Exception:
-                LOG.exception("Port deletion failed")
-
-        # associated subnets are automatically deleted by neutron
-        for net in self.networks:
-            LOG.info("Deleting network %s...", net['name'])
-            try:
-                self.neutron_client.delete_network(net['id'])
-            except Exception:
-                LOG.exception("Network deletion failed")
+    def get_cleaner_code(self):
+        return "networks and ports"
+
+    def clean_needed(self, clean_options):
+        if clean_options is None:
+            return True
+        code = self.get_cleaner_code()
+        return code[0] in clean_options
+
+    def clean(self, clean_options):
+        if self.clean_needed(clean_options):
+            for port in self.ports:
+                LOG.info("Deleting port %s...", port['id'])
+                try:
+                    self.neutron_client.delete_port(port['id'])
+                except Exception:
+                    LOG.exception("Port deletion failed")
+
+            # associated subnets are automatically deleted by neutron
+            for net in self.networks:
+                LOG.info("Deleting network %s...", net['name'])
+                try:
+                    self.neutron_client.delete_network(net['id'])
+                except Exception:
+                    LOG.exception("Network deletion failed")
+
+
+class RouterCleaner(object):
+    """A cleaner for router resources."""
+
+    def __init__(self, neutron_client, router_names):
+        self.neutron_client = neutron_client
+        LOG.info('Discovering routers...')
+        all_routers = self.neutron_client.list_routers()['routers']
+        self.routers = []
+        self.ports = []
+        self.routes = []
+        rtr_ids = []
+        for rtr in all_routers:
+            rtrname = rtr['name']
+            for name in router_names:
+                if rtrname == name:
+                    self.routers.append(rtr)
+                    rtr_ids.append(rtr['id'])
+
+                    LOG.info('Discovering router routes for router %s...', rtr['name'])
+                    all_routes = rtr['routes']
+                    for route in all_routes:
+                        LOG.info("destination: %s, nexthop: %s", route['destination'],
+                                 route['nexthop'])
+
+                    LOG.info('Discovering router ports for router %s...', rtr['name'])
+                    self.ports.extend(self.neutron_client.list_ports(device_id=rtr['id'])['ports'])
+                    break
+
+    def get_resource_list(self):
+        res_list = [["Router", rtr['name'], rtr['id']] for rtr in self.routers]
+        return res_list
+
+    def get_cleaner_code(self):
+        return "router"
+
+    def clean_needed(self, clean_options):
+        if clean_options is None:
+            return True
+        code = self.get_cleaner_code()
+        return code[0] in clean_options
+
+    def clean(self, clean_options):
+        if self.clean_needed(clean_options):
+            # associated routes needs to be deleted before deleting routers
+            for rtr in self.routers:
+                LOG.info("Deleting routes for %s...", rtr['name'])
+                try:
+                    body = {
+                        'router': {
+                            'routes': []
+                        }
+                    }
+                    self.neutron_client.update_router(rtr['id'], body)
+                except Exception:
+                    LOG.exception("Router routes deletion failed")
+                LOG.info("Deleting ports for %s...", rtr['name'])
+                try:
+                    for port in self.ports:
+                        body = {
+                            'port_id': port['id']
+                        }
+                        self.neutron_client.remove_interface_router(rtr['id'], body)
+                except Exception:
+                    LOG.exception("Router ports deletion failed")
+                LOG.info("Deleting router %s...", rtr['name'])
+                try:
+                    self.neutron_client.delete_router(rtr['id'])
+                except Exception:
+                    LOG.exception("Router deletion failed")
+
 
 class FlavorCleaner(object):
     """Cleaner for NFVbench flavor."""
@@ -131,13 +226,24 @@ class FlavorCleaner(object):
             return [['Flavor', self.name, self.flavor.id]]
         return None
 
-    def clean(self):
-        if self.flavor:
-            LOG.info("Deleting flavor %s...", self.flavor.name)
-            try:
-                self.flavor.delete()
-            except Exception:
-                LOG.exception("Flavor deletion failed")
+    def get_cleaner_code(self):
+        return "flavor"
+
+    def clean_needed(self, clean_options):
+        if clean_options is None:
+            return True
+        code = self.get_cleaner_code()
+        return code[0] in clean_options
+
+    def clean(self, clean_options):
+        if self.clean_needed(clean_options):
+            if self.flavor:
+                LOG.info("Deleting flavor %s...", self.flavor.name)
+                try:
+                    self.flavor.delete()
+                except Exception:
+                    LOG.exception("Flavor deletion failed")
+
 
 class Cleaner(object):
     """Cleaner for all NFVbench resources."""
@@ -148,12 +254,15 @@ class Cleaner(object):
         self.neutron_client = nclient.Client('2.0', session=session)
         self.nova_client = Client(2, session=session)
         network_names = [inet['name'] for inet in config.internal_networks.values()]
+        network_names.extend([inet['name'] for inet in config.edge_networks.values()])
+        router_names = [rtr['router_name'] for rtr in config.edge_networks.values()]
         # add idle networks as well
         if config.idle_networks.name:
             network_names.append(config.idle_networks.name)
         self.cleaners = [ComputeCleaner(self.nova_client, config.loop_vm_name),
                          FlavorCleaner(self.nova_client, config.flavor_type),
-                         NetworkCleaner(self.neutron_client, network_names)]
+                         NetworkCleaner(self.neutron_client, network_names),
+                         RouterCleaner(self.neutron_client, router_names)]
 
     def show_resources(self):
         """Show all NFVbench resources."""
@@ -172,11 +281,37 @@ class Cleaner(object):
 
     def clean(self, prompt):
         """Clean all resources."""
-        LOG.info("NFVbench will delete all resources shown...")
+        LOG.info("NFVbench will delete resources shown...")
+        clean_options = None
         if prompt:
-            answer = raw_input("Are you sure? (y/n) ")
+            answer = raw_input("Do you want to delete all ressources? (y/n) ")
             if answer.lower() != 'y':
-                LOG.info("Exiting without deleting any resource")
-                sys.exit(0)
+                print "What kind of resources do you want to delete?"
+                all_option = ""
+                all_option_codes = []
+                for cleaner in self.cleaners:
+                    code = cleaner.get_cleaner_code()
+                    print "%s: %s" % (code[0], code)
+                    all_option += code[0]
+                    all_option_codes.append(code)
+                print "a: all resources - a shortcut for '%s'" % all_option
+                all_option_codes.append("all resources")
+                print "q: quit"
+                answer_res = raw_input(":").lower()
+                # Check only first character because answer_res can be "flavor" and it is != all
+                if answer_res[0] == "a":
+                    clean_options = all_option
+                elif answer_res[0] != 'q':
+                    # if user write complete code instead of shortcuts
+                    # Get only first character of clean code to avoid false clean request
+                    # i.e "networks and ports" and "router" have 1 letter in common and router clean
+                    # will be called even if user ask for networks and ports
+                    if answer_res in all_option_codes:
+                        clean_options = answer_res[0]
+                    else:
+                        clean_options = answer_res
+                else:
+                    LOG.info("Exiting without deleting any resource")
+                    sys.exit(0)
         for cleaner in self.cleaners:
-            cleaner.clean()
+            cleaner.clean(clean_options)
diff --git a/nfvbench/nfvbench.py b/nfvbench/nfvbench.py
index b2163ba..4a2a285 100644
--- a/nfvbench/nfvbench.py
+++ b/nfvbench/nfvbench.py
@@ -326,6 +326,11 @@ def _parse_opts_from_cli():
                         action='store',
                         help='Traffic generator profile to use')
 
+    parser.add_argument('-l3', '--l3-router', dest='l3_router',
+                        default=None,
+                        action='store_true',
+                        help='Use L3 neutron routers to handle traffic')
+
     parser.add_argument('-0', '--no-traffic', dest='no_traffic',
                         default=None,
                         action='store_true',
diff --git a/nfvbench/traffic_client.py b/nfvbench/traffic_client.py
index 75c40c1..d69da0e 100755
--- a/nfvbench/traffic_client.py
+++ b/nfvbench/traffic_client.py
@@ -23,7 +23,9 @@ from attrdict import AttrDict
 import bitmath
 from netaddr import IPNetwork
 # pylint: disable=import-error
+from trex.stl.api import Ether
 from trex.stl.api import STLError
+from trex.stl.api import UDP
 # pylint: enable=import-error
 
 from log import LOG
@@ -241,6 +243,11 @@ class Device(object):
         self.vnis = vnis
         LOG.info("Port %d: VNIs %s", self.port, self.vnis)
 
+    def set_gw_ip(self, gateway_ip):
+        self.gw_ip_block = IpBlock(gateway_ip,
+                                   self.generator_config.gateway_ip_addrs_step,
+                                   self.chain_count)
+
     def get_gw_ip(self, chain_index):
         """Retrieve the IP address assigned for the gateway of a given chain."""
         return self.gw_ip_block.get_ip(chain_index)
@@ -611,11 +618,10 @@ class TrafficClient(object):
             self.gen.stop_traffic()
             self.gen.fetch_capture_packets()
             self.gen.stop_capture()
-
             for packet in self.gen.packet_list:
                 mac_id = get_mac_id(packet)
                 src_mac = ':'.join(["%02x" % ord(x) for x in mac_id])
-                if src_mac in mac_map:
+                if src_mac in mac_map and self.is_udp(packet):
                     port, chain = mac_map[src_mac]
                     LOG.info('Received packet from mac: %s (chain=%d, port=%d)',
                              src_mac, chain, port)
@@ -624,9 +630,18 @@ class TrafficClient(object):
                 if not mac_map:
                     LOG.info('End-to-end connectivity established')
                     return
-
+            if self.config.l3_router and not self.config.no_arp:
+                # In case of L3 traffic mode, routers are not able to route traffic
+                # until VM interfaces are up and ARP requests are done
+                LOG.info('Waiting for loopback service completely started...')
+                LOG.info('Sending ARP request to assure end-to-end connectivity established')
+                self.ensure_arp_successful()
         raise TrafficClientException('End-to-end connectivity cannot be ensured')
 
+    def is_udp(self, packet):
+        pkt = Ether(packet['binary'])
+        return UDP in pkt
+
     def ensure_arp_successful(self):
         """Resolve all IP using ARP and throw an exception in case of failure."""
         dest_macs = self.gen.resolve_arp()
diff --git a/requirements.txt b/requirements.txt
index 490864c..9eb76c4 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -21,3 +21,4 @@ requests>=2.13.0
 tabulate>=0.7.5
 flask>=0.12
 fluent-logger>=0.5.3
+netaddr>=0.7.19
diff --git a/test/test_chains.py b/test/test_chains.py
index 5490dfc..5fd1ce6 100644
--- a/test/test_chains.py
+++ b/test/test_chains.py
@@ -271,6 +271,7 @@ def _mock_get_mac(dummy):
 @patch.object(Compute, 'find_image', _mock_find_image)
 @patch.object(TrafficClient, 'skip_sleep', lambda x: True)
 @patch.object(ChainVnfPort, 'get_mac', _mock_get_mac)
+@patch.object(TrafficClient, 'is_udp', lambda x, y: True)
 @patch('nfvbench.chaining.Client')
 @patch('nfvbench.chaining.neutronclient')
 @patch('nfvbench.chaining.glanceclient')
@@ -287,6 +288,7 @@ def test_nfvbench_run(mock_cred, mock_glance, mock_neutron, mock_client):
 
 @patch.object(Compute, 'find_image', _mock_find_image)
 @patch.object(TrafficClient, 'skip_sleep', lambda x: True)
+@patch.object(TrafficClient, 'is_udp', lambda x, y: True)
 @patch('nfvbench.chaining.Client')
 @patch('nfvbench.chaining.neutronclient')
 @patch('nfvbench.chaining.glanceclient')
@@ -302,6 +304,7 @@ def test_nfvbench_ext_arp(mock_cred, mock_glance, mock_neutron, mock_client):
 
 @patch.object(Compute, 'find_image', _mock_find_image)
 @patch.object(TrafficClient, 'skip_sleep', lambda x: True)
+@patch.object(TrafficClient, 'is_udp', lambda x, y: True)
 @patch('nfvbench.chaining.Client')
 @patch('nfvbench.chaining.neutronclient')
 @patch('nfvbench.chaining.glanceclient')
@@ -466,6 +469,7 @@ def test_summarizer():
         assert stats == exp_stats
 
 @patch.object(TrafficClient, 'skip_sleep', lambda x: True)
+@patch.object(TrafficClient, 'is_udp', lambda x, y: True)
 def test_fixed_rate_no_openstack():
     """Test FIxed Rate run - no openstack."""
     config = _get_chain_config(ChainType.EXT, 1, True, rate='100%')
diff --git a/test/test_nfvbench.py b/test/test_nfvbench.py
index 2a7ca77..7c5fb83 100644
--- a/test/test_nfvbench.py
+++ b/test/test_nfvbench.py
@@ -152,6 +152,19 @@ def test_ip_block():
     assert ipb.get_ip(255) == '10.0.0.255'
     with pytest.raises(IndexError):
         ipb.get_ip(256)
+    ipb = IpBlock('10.0.0.0', '0.0.0.1', 1)
+    assert ipb.get_ip() == '10.0.0.0'
+    with pytest.raises(IndexError):
+        ipb.get_ip(1)
+
+    ipb = IpBlock('10.0.0.0', '0.0.0.2', 256)
+    assert ipb.get_ip() == '10.0.0.0'
+    assert ipb.get_ip(1) == '10.0.0.2'
+    assert ipb.get_ip(127) == '10.0.0.254'
+    assert ipb.get_ip(128) == '10.0.1.0'
+    with pytest.raises(IndexError):
+        ipb.get_ip(256)
+
     # verify with step larger than 1
     ipb = IpBlock('10.0.0.0', '0.0.0.2', 256)
     assert ipb.get_ip() == '10.0.0.0'
@@ -341,7 +354,7 @@ def test_ndr_at_lr():
     # tx packets should be line rate for 64B and no drops...
     assert tg.get_tx_pps_dropped_pps(100) == (LR_64B_PPS, 0)
     # NDR and PDR should be at 100%
-    traffic_client.ensure_end_to_end()
+    # traffic_client.ensure_end_to_end()
     results = traffic_client.get_ndr_and_pdr()
     assert_ndr_pdr(results, 200.0, 0.0, 200.0, 0.0)