vFW: changes for gateway packet forwarding

JIRA: SAMPLEVNF-59

Changes to support gateway packet forwarding to vFW.

Change-Id: Ie9164b092f6864cfcdb02e7b325c4e8796a7e422
Signed-off-by: Anand B Jyoti <anand.b.jyoti@intel.com>

6 files changed, 296 insertions, 233 deletions

diff --git a/VNFs/vFW/Makefile b/VNFs/vFW/Makefile
index c96246b9..1e79646d 100644
--- a/VNFs/vFW/Makefile
+++ b/VNFs/vFW/Makefile
@@ -41,6 +41,7 @@ VPATH += $(SRCDIR)/pipeline
 VPATH += $(VNF_CORE)/common/VIL/pipeline_txrx
 VPATH += $(VNF_CORE)/common/VIL/acl
 VPATH += $(VNF_CORE)/common/VIL/l2l3_stack
+VPATH += $(VNF_CORE)/common/VIL/gateway
 
 INC += $(wildcard *.h)
 INC += $(wildcard pipeline/*.h)
@@ -54,6 +55,7 @@ INC += $(wildcard $(VNF_CORE)/common/VIL/pipeline_master/*.h)
 INC += $(wildcard $(VNF_CORE)/common/VIL/pipeline_passthrough/*.h)
 INC += $(wildcard $(VNF_CORE)/common/VIL/pipeline_txrx/*.h)
 INC += $(wildcard $(VNF_CORE)/common/VIL/acl/*.h)
+INC += $(wildcard $(VNF_CORE)/common/VIL/gateway/*.h)
 
 CFLAGS += -I$(SRCDIR) -mrtm -mhle -I$(SRCDIR)/pipeline -I$(VNF_CORE)/common/vnf_common
 CFLAGS += -I$(VNF_CORE)/common/VIL/conntrack -I$(VNF_CORE)/common/VIL/l2l3_stack
@@ -62,6 +64,7 @@ CFLAGS += -I$(VNF_CORE)/common/VIL/pipeline_master -I$(VNF_CORE)/common/VIL/pipe
 CFLAGS += -I$(VNF_CORE)/common/VIL/pipeline_txrx
 CFLAGS += -I$(VNF_CORE)/common/VIL/acl
 CFLAGS += -I$(VNF_CORE)/common/VIL/pipeline_arpicmp
+CFLAGS += -I$(VNF_CORE)/common/VIL/gateway
 
 # all source are stored in SRCS-y
 SRCS-$(CONFIG_RTE_LIBRTE_PIPELINE) := main.c
@@ -104,6 +107,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_PIPELINE) += pipeline_arpicmp_be.c
 SRCS-$(CONFIG_RTE_LIBRTE_PIPELINE) += pipeline_txrx.c
 SRCS-$(CONFIG_RTE_LIBRTE_PIPELINE) += pipeline_txrx_be.c
 SRCS-$(CONFIG_RTE_LIBRTE_PIPELINE) += lib_acl.c
+SRCS-$(CONFIG_RTE_LIBRTE_PIPELINE) += gateway.c
 
 
 CFLAGS += -O3
diff --git a/VNFs/vFW/config/VFW_HWLB_MultiPortPair_script.tc b/VNFs/vFW/config/VFW_HWLB_MultiPortPair_script.tc
index 46355bee..b27c66fa 100644
--- a/VNFs/vFW/config/VFW_HWLB_MultiPortPair_script.tc
+++ b/VNFs/vFW/config/VFW_HWLB_MultiPortPair_script.tc
@@ -32,18 +32,17 @@ link 3 config 172.16.40.10 8
 ;link 3 config 2016:0000:0000:0000:6a05:caff:fe30:2071 64
 link 3 up
 
-; routeadd <port #> <ipv4 nhip address in decimal> <Mask>
-routeadd 0 203.16.100.20 0xff000000
-routeadd 1 202.16.100.20 0xff000000
-routeadd 2 173.16.40.20  0xff000000
-routeadd 3 172.16.40.20  0xff000000
-
-;routeadd <port #> <ipv6 nhip address in hex> <Depth>
-;routeadd 0 fec0:0000:0000:0000:6a05:caff:fe30:21a0 64
-;routeadd 1 fec1:0000:0000:0000:6a05:caff:fe30:21a0 64
-;routeadd 2 2012:0000:0000:0000:6a05:caff:fe30:2071 64
-;routeadd 3 2016:0000:0000:0000:6a05:caff:fe30:2071 64
-
+; routeadd <net/host> <port #> <ipv4 nhip address in decimal> <Mask/NotApplicable>
+routeadd net 0 203.16.100.20 0xff000000
+routeadd net 1 202.16.100.20 0xff000000
+routeadd net 2 173.16.40.20  0xff000000
+routeadd net 3 172.16.40.20  0xff000000
+
+;routeadd <net/host> <port #> <ipv6 nhip address in hex> <Depth/NotApplicable>
+;routeadd net 0 fec0:0000:0000:0000:6a05:caff:fe30:21a0 64
+;routeadd net 1 fec1:0000:0000:0000:6a05:caff:fe30:21a0 64
+;routeadd net 2 2012:0000:0000:0000:6a05:caff:fe30:2071 64
+;routeadd net 3 2016:0000:0000:0000:6a05:caff:fe30:2071 64
 
 ; IPv4 Static ARP
 ;p 1 arpadd 0 203.16.100.20 00:00:00:00:00:01
diff --git a/VNFs/vFW/config/VFW_HWLB_SinglePortPair_script.tc b/VNFs/vFW/config/VFW_HWLB_SinglePortPair_script.tc
index f20796ac..35bb519d 100644
--- a/VNFs/vFW/config/VFW_HWLB_SinglePortPair_script.tc
+++ b/VNFs/vFW/config/VFW_HWLB_SinglePortPair_script.tc
@@ -22,13 +22,13 @@ link 1 config 172.16.40.10 8
 ;link 1 config 2012:0000:0000:0000:6a05:caff:fe30:2071 64
 link 1 up
 
-; routeadd <port #> <ipv4 nhip address in decimal> <Mask>
-routeadd 0 202.16.100.20 0xff000000
-routeadd 1 172.16.40.20  0xff000000
+; routeadd <net/host> <port #> <ipv4 nhip address in decimal> <Mask/NotApplicable>
+routeadd net 0 202.16.100.20 0xff000000
+routeadd net 1 172.16.40.20  0xff000000
 
-;routeadd <port #> <ipv6 nhip address in hex> <Depth>
-;routeadd 0 fec0::6a05:caff:fe30:21b0  64
-;routeadd 1 2012::6a05:caff:fe30:2081 64
+;routeadd <net/host> <port #> <ipv6 nhip address in hex> <Depth/NotApplicable>
+;routeadd net 0 fec0::6a05:caff:fe30:21b0 64
+;routeadd net 1 2012::6a05:caff:fe30:2081 64
 
 ; IPv4 static ARP
 ;p 1 arpadd 1 172.16.40.20 00:00:00:00:00:04
diff --git a/VNFs/vFW/config/VFW_SWLB_MultiPortPair_script.tc b/VNFs/vFW/config/VFW_SWLB_MultiPortPair_script.tc
index 392320e3..ff502e55 100644
--- a/VNFs/vFW/config/VFW_SWLB_MultiPortPair_script.tc
+++ b/VNFs/vFW/config/VFW_SWLB_MultiPortPair_script.tc
@@ -32,17 +32,17 @@ link 3 config 172.16.40.10 8
 ;link 3 config 2016:0000:0000:0000:6a05:caff:fe30:2071 64
 link 3 up
 
-; routeadd <port #> <ipv4 nhip address in decimal> <Mask>
-routeadd 0 203.16.100.20 0xff000000
-routeadd 1 202.16.100.20 0xff000000
-routeadd 2 173.16.40.20  0xff000000
-routeadd 3 172.16.40.20  0xff000000
-
-;routeadd <port #> <ipv6 nhip address in hex> <Depth>
-;routeadd 0 fec0:0000:0000:0000:6a05:caff:fe30:21a0 64
-;routeadd 1 fec1:0000:0000:0000:6a05:caff:fe30:21a0 64
-;routeadd 2 2012:0000:0000:0000:6a05:caff:fe30:2071 64
-;routeadd 3 2016:0000:0000:0000:6a05:caff:fe30:2071 64
+; routeadd <net/host> <port #> <ipv4 nhip address in decimal> <Mask/NotApplicable>
+routeadd net 0 203.16.100.20 0xff000000
+routeadd net 1 202.16.100.20 0xff000000
+routeadd net 2 173.16.40.20  0xff000000
+routeadd net 3 172.16.40.20  0xff000000
+
+;routeadd <net/host> <port #> <ipv6 nhip address in hex> <Depth/NotApplicable>
+;routeadd net 0 fec0:0000:0000:0000:6a05:caff:fe30:21a0 64
+;routeadd net 1 fec1:0000:0000:0000:6a05:caff:fe30:21a0 64
+;routeadd net 2 2012:0000:0000:0000:6a05:caff:fe30:2071 64
+;routeadd net 3 2016:0000:0000:0000:6a05:caff:fe30:2071 64
 
 ; IPv4 Static ARP
 ;p 1 arpadd 0 203.16.100.20 00:00:00:00:00:01
diff --git a/VNFs/vFW/config/VFW_SWLB_SinglePortPair_script.tc b/VNFs/vFW/config/VFW_SWLB_SinglePortPair_script.tc
index d85e17b9..b9dd226b 100644
--- a/VNFs/vFW/config/VFW_SWLB_SinglePortPair_script.tc
+++ b/VNFs/vFW/config/VFW_SWLB_SinglePortPair_script.tc
@@ -22,17 +22,21 @@ link 1 config 172.16.40.10 8
 ;link 1 config 2012:0000:0000:0000:6a05:caff:fe30:2071 64
 link 1 up
 
-; routeadd <port #> <ipv4 nhip address in decimal> <Mask>
-routeadd 0 202.16.100.20 0xff000000
-routeadd 1 172.16.40.20 0xff000000
+; routeadd <net/host> <port #> <ipv4 nhip address in decimal> <Mask/NotApplicable>
+;routeadd host 0 202.16.100.20
+;routeadd host 1 172.16.40.20
+routeadd net 0 202.16.100.20 0xff000000
+routeadd net 1 172.16.40.20 0xff000000
 
-;routeadd <port #> <ipv6 nhip address in hex> <Depth>
-;routeadd 0 fec0::6a05:caff:fe30:21b0 64
-;routeadd 1 2012::6a05:caff:fe30:2081 64
+;routeadd <net/host> <port #> <ipv6 nhip address in hex> <Depth/NotApplicable>
+;routeadd host 0 fec0::6a05:caff:fe30:21b0
+;routeadd host 1 2012::6a05:caff:fe30:2081
+routeadd net 0 fec0::6a05:caff:fe30:21b0 64
+routeadd net 1 2012::6a05:caff:fe30:2081 64
 
 ; IPv4 static ARP
-;p 1 arpadd 1 172.16.40.20 00:00:00:00:00:04
-;p 1 arpadd 0 202.16.100.20 00:00:00:00:00:01
+p 1 arpadd 1 172.16.40.20 00:00:00:00:00:02
+p 1 arpadd 0 202.16.100.20 00:00:00:00:00:01
 
 ; IPv6 static ARP
 ;p 1 arpadd 0 fec0::6a05:caff:fe30:21b0 00:00:00:00:00:01
diff --git a/VNFs/vFW/pipeline/pipeline_vfw_be.c b/VNFs/vFW/pipeline/pipeline_vfw_be.c
index fed424e5..f0eab34b 100644
--- a/VNFs/vFW/pipeline/pipeline_vfw_be.c
+++ b/VNFs/vFW/pipeline/pipeline_vfw_be.c
@@ -24,7 +24,7 @@
  */
 
 #define EN_SWP_ACL 1
-#define EN_SWP_ARP 1
+//#define EN_SWP_ARP 1
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -64,6 +64,7 @@
 #include "lib_arp.h"
 #include "lib_icmpv6.h"
 #include "pipeline_common_fe.h"
+#include "gateway.h"
 
 uint32_t timer_lcore;
 
@@ -94,13 +95,6 @@ struct pipeline_vfw {
        uint8_t traffic_type;
        uint8_t links_map[PIPELINE_MAX_PORT_IN];
        uint8_t outport_id[PIPELINE_MAX_PORT_IN];
-       /* Local ARP & ND Tables */
-       struct lib_arp_route_table_entry
-              local_lib_arp_route_table[MAX_ARP_RT_ENTRY];
-       uint8_t local_lib_arp_route_ent_cnt;
-       struct lib_nd_route_table_entry
-              local_lib_nd_route_table[MAX_ND_RT_ENTRY];
-       uint8_t local_lib_nd_route_ent_cnt;
 
 } __rte_cache_aligned;
 /**
@@ -799,10 +793,12 @@ pkt4_work_vfw_arp_ipv4_packets(struct rte_mbuf **pkts,
                                 if (ret_arp_data->num_pkts >= NUM_DESC) {
 					/* ICMP req sent, drop packet by
 						* changing the mask */
-					vfw_pipe->counters->pkts_drop_without_arp_entry++;
+					vfw_pipe->counters->
+						pkts_drop_without_arp_entry++;
                                         continue;
                                 } else {
-                                        arp_pkts_mask |= pkt_mask;
+                                        //arp_pkts_mask |= pkt_mask;
+					*arp_hijack_mask |= pkt_mask;
                                         arp_queue_unresolved_packet(ret_arp_data, pkt);
                                         continue;
                      }
@@ -895,7 +891,8 @@ pkt_work_vfw_arp_ipv4_packets(struct rte_mbuf *pkts,
                                 if (ret_arp_data->num_pkts >= NUM_DESC) {
 					/* ICMP req sent, drop packet by
 						* changing the mask */
-					vfw_pipe->counters->pkts_drop_without_arp_entry++;
+					vfw_pipe->counters->
+						pkts_drop_without_arp_entry++;
                                         return;
                                 } else {
                                         arp_pkts_mask |= pkt_mask;
@@ -999,7 +996,8 @@ pkt4_work_vfw_arp_ipv6_packets(struct rte_mbuf **pkts,
 			  if (ret_nd_data->num_pkts >= NUM_DESC) {
                                 /* Drop the pkt */
                                 *pkts_mask &= ~pkt_mask;
-				vfw_pipe->counters->pkts_drop_without_arp_entry++;
+                                vfw_pipe->counters->
+					pkts_drop_without_arp_entry++;
 				continue;
                           } else {
                                 arp_pkts_mask |= pkt_mask;
@@ -1099,7 +1097,8 @@ pkt_work_vfw_arp_ipv6_packets(struct rte_mbuf *pkts,
                           if (ret_nd_data->num_pkts >= NUM_DESC) {
                                 /* Drop the pkt */
                                 *pkts_mask &= ~pkt_mask;
-				vfw_pipe->counters->pkts_drop_without_arp_entry++;
+                                vfw_pipe->counters->
+                                    pkts_drop_without_arp_entry++;
                                 return;
                           } else {
                                 arp_pkts_mask |= pkt_mask;
@@ -1116,211 +1115,257 @@ pkt_work_vfw_arp_ipv6_packets(struct rte_mbuf *pkts,
 #else
 
 /**
- * walk every valid mbuf (denoted by pkts_mask) and apply arp to the packet.
+ * walk every valid mbuf (denoted by pkts_mask) and forward the packet.
  * To support synproxy, some (altered) packets may need to be sent back where
  * they came from. The ip header has already been adjusted, but the ethernet
  * header has not, so this must be performed here.
- * Return an updated pkts_mask, since arp may drop some packets
+ * Return an updated pkts_mask and arp_hijack_mask since arp may drop some packets
  *
  * @param pkts
- *  A pointer to the packet.
+ *  A pointer to the packet array.
  * @param pkts_mask
- *  Packet mask
- * @param synproxy_reply_mask
- *  Reply Packet mask for Synproxy
+ *  Packets mask to be processed
+ * @param arp_hijack_mask
+ *  Packets to be hijacked for arp buffering
  * @param vfw_pipe
  *  A pointer to VFW pipeline.
  */
-static uint64_t
-rte_vfw_arp_ipv4_packets(struct rte_mbuf **pkts,
-              uint64_t pkts_mask,
-              uint64_t synproxy_reply_mask,
-              struct pipeline_vfw *vfw_pipe)
+static void vfw_fwd_pkts_ipv4(struct rte_mbuf **pkts, uint64_t *pkts_mask,
+		uint64_t *arp_hijack_mask, struct pipeline_vfw *vfw_pipe)
 {
-       uint64_t pkts_to_arp = pkts_mask;
+	uint64_t pkts_to_arp = *pkts_mask;
 
-       uint32_t ret;
-       uint32_t dest_if = INVALID_DESTIF;
-       for (; pkts_to_arp;) {
-              struct ether_addr hw_addr;
-              struct mbuf_tcp_meta_data *meta_data_addr;
-              struct ether_hdr *ehdr;
-              struct rte_mbuf *pkt;
-              uint16_t phy_port;
-
-              uint8_t pos = (uint8_t) __builtin_ctzll(pkts_to_arp);
-              /* bitmask representing only this packet */
-              uint64_t pkt_mask = 1LLU << pos;
-              /* remove this packet from remaining list */
-              pkts_to_arp &= ~pkt_mask;
-              pkt = pkts[pos];
-              int must_reverse = ((synproxy_reply_mask & pkt_mask) != 0);
+	for (; pkts_to_arp;) {
 
-              phy_port = pkt->port;
-              meta_data_addr = (struct mbuf_tcp_meta_data *)
-                     RTE_MBUF_METADATA_UINT32_PTR(pkt, META_DATA_OFFSET);
-              ehdr = rte_vfw_get_ether_addr(pkt);
+		struct mbuf_tcp_meta_data *meta_data_addr;
+		struct ether_hdr *ehdr;
+		struct rte_mbuf *pkt;
+		uint32_t src_phy_port;
 
+		uint8_t pos = (uint8_t) __builtin_ctzll(pkts_to_arp);
+		/* bitmask representing only this packet */
+		uint64_t pkt_mask = 1LLU << pos;
+		/* remove this packet from remaining list */
+		pkts_to_arp &= ~pkt_mask;
+		pkt = pkts[pos];
 
-              struct ipv4_hdr *ihdr = (struct ipv4_hdr *)
-                     RTE_MBUF_METADATA_UINT32_PTR(pkt, IP_START);
-              uint32_t nhip = 0;
+		if(VFW_DEBUG) {
+			printf("----------------\n");
+			print_pkt(pkt);
+		}
 
-              uint32_t dest_address = rte_bswap32(ihdr->dst_addr);
-              if (must_reverse)
-                     rte_sp_exchange_mac_addresses(ehdr);
-		struct arp_entry_data *ret_arp_data = NULL;
-                     ret_arp_data = get_dest_mac_addr_port(dest_address,
-                                   &dest_if, &ehdr->d_addr);
-		meta_data_addr->output_port =  vfw_pipe->outport_id[dest_if];
-        if (arp_cache_dest_mac_present(dest_if)) {
+		meta_data_addr = (struct mbuf_tcp_meta_data *)
+			RTE_MBUF_METADATA_UINT32_PTR(pkt, META_DATA_OFFSET);
 
-                ether_addr_copy(get_link_hw_addr(dest_if), &ehdr->s_addr);
-		update_nhip_access(dest_if);
-                if (unlikely(ret_arp_data && ret_arp_data->num_pkts)) {
+		ehdr = (struct ether_hdr *)
+			RTE_MBUF_METADATA_UINT32_PTR(pkt, ETHERNET_START);
 
-                        arp_send_buffered_pkts(ret_arp_data,
-                                 &ehdr->d_addr, vfw_pipe->outport_id[dest_if]);
+		src_phy_port = pkt->port;
+		uint32_t dst_phy_port = INVALID_DESTIF;
 
-                            }
+		if(is_gateway()){
+			struct ipv4_hdr *ipv4hdr = (struct ipv4_hdr *)
+				RTE_MBUF_METADATA_UINT32_PTR(pkt, IP_START);
 
-                     } else {
-                if (unlikely(ret_arp_data == NULL)) {
+			/* Gateway Proc Starts */
 
-			if (VFW_DEBUG)
-                        printf("%s: NHIP Not Found, nhip:%x , "
-                        "outport_id: %d\n", __func__, nhip,
-                        vfw_pipe->outport_id[dest_if]);
+			struct arp_entry_data *ret_arp_data = NULL;
+			struct ether_addr dst_mac;
+			uint32_t nhip = 0;
+			uint32_t dst_ip_addr = rte_bswap32(ipv4hdr->dst_addr);
 
-                        /* Drop the pkt */
-                                   vfw_pipe->counters->
-                                          pkts_drop_without_arp_entry++;
-                        continue;
-                     }
-		if (ret_arp_data->status == INCOMPLETE ||
-                           ret_arp_data->status == PROBE) {
-                                if (ret_arp_data->num_pkts >= NUM_DESC) {
-					/* ICMP req sent, drop packet by
-						* changing the mask */
-					vfw_pipe->counters->pkts_drop_without_arp_entry++;
-                                        continue;
-                                } else {
-                                        arp_pkts_mask |= pkt_mask;
-                                        arp_queue_unresolved_packet(ret_arp_data, pkt);
-                                        continue;
-              }
-}
-	}
+			gw_get_nh_port_ipv4(dst_ip_addr, &dst_phy_port, &nhip);
 
-       }
+			ret_arp_data = get_dest_mac_addr_ipv4(nhip, dst_phy_port, &dst_mac);
+
+			/* Gateway Proc Ends */
+
+			if (arp_cache_dest_mac_present(dst_phy_port)) {
+
+				ether_addr_copy(&dst_mac, &ehdr->d_addr);
+				ether_addr_copy(get_link_hw_addr(dst_phy_port), &ehdr->s_addr);
+
+				meta_data_addr->output_port = vfw_pipe->outport_id[dst_phy_port];
+
+				update_nhip_access(dst_phy_port);
+
+				if (unlikely(ret_arp_data && ret_arp_data->num_pkts)) {
+
+					arp_send_buffered_pkts(ret_arp_data, &ehdr->d_addr,
+							vfw_pipe->outport_id[dst_phy_port]);
+				}
+
+			} else {
+				if (unlikely(ret_arp_data == NULL)) {
+
+					printf("NHIP Not Found\n");
+
+					/* Drop the pkt */
+					vfw_pipe->counters->
+						pkts_drop_without_arp_entry++;
+					continue;
+				}
+				if (ret_arp_data->status == INCOMPLETE ||
+						ret_arp_data->status == PROBE) {
+					if (ret_arp_data->num_pkts >= NUM_DESC) {
+						/* ICMP req sent, drop packet by
+						 * changing the mask */
+						vfw_pipe->counters->pkts_drop_without_arp_entry++;
+						continue;
+					} else {
+						*arp_hijack_mask |= pkt_mask;
+						arp_queue_unresolved_packet(ret_arp_data, pkt);
+						continue;
+					}
+				}
+			}
+		} else {
+			/* IP Pkt forwarding based on  pub/prv mapping */
+			if(is_phy_port_privte(src_phy_port))
+				dst_phy_port = prv_to_pub_map[src_phy_port];
+			else
+				dst_phy_port = pub_to_prv_map[src_phy_port];
+
+			meta_data_addr->output_port = vfw_pipe->outport_id[dst_phy_port];
+
+			if(VFW_DEBUG) {
+				printf("IP_PKT_FWD: src_phy_port=%d, dst_phy_port=%d\n",
+						src_phy_port, dst_phy_port);
+			}
+		}
+
+		if(VFW_DEBUG)
+			print_pkt(pkt);
+	}
 
-       return pkts_mask;
 }
+
 /**
- * walk every valid mbuf (denoted by pkts_mask) and apply arp to the packet.
+ * walk every valid mbuf (denoted by pkts_mask) and forward the packet.
  * To support synproxy, some (altered) packets may need to be sent back where
  * they came from. The ip header has already been adjusted, but the ethernet
  * header has not, so this must be performed here.
- * Return an updated pkts_mask, since arp may drop some packets
+ * Return an updated pkts_mask and arp_hijack_mask since arp may drop some packets
  *
  * @param pkts
- *  A pointer to the packet.
+ *  A pointer to the packet array.
  * @param pkts_mask
- *  Packet mask
- * @param synproxy_reply_mask
- *  Reply Packet mask for Synproxy
+ *  Packets mask to be processed
+ * @param arp_hijack_mask
+ *  Packets to be hijacked for arp buffering
  * @param vfw_pipe
  *  A pointer to VFW pipeline.
  */
-
-       static uint64_t
-rte_vfw_arp_ipv6_packets(struct rte_mbuf **pkts,
-              uint64_t pkts_mask,
-              uint64_t synproxy_reply_mask,
-              struct pipeline_vfw *vfw_pipe)
+static void vfw_fwd_pkts_ipv6(struct rte_mbuf **pkts, uint64_t *pkts_mask,
+			uint64_t *arp_hijack_mask, struct pipeline_vfw *vfw_pipe)
 {
-       uint64_t pkts_to_arp = pkts_mask;
-       uint8_t nh_ipv6[IPV6_ADD_SIZE];
-       uint32_t ret;
-       uint32_t dest_if = INVALID_DESTIF;
+	uint64_t pkts_to_arp = *pkts_mask;
 
-       for (; pkts_to_arp;) {
-              struct ether_addr hw_addr;
-              struct mbuf_tcp_meta_data *meta_data_addr;
-              struct ether_hdr *ehdr;
-              struct rte_mbuf *pkt;
-              uint16_t phy_port;
+	for (; pkts_to_arp;) {
 
-              uint8_t pos = (uint8_t) __builtin_ctzll(pkts_to_arp);
-              /* bitmask representing only this packet */
-              uint64_t pkt_mask = 1LLU << pos;
-              /* remove this packet from remaining list */
-              pkts_to_arp &= ~pkt_mask;
-              pkt = pkts[pos];
-              int must_reverse = ((synproxy_reply_mask & pkt_mask) != 0);
+		struct mbuf_tcp_meta_data *meta_data_addr;
+		struct ether_hdr *ehdr;
+		struct rte_mbuf *pkt;
+		uint32_t src_phy_port;
 
-              phy_port = pkt->port;
-              meta_data_addr = (struct mbuf_tcp_meta_data *)
-                     RTE_MBUF_METADATA_UINT32_PTR(pkt, META_DATA_OFFSET);
-              ehdr = rte_vfw_get_ether_addr(pkt);
+		struct nd_entry_data *ret_nd_data = NULL;
 
-              struct ipv6_hdr *ihdr = (struct ipv6_hdr *)
-                     RTE_MBUF_METADATA_UINT32_PTR(pkt, IP_START);
+		uint8_t pos = (uint8_t) __builtin_ctzll(pkts_to_arp);
+		/* bitmask representing only this packet */
+		uint64_t pkt_mask = 1LLU << pos;
+		/* remove this packet from remaining list */
+		pkts_to_arp &= ~pkt_mask;
+		pkt = pkts[pos];
 
-              uint8_t nhip[IPV6_ADD_SIZE];
-              uint8_t dest_address[IPV6_ADD_SIZE];
+		if(VFW_DEBUG) {
+			printf("----------------\n");
+			print_pkt(pkt);
+		}
 
-              memset(nhip, 0, IPV6_ADD_SIZE);
-              if (must_reverse)
-                     rte_sp_exchange_mac_addresses(ehdr);
+		meta_data_addr = (struct mbuf_tcp_meta_data *)
+			RTE_MBUF_METADATA_UINT32_PTR(pkt, META_DATA_OFFSET);
 
-              rte_mov16(dest_address, ihdr->dst_addr);
-              memset(nh_ipv6, 0, IPV6_ADD_SIZE);
-              struct nd_entry_data *ret_nd_data = NULL;
-              ret_nd_data = get_dest_mac_address_ipv6_port(
-                                   &dest_address[0],
-                                   &dest_if,
-                                   &hw_addr,
-                                   &nh_ipv6[0]);
+		ehdr = (struct ether_hdr *)
+			RTE_MBUF_METADATA_UINT32_PTR(pkt, ETHERNET_START);
 
-	      meta_data_addr->output_port = vfw_pipe->
-                                    outport_id[dest_if];
-              if (nd_cache_dest_mac_present(dest_if)) {
-                     ether_addr_copy(get_link_hw_addr(dest_if),
-                                   &ehdr->s_addr);
-		    update_nhip_access(dest_if);
+		src_phy_port = pkt->port;
+		uint32_t dst_phy_port = INVALID_DESTIF;
 
-                    if (unlikely(ret_nd_data && ret_nd_data->num_pkts)) {
-                        nd_send_buffered_pkts(ret_nd_data,
-                               &ehdr->d_addr, meta_data_addr->output_port);
-                     }
+		if(is_gateway()){
+			struct ipv6_hdr *ipv6hdr = (struct ipv6_hdr *)
+				RTE_MBUF_METADATA_UINT32_PTR(pkt, IP_START);
 
-              } else {
-                    if (unlikely(ret_nd_data == NULL)) {
-                     pkts_mask &= ~pkt_mask;
-			  vfw_pipe->counters->
-				pkts_drop_without_arp_entry++;
-                          continue;
-                    }
-		    if (ret_nd_data->status == INCOMPLETE ||
-                          ret_nd_data->status == PROBE) {
-                          if (ret_nd_data->num_pkts >= NUM_DESC) {
-                                /* Drop the pkt */
-				pkts_mask &= ~pkt_mask;
-                                vfw_pipe->counters->
-                                    pkts_drop_without_arp_entry++;
-                                continue;
-                          } else {
-                                arp_pkts_mask |= pkt_mask;
-                                nd_queue_unresolved_packet(ret_nd_data, pkt);
-                                continue;
-                          }
-                    }
-	      }
+			/* Gateway Proc Starts */
 
-       }
+			struct ether_addr dst_mac;
+			uint32_t dst_phy_port = INVALID_DESTIF;
+			uint8_t nhipv6[IPV6_ADD_SIZE];
+			uint8_t dest_ipv6_address[IPV6_ADD_SIZE];
+			memset(nhipv6, 0, IPV6_ADD_SIZE);
+			src_phy_port = pkt->port;
+			rte_mov16(dest_ipv6_address, (uint8_t *)ipv6hdr->dst_addr);
+
+			gw_get_nh_port_ipv6(dest_ipv6_address, &dst_phy_port, nhipv6);
+
+			ret_nd_data = get_dest_mac_addr_ipv6(nhipv6, dst_phy_port, &dst_mac);
+
+			/* Gateway Proc Ends */
+
+			if (nd_cache_dest_mac_present(dst_phy_port)) {
+
+				ether_addr_copy(&dst_mac, &ehdr->d_addr);
+				ether_addr_copy(get_link_hw_addr(dst_phy_port), &ehdr->s_addr);
+
+				meta_data_addr->output_port = vfw_pipe->outport_id[dst_phy_port];
 
-       return pkts_mask;
+				update_nhip_access(dst_phy_port);
+
+				if (unlikely(ret_nd_data && ret_nd_data->num_pkts)) {
+					nd_send_buffered_pkts(ret_nd_data, &ehdr->d_addr,
+							vfw_pipe->outport_id[dst_phy_port]);
+				}
+
+			} else {
+				if (unlikely(ret_nd_data == NULL)) {
+
+					printf("NHIP Not Found\n");
+
+					/* Drop the pkt */
+					vfw_pipe->counters->pkts_drop_without_arp_entry++;
+					continue;
+				}
+				if (ret_nd_data->status == INCOMPLETE ||
+						ret_nd_data->status == PROBE) {
+					if (ret_nd_data->num_pkts >= NUM_DESC) {
+						/* ICMP req sent, drop packet by
+						 * changing the mask */
+						vfw_pipe->counters->pkts_drop_without_arp_entry++;
+						continue;
+					} else {
+						*arp_hijack_mask |= pkt_mask;
+						nd_queue_unresolved_packet(ret_nd_data, pkt);
+						continue;
+					}
+				}
+			}
+
+		} else {
+			/* IP Pkt forwarding based on  pub/prv mapping */
+			if(is_phy_port_privte(src_phy_port))
+				dst_phy_port = prv_to_pub_map[src_phy_port];
+			else
+				dst_phy_port = pub_to_prv_map[src_phy_port];
+
+			meta_data_addr->output_port = vfw_pipe->outport_id[dst_phy_port];
+
+			if(VFW_DEBUG) {
+				printf("IP_PKT_FWD: src_phy_port=%d, dst_phy_port=%d\n",
+						src_phy_port, dst_phy_port);
+			}
+		}
+		if(VFW_DEBUG)
+			print_pkt(pkt);
+	}
 }
 
 #endif
@@ -1516,9 +1561,9 @@ vfw_port_in_action_ipv4(struct rte_pipeline *p,
 
        uint64_t packet_mask_in = RTE_LEN2MASK(n_pkts, uint64_t);
        uint64_t pkts_drop_mask;
-       uint64_t hijack_mask = 0;
-       arp_pkts_mask = 0;
-       uint64_t synproxy_reply_mask = 0;       /* for synproxy */
+       uint64_t synp_hijack_mask = 0;
+       uint64_t arp_hijack_mask = 0;
+//       uint64_t synproxy_reply_mask;       /* for synproxy */
        uint64_t keep_mask = packet_mask_in;
 
        uint64_t conntrack_mask = 0, connexist_mask = 0;
@@ -1598,8 +1643,8 @@ vfw_port_in_action_ipv4(struct rte_pipeline *p,
        if (likely(cnxn_tracking_is_active)) {
               rte_ct_cnxn_tracker_batch_lookup_type(ct, pkts,
                             &keep_mask, &ct_helper, IPv4_HEADER_SIZE);
-              synproxy_reply_mask = ct_helper.reply_pkt_mask;
-              hijack_mask = ct_helper.hijack_mask;
+//              synproxy_reply_mask = ct_helper.reply_pkt_mask;
+              synp_hijack_mask = ct_helper.hijack_mask;
 
        }
 
@@ -1637,9 +1682,9 @@ vfw_port_in_action_ipv4(struct rte_pipeline *p,
 #else
        rte_prefetch0((void*)in_port_dir_a);
        rte_prefetch0((void*)prv_to_pub_map);
-       rte_prefetch0((void*) & vfw_pipe->local_lib_arp_route_table);
-       keep_mask = rte_vfw_arp_ipv4_packets(pkts, keep_mask,
-                     synproxy_reply_mask, vfw_pipe);
+
+	vfw_fwd_pkts_ipv4(pkts, &keep_mask, &arp_hijack_mask, vfw_pipe);
+
 #endif
 
        if (vfw_debug > 1) {
@@ -1651,9 +1696,14 @@ vfw_port_in_action_ipv4(struct rte_pipeline *p,
                                    (void *)keep_mask);
        }
 
-       /* Update mask before returning, so that bad packets are dropped */
-        if (arp_pkts_mask) {
-                rte_pipeline_ah_packet_hijack(p, arp_pkts_mask);
+	   /* Hijack the Synproxy and ARP buffered packets */
+
+       if (unlikely(arp_hijack_mask || synp_hijack_mask)) {
+
+//                printf("Pkts hijacked arp = %lX, synp = %lX\n",
+//			              arp_hijack_mask, synp_hijack_mask);
+
+                rte_pipeline_ah_packet_hijack(p,(arp_hijack_mask | synp_hijack_mask));
         }
 
        pkts_drop_mask = packet_mask_in & ~keep_mask;
@@ -1663,9 +1713,6 @@ vfw_port_in_action_ipv4(struct rte_pipeline *p,
               rte_pipeline_ah_packet_drop(p, pkts_drop_mask);
        }
 
-       if (unlikely(hijack_mask != 0))
-              rte_pipeline_ah_packet_hijack(p, hijack_mask);
-
        vfw_pipe->counters->num_batch_pkts_sum += n_pkts;
        vfw_pipe->counters->num_pkts_measurements++;
 
@@ -1705,8 +1752,10 @@ vfw_port_in_action_ipv6(struct rte_pipeline *p,
 
        uint64_t packet_mask_in = RTE_LEN2MASK(n_pkts, uint64_t);
        uint64_t pkts_drop_mask;
-       uint64_t hijack_mask = 0;
-       uint64_t synproxy_reply_mask = 0;       /* for synproxy */
+       uint64_t synp_hijack_mask = 0;
+       uint64_t arp_hijack_mask = 0;
+//       uint64_t hijack_mask = 0;
+//       uint64_t synproxy_reply_mask = 0;       /* for synproxy */
        uint64_t keep_mask = packet_mask_in;
 
        uint64_t conntrack_mask = 0, connexist_mask = 0;
@@ -1782,8 +1831,8 @@ vfw_port_in_action_ipv6(struct rte_pipeline *p,
        if (likely(cnxn_tracking_is_active)) {
               rte_ct_cnxn_tracker_batch_lookup_type(ct, pkts,
                             &keep_mask, &ct_helper, IPv6_HEADER_SIZE);
-              synproxy_reply_mask = ct_helper.reply_pkt_mask;
-              hijack_mask = ct_helper.hijack_mask;
+//              synproxy_reply_mask = ct_helper.reply_pkt_mask;
+              synp_hijack_mask = ct_helper.hijack_mask;
 
        }
 
@@ -1795,7 +1844,7 @@ vfw_port_in_action_ipv6(struct rte_pipeline *p,
                                    ETHERNET_START));
        }
        rte_prefetch0((void*)in_port_dir_a);
-       rte_prefetch0(vfw_pipe->local_lib_nd_route_table);
+ //      rte_prefetch0(vfw_pipe->local_lib_nd_route_table);
        uint32_t i;
 
        for (i = 0; i < (n_pkts & (~0x3LLU)); i += 4) {
@@ -1820,9 +1869,9 @@ vfw_port_in_action_ipv6(struct rte_pipeline *p,
        }
 #else
        rte_prefetch0((void*)in_port_dir_a);
-       rte_prefetch0((void*) & vfw_pipe->local_lib_arp_route_table);
-       keep_mask = rte_vfw_arp_ipv6_packets(pkts, keep_mask,
-                     synproxy_reply_mask, vfw_pipe);
+
+	vfw_fwd_pkts_ipv6(pkts, &keep_mask, &arp_hijack_mask, vfw_pipe);
+
 #endif
 
        if (vfw_debug > 1) {
@@ -1834,6 +1883,16 @@ vfw_port_in_action_ipv6(struct rte_pipeline *p,
                                    (void *)keep_mask);
        }
 
+	/* Hijack the Synproxy and ARP buffered packets */
+
+        if (unlikely(arp_hijack_mask || synp_hijack_mask)) {
+
+//                printf("Pkts hijacked arp = %lX, synp = %lX\n",
+//			              arp_hijack_mask, synp_hijack_mask);
+
+                rte_pipeline_ah_packet_hijack(p,(arp_hijack_mask | synp_hijack_mask));
+        }
+
        /* Update mask before returning, so that bad packets are dropped */
 
        pkts_drop_mask = packet_mask_in & ~keep_mask;
@@ -1843,9 +1902,6 @@ vfw_port_in_action_ipv6(struct rte_pipeline *p,
               rte_pipeline_ah_packet_drop(p, pkts_drop_mask);
        }
 
-       if (unlikely(hijack_mask != 0))
-              rte_pipeline_ah_packet_hijack(p, hijack_mask);
-
        vfw_pipe->counters->num_batch_pkts_sum += n_pkts;
        vfw_pipe->counters->num_pkts_measurements++;