[l2l3 stack] implements new arp state machine & arp buffering

JIRA: SAMPLEVNF-23

This patch implements
  ~New arp state machine implementing new states
   like INCOMPLETE, COMPLETE, PROBE, STALE.
  ~removing unwanted code in arpicmp pipeline
  ~Implementing arp buffering, when arp is unresolved.
  ~Integratig the new changes with vCGNAPT
  ~Integrating the new changes with vACL
  ~Integrating the new changes with vFW.

Change-Id: If467ec035bc8de58463ea50d9e603a97f168c1a2
Signed-off-by: Vishwesh M Rudramuni <vishwesh.m.rudramuni@intel.com>

6 files changed, 588 insertions, 716 deletions

diff --git a/VNFs/vACL/pipeline/pipeline_acl_be.c b/VNFs/vACL/pipeline/pipeline_acl_be.c
index 039d6d59..b9386e61 100644
--- a/VNFs/vACL/pipeline/pipeline_acl_be.c
+++ b/VNFs/vACL/pipeline/pipeline_acl_be.c
@@ -144,6 +144,7 @@ static void *pipeline_acl_msg_req_dbg_handler(struct pipeline *p, void *msg);
 static pipeline_msg_req_handler custom_handlers[] = {
 	[PIPELINE_ACL_MSG_REQ_DBG] = pipeline_acl_msg_req_dbg_handler,
 };
+uint64_t arp_pkts_mask;
 
 uint8_t ACL_DEBUG;
 uint32_t local_get_nh_ipv4(uint32_t ip,
@@ -154,8 +155,8 @@ uint32_t local_get_nh_ipv4(uint32_t ip,
 
 	for (i = 0; i < p_acl->local_lib_arp_route_ent_cnt; i++) {
 		if (((p_acl->local_lib_arp_route_table[i].ip &
-		      p_acl->local_lib_arp_route_table[i].mask) ==
-		     (ip & p_acl->local_lib_arp_route_table[i].mask))) {
+			p_acl->local_lib_arp_route_table[i].mask) ==
+			(ip & p_acl->local_lib_arp_route_table[i].mask))) {
 			*port = p_acl->local_lib_arp_route_table[i].port;
 
 			*nhip = p_acl->local_lib_arp_route_table[i].nh;
@@ -175,8 +176,8 @@ static void do_local_nh_ipv4_cache(uint32_t dest_if, struct pipeline_acl *p_acl)
 		if (lib_arp_route_table[i].port == dest_if) {
 
 			struct lib_arp_route_table_entry *lentry =
-			    &p_acl->local_lib_arp_route_table
-			    [p_acl->local_lib_arp_route_ent_cnt];
+				&p_acl->local_lib_arp_route_table
+				[p_acl->local_lib_arp_route_ent_cnt];
 
 			lentry->ip = lib_arp_route_table[i].ip;
 			lentry->mask = lib_arp_route_table[i].mask;
@@ -463,6 +464,7 @@ pkt_work_acl_key(struct rte_pipeline *p,
 	uint64_t conntrack_mask = 0;
 	uint64_t connexist_mask = 0;
 	uint32_t dest_address = 0;
+	arp_pkts_mask = 0;
 	int dest_if = 0;
 	int status;
 	uint64_t pkts_drop_mask, pkts_mask = RTE_LEN2MASK(n_pkts, uint64_t);
@@ -918,103 +920,57 @@ pkt_work_acl_key(struct rte_pipeline *p,
 								       nhip));
 			uint32_t packet_length = rte_pktmbuf_pkt_len(pkt);
 			*nhip = 0;
-			if (is_phy_port_privte(phy_port)) {
-				dest_address = rte_bswap32(*dst_addr);
-				ret =
-				    local_get_nh_ipv4(dest_address, &dest_if,
-						      nhip, p_acl);
-				if (!ret) {
-					dest_if =
-					    get_prv_to_pub_port(&dest_address,
-								IP_VERSION_4);
-					do_local_nh_ipv4_cache(dest_if, p_acl);
+			struct arp_entry_data *ret_arp_data = NULL;
+			ret_arp_data = get_dest_mac_addr_port
+			    (dest_address, &dest_if, (struct ether_addr *) eth_dest);
+			*port_out_id = p_acl->port_out_id[dest_if];
+			if (arp_cache_dest_mac_present(dest_if)) {
+				ether_addr_copy(get_link_hw_addr(dest_if), (struct ether_addr *)eth_src);
+				arp_data_ptr[dest_if]->n_last_update = time(NULL);
+
+				if (unlikely(ret_arp_data && ret_arp_data->num_pkts)) {
+					printf("sending buffered packets\n");
+					arp_send_buffered_pkts(ret_arp_data,
+					(struct ether_addr *)eth_dest, *port_out_id);
+
 				}
-				*port_out_id = p_acl->port_out_id[dest_if];
 			} else {
-				dest_address = rte_bswap32(*dst_addr);
-
-				ret = local_get_nh_ipv4(dest_address, &dest_if,
-							nhip, p_acl);
-				if (!ret) {
-					dest_if =
-					    get_pub_to_prv_port(&dest_address,
-								IP_VERSION_4);
-					do_local_nh_ipv4_cache(dest_if, p_acl);
-				};
-				*port_out_id = p_acl->port_out_id[dest_if];
-			}
-			/* port = ACL_PRV_PORT_ID; */
+				if (unlikely(ret_arp_data == NULL)) {
 
-			int ret_mac = 0;
+					printf("%s: NHIP Not Found, "
+					"outport_id: %d\n", __func__,
+					*port_out_id);
 
-			ret_mac = get_dest_mac_addr_port
-			    (dest_address, &dest_if, &hw_addr);
-			if (ret_mac == ARP_FOUND) {
-				if (ACL_DEBUG) {
-					printf("MAC found for ip 0x%x,  "
-				"port %d - %02x:%02x:%02x:%02x:%02x:%02x\n",
-					     dest_address, phy_port,
-					     hw_addr.addr_bytes[0],
-					     hw_addr.addr_bytes[1],
-					     hw_addr.addr_bytes[2],
-					     hw_addr.addr_bytes[3],
-					     hw_addr.addr_bytes[4],
-					     hw_addr.addr_bytes[5]);
-					printf("Dest MAC before -  "
-					"%02x:%02x:%02x:%02x:%02x:%02x\n",
-					     eth_dest[0], eth_dest[1],
-					     eth_dest[2], eth_dest[3],
-					     eth_dest[4], eth_dest[5]);
+					/* Drop the pkt */
+					pkts_mask &= ~(1LLU << pos);
+					if (ACL_DEBUG)
+						printf("ACL after drop pkt_mask  "
+						"%lu, pkt_num %d\n",
+						pkts_mask, pos);
+						p_acl->counters->pkts_drop++;
+						continue;
 				}
 
-				memcpy(eth_dest, &hw_addr,
-				       sizeof(struct ether_addr));
-				if (ACL_DEBUG) {
-					printf("PktP %p, dest_macP %p\n", pkt,
-					       eth_dest);
-					printf("Dest MAC after -  "
-					"%02x:%02x:%02x:%02x:%02x:%02x\n",
-					     eth_dest[0], eth_dest[1],
-					     eth_dest[2], eth_dest[3],
-					     eth_dest[4], eth_dest[5]);
+				if (ret_arp_data->status == INCOMPLETE ||
+					ret_arp_data->status == PROBE) {
+					if (ret_arp_data->num_pkts >= NUM_DESC) {
+						/* Drop the pkt */
+						pkts_mask &= ~(1LLU << pos);
+						if (ACL_DEBUG)
+							printf("ACL after drop pkt_mask  "
+							"%lu, pkt_num %d\n",
+							pkts_mask, pos);
+						p_acl->counters->pkts_drop++;
+						continue;
+					} else {
+						arp_pkts_mask |= pkt_mask;
+						arp_queue_unresolved_packet(ret_arp_data, pkt);
+						continue;
+					}
 				}
-				if (is_phy_port_privte(phy_port))
-					memcpy(eth_src,
-					       get_link_hw_addr(dest_if),
-					       sizeof(struct ether_addr));
-				else
-					memcpy(eth_src,
-					       get_link_hw_addr(dest_if),
-					       sizeof(struct ether_addr));
-				p_acl->counters->tpkts_processed++;
-				p_acl->counters->bytes_processed +=
-				    packet_length;
 			}
 
-			else {
-				if (*nhip != 0) {
-					if (ACL_DEBUG)
-						printf("ACL requesting ARP for "
-							"ip %x, port %d\n",
-						     dest_address, phy_port);
-					if (ret_mac == ARP_NOT_FOUND)
-						request_arp(dest_if, *nhip);
-
-		/*      request_arp(p_acl->links_map[phy_port], *nhip); */
-				}
-				/* Drop packet by changing the mask */
-				if (ACL_DEBUG)
-					printf("ACL before drop pkt_mask  "
-							"%lu, pkt_num %d\n",
-					     pkts_mask, pos);
-				pkts_mask &= ~(1LLU << pos);
-				if (ACL_DEBUG)
-					printf("ACL after drop pkt_mask  "
-							"%lu, pkt_num %d\n",
-					     pkts_mask, pos);
-				p_acl->counters->pkts_drop++;
-			}
-		}
+		} /* end of if (hdr_chk == IPv4_HDR_VERSION) */
 
 		if (hdr_chk == IPv6_HDR_VERSION) {
 
@@ -1053,7 +1009,7 @@ pkt_work_acl_key(struct rte_pipeline *p,
 			uint8_t nhip[16];
 
 			nhip[0] =
-			    RTE_MBUF_METADATA_UINT8(pkt,
+			RTE_MBUF_METADATA_UINT8(pkt,
 						    META_DATA_OFFSET +
 						    offsetof(struct
 							     mbuf_acl_meta_data,
@@ -1176,6 +1132,11 @@ pkt_work_acl_key(struct rte_pipeline *p,
 	rte_pipeline_ah_packet_drop(p, pkts_drop_mask);
 	keep_mask = pkts_mask;
 
+	if (arp_pkts_mask) {
+		keep_mask &= ~(arp_pkts_mask);
+		rte_pipeline_ah_packet_hijack(p, arp_pkts_mask);
+	}
+
 	/* don't bother measuring if traffic very low, might skew stats */
 	uint32_t packets_this_iteration = __builtin_popcountll(pkts_mask);
 
@@ -1238,6 +1199,7 @@ pkt_work_acl_ipv4_key(struct rte_pipeline *p,
 	uint64_t conntrack_mask = 0;
 	uint64_t connexist_mask = 0;
 	uint32_t dest_address = 0;
+	arp_pkts_mask = 0;
 	int dest_if = 0;
 	int status;
 	uint64_t pkts_drop_mask, pkts_mask = RTE_LEN2MASK(n_pkts, uint64_t);
@@ -1711,76 +1673,56 @@ pkt_work_acl_ipv4_key(struct rte_pipeline *p,
 				*port_out_id = p_acl->port_out_id[dest_if];
 			}
 			/* port = ACL_PRV_PORT_ID; */
-			int ret_mac = 0;
 
-			ret_mac = get_dest_mac_addr_port
-			    (dest_address, &dest_if, &hw_addr);
+			struct arp_entry_data *ret_arp_data = NULL;
+			ret_arp_data = get_dest_mac_addr_port
+			    (dest_address, &dest_if, (struct ether_addr *)eth_dest);
+			*port_out_id = p_acl->port_out_id[dest_if];
 
-			if (ret_mac == ARP_FOUND) {
-				if (ACL_DEBUG) {
-					printf("MAC found for ip 0x%x, port  "
-					"%d - %02x:%02x:%02x:%02x:%02x:%02x\n",
-					     dest_address, phy_port,
-					     hw_addr.addr_bytes[0],
-					     hw_addr.addr_bytes[1],
-					     hw_addr.addr_bytes[2],
-					     hw_addr.addr_bytes[3],
-					     hw_addr.addr_bytes[4],
-					     hw_addr.addr_bytes[5]);
-					printf("Dest MAC before -  "
-					"%02x:%02x:%02x:%02x:%02x:%02x\n",
-					     eth_dest[0], eth_dest[1],
-					     eth_dest[2], eth_dest[3],
-					     eth_dest[4], eth_dest[5]);
-				}
+			if (arp_cache_dest_mac_present(dest_if)) {
+				ether_addr_copy(get_link_hw_addr(dest_if), (struct ether_addr *)eth_src);
+				arp_data_ptr[dest_if]->n_last_update = time(NULL);
+
+				if (unlikely(ret_arp_data && ret_arp_data->num_pkts)) {
+					printf("sending buffered packets\n");
+					arp_send_buffered_pkts(ret_arp_data,
+					(struct ether_addr *)eth_dest, *port_out_id);
 
-				memcpy(eth_dest, &hw_addr,
-				       sizeof(struct ether_addr));
-				if (ACL_DEBUG) {
-					printf("PktP %p, dest_macP %p\n", pkt,
-					       eth_dest);
-					printf("Dest MAC after -  "
-					"%02x:%02x:%02x:%02x:%02x:%02x\n",
-					     eth_dest[0], eth_dest[1],
-					     eth_dest[2], eth_dest[3],
-					     eth_dest[4], eth_dest[5]);
 				}
-				if (is_phy_port_privte(phy_port))
-					memcpy(eth_src,
-					       get_link_hw_addr(dest_if),
-					       sizeof(struct ether_addr));
-				else
-					memcpy(eth_src,
-					       get_link_hw_addr(dest_if),
-					       sizeof(struct ether_addr));
-				p_acl->counters->tpkts_processed++;
-				p_acl->counters->bytes_processed +=
-				    packet_length;
-			}
+			} else {
+				if (unlikely(ret_arp_data == NULL)) {
 
-			else {
-				if (*nhip != 0) {
-					if (ACL_DEBUG)
-						printf("ACL requesting ARP for "
-							"ip %x, port %d\n",
-						     dest_address, phy_port);
-					if (ret_mac == ARP_NOT_FOUND)
-						request_arp(dest_if, *nhip);
+					printf("%s: NHIP Not Found, "
+					"outport_id: %d\n", __func__,
+					*port_out_id);
 
-			/*  request_arp(p_acl->links_map[phy_port], *nhip); */
+					/* Drop the pkt */
+					pkts_mask &= ~(1LLU << pos);
+					if (ACL_DEBUG)
+						printf("ACL after drop pkt_mask  "
+						"%lu, pkt_num %d\n",
+						pkts_mask, pos);
+						p_acl->counters->pkts_drop++;
+					continue;
 				}
-				/* Drop packet by changing the mask */
-				if (ACL_DEBUG)
-					printf
-					    ("ACL before drop pkt_mask  "
-							"%lu, pkt_num %d\n",
-					     pkts_mask, pos);
-				pkts_mask &= ~(1LLU << pos);
-				if (ACL_DEBUG)
-					printf("ACL after drop pkt_mask  "
+
+				if (ret_arp_data->status == INCOMPLETE ||
+				ret_arp_data->status == PROBE) {
+					if (ret_arp_data->num_pkts >= NUM_DESC) {
+						/* Drop the pkt */
+						pkts_mask &= ~(1LLU << pos);
+						if (ACL_DEBUG)
+							printf("ACL after drop pkt_mask  "
 							"%lu, pkt_num %d\n",
-					     pkts_mask, pos);
-				p_acl->counters->pkts_drop++;
+							pkts_mask, pos);
+						p_acl->counters->pkts_drop++;
+						continue;
+					} else {
+					arp_pkts_mask |= pkt_mask;
+					arp_queue_unresolved_packet(ret_arp_data, pkt);
+					continue;
+					}
+				}
 			}
 		}
 #if 0
@@ -1923,6 +1865,11 @@ pkt_work_acl_ipv4_key(struct rte_pipeline *p,
 	rte_pipeline_ah_packet_drop(p, pkts_drop_mask);
 	keep_mask = pkts_mask;
 
+	if (arp_pkts_mask) {
+		keep_mask &= ~(arp_pkts_mask);
+		rte_pipeline_ah_packet_hijack(p, arp_pkts_mask);
+	}
+
 	/* don't bother measuring if traffic very low, might skew stats */
 	uint32_t packets_this_iteration = __builtin_popcountll(pkts_mask);
 
@@ -1934,7 +1881,7 @@ pkt_work_acl_ipv4_key(struct rte_pipeline *p,
 	}
 	if (ACL_DEBUG)
 		printf("Leaving pkt_work_acl_key pkts_mask = %p\n",
-		       (void *)pkts_mask);
+			(void *)pkts_mask);
 
 	return 0;
 }
@@ -1983,6 +1930,7 @@ pkt_work_acl_ipv6_key(struct rte_pipeline *p,
 	uint64_t conntrack_mask = 0;
 	uint64_t connexist_mask = 0;
 	uint32_t dest_address = 0;
+	arp_pkts_mask = 0;
 	int dest_if = 0;
 	int status;
 	uint64_t pkts_drop_mask, pkts_mask = RTE_LEN2MASK(n_pkts, uint64_t);
@@ -2691,6 +2639,11 @@ pkt_work_acl_ipv6_key(struct rte_pipeline *p,
 	rte_pipeline_ah_packet_drop(p, pkts_drop_mask);
 	keep_mask = pkts_mask;
 
+	if (arp_pkts_mask) {
+		keep_mask &= ~(arp_pkts_mask);
+		rte_pipeline_ah_packet_hijack(p, arp_pkts_mask);
+	}
+
 	/* don't bother measuring if traffic very low, might skew stats */
 	uint32_t packets_this_iteration = __builtin_popcountll(pkts_mask);
 
diff --git a/VNFs/vCGNAPT/init.c b/VNFs/vCGNAPT/init.c
index aa3868d8..3e731839 100644
--- a/VNFs/vCGNAPT/init.c
+++ b/VNFs/vCGNAPT/init.c
@@ -1729,11 +1729,13 @@ int app_init(struct app_params *app)
 	app_init_pipelines(app);
 	app_init_threads(app);
 
+	#ifdef L3_STACK_SUPPORT
         l3fwd_init();
         create_arp_table();
         create_nd_table();
         populate_lpm_routes();
         print_interface_details();
+	#endif
 
 	return 0;
 }
diff --git a/VNFs/vCGNAPT/pipeline/pipeline_cgnapt_be.c b/VNFs/vCGNAPT/pipeline/pipeline_cgnapt_be.c
index 2da8b5e4..2fbc71d8 100644
--- a/VNFs/vCGNAPT/pipeline/pipeline_cgnapt_be.c
+++ b/VNFs/vCGNAPT/pipeline/pipeline_cgnapt_be.c
@@ -80,7 +80,10 @@
 /* To maintain all cgnapt pipeline pointers used for all stats */
 struct pipeline_cgnapt *all_pipeline_cgnapt[128];
 uint8_t n_cgnapt_pipeline;
+struct pipeline_cgnapt *global_pnat;
 
+uint64_t arp_pkts_mask;
+extern struct arp_entry_data *arp_data_ptr[16];
 /* To know egress or ingress port */
 static uint8_t cgnapt_in_port_egress_prv[PIPELINE_MAX_PORT_IN];
 static uint8_t cgnapt_prv_que_port_index[PIPELINE_MAX_PORT_IN];
@@ -140,6 +143,7 @@ struct rte_hash_parameters napt_common_table_hash_params = {
 };
 
 /***** ARP local cache *****/
+
 uint8_t link_hw_laddr_valid[MAX_NUM_LOCAL_MAC_ADDRESS] = {
 	0, 0, 0, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0, 0, 0
@@ -178,7 +182,6 @@ static uint32_t local_get_nh_ipv4(
 	uint32_t *port,
 	uint32_t *nhip,
 	struct pipeline_cgnapt *p_nat);
-
 static void do_local_nh_ipv4_cache(
 	uint32_t dest_if,
 	struct pipeline_cgnapt *p_nat);
@@ -220,6 +223,11 @@ uint64_t nextPowerOf2(uint64_t n)
 	return n;
 }
 
+void remove_local_cache(uint8_t port)
+{
+	link_hw_laddr_valid[port] = 0;
+}
+
 /**
  * Function to get MAC addr of local link
  *
@@ -230,10 +238,10 @@ uint64_t nextPowerOf2(uint64_t n)
  *  Outport Link MAC addr
  */
 
-struct ether_addr *get_local_link_hw_addr(uint8_t out_port)
-{
-	return &link_hw_laddr[out_port];
-}
+//struct ether_addr *get_local_link_hw_addr(uint8_t out_port)
+//{
+//	return &link_hw_laddr[out_port];
+//}
 
 /**
  * Function to get MAC addr from array instead of hash table
@@ -910,6 +918,33 @@ void sw_checksum(struct rte_mbuf *pkt, enum PKT_TYPE ver)
 	}
 }
 
+void print_pkt_info(uint8_t *eth_dest, struct ether_addr *hw_addr, 
+		uint32_t dest_address, uint32_t port_id, struct rte_mbuf *pkt)
+{
+
+if (CGNAPT_DEBUG > 2) {
+	printf("MAC Found ip 0x%x, port %d - %02x:%02x:%02x:%02x:%02x:%02x  \n",
+	dest_address, port_id, hw_addr->addr_bytes[0], hw_addr->addr_bytes[1],
+	hw_addr->addr_bytes[2], hw_addr->addr_bytes[3], hw_addr->addr_bytes[4],
+	hw_addr->addr_bytes[5]);
+
+	printf("Dest MAC before - %02x:%02x:%02x:%02x:%02x:%02x      \n",
+		eth_dest[0], eth_dest[1], eth_dest[2], eth_dest[3], eth_dest[4],
+		eth_dest[5]);
+}
+
+if (CGNAPT_DEBUG > 2) {
+	printf("Dest MAC after - "
+		"%02x:%02x:%02x:%02x:%02x:%02x      \n",
+		eth_dest[0], eth_dest[1],
+		eth_dest[2], eth_dest[3],
+		eth_dest[4], eth_dest[5]);
+}
+
+if (CGNAPT_DEBUG > 4)
+	print_pkt(pkt);
+}
+
 static uint8_t check_arp_icmp(
 	struct rte_mbuf *pkt,
 	uint64_t pkt_mask,
@@ -925,7 +960,7 @@ static uint8_t check_arp_icmp(
 
 	/* ARP outport number */
 	uint16_t out_port = p_nat->p.n_ports_out - 1;
-
+	printf("check_arp_icmp called*****\n");
 	uint8_t *protocol;
 	uint32_t prot_offset;
 
@@ -2169,11 +2204,34 @@ static int cgnapt_in_port_ah_mix(struct rte_pipeline *rte_p,
 			}
 
 			*outport_id = p_nat->outport_id[dest_if];
-			int ret;
-			ret = get_dest_mac_addr_port(dest_address,
-				&dest_if, &hw_addr);
+			struct arp_entry_data *ret_arp_data;
+			ret_arp_data = get_dest_mac_addr_port(dest_address,
+				&dest_if, (struct ether_addr *)&hw_addr);
+
+			if (unlikely(ret_arp_data == NULL)) {
+
+				printf("%s: NHIP Not Found, nhip: %x, "
+				"outport_id: %d\n", __func__, nhip,
+				*outport_id);
+
+				/* Drop the pkt */
+				p_nat->invalid_packets |= pkt_mask;
+				p_nat->naptDroppedPktCount++;
+
+				#ifdef CGNAPT_DEBUGGING
+				p_nat->naptDroppedPktCount4++;
+				#endif
+				continue;
+			}
+
+			if (ret_arp_data->status == COMPLETE) {
+				
+				if (ret_arp_data->num_pkts) {
+					p_nat->naptedPktCount += ret_arp_data->num_pkts;
+					arp_send_buffered_pkts(ret_arp_data,
+						 &hw_addr, *outport_id);
+				}
 
-			if (ret == ARP_FOUND) {
 				memcpy(eth_dest, &hw_addr,
 					sizeof(struct ether_addr));
 				memcpy(eth_src, get_link_hw_addr(dest_if),
@@ -2208,23 +2266,22 @@ static int cgnapt_in_port_ah_mix(struct rte_pipeline *rte_p,
 					print_pkt(pkts[pkt_index]);
 				#endif
 
-			} else{
-				if (ret == ARP_NOT_FOUND) {
-					/* Commented code may be required
-					 * for future use, Please keep it */
-					//request_arp(*outport_id, nhip,
-					//	p_nat->p.p);
-					printf("%s: ARP Not Found, nhip: %x, "
-					"outport_id: %d\n", __func__, nhip,
-					*outport_id);
-				}
+			} else if (ret_arp_data->status == INCOMPLETE || 
+				ret_arp_data->status == PROBE) {
+				if (ret_arp_data->num_pkts >= NUM_DESC) {
+					/* Drop the pkt */
+					p_nat->invalid_packets |= pkt_mask;
+					p_nat->naptDroppedPktCount++;
 
-				p_nat->invalid_packets |= pkt_mask;
-				p_nat->naptDroppedPktCount++;
-				#ifdef CGNAPT_DEBUGGING
-				p_nat->naptDroppedPktCount4++;
-				#endif
-				continue;
+					#ifdef CGNAPT_DEBUGGING
+					p_nat->naptDroppedPktCount4++;
+					#endif
+					continue;
+				} else {
+					arp_queue_unresolved_packet(ret_arp_data,
+						pkts[pkt_index]);
+					continue;
+				}
 			}
 
 			#ifdef CGNAPT_DBG_PRNT
@@ -2396,61 +2453,76 @@ static int cgnapt_in_port_ah_mix(struct rte_pipeline *rte_p,
 				};
 
 				*outport_id = p_nat->outport_id[dest_if];
-				int ret;
-				ret = get_dest_mac_addr_port(dest_address,
-					&dest_if, &hw_addr);
+				struct arp_entry_data *ret_arp_data;
+				ret_arp_data = get_dest_mac_addr_port(dest_address,
+					&dest_if, (struct ether_addr *)&hw_addr);
+
+				if (unlikely(ret_arp_data == NULL)) {
+
+					printf("%s: NHIP Not Found, nhip: %x, "
+					"outport_id: %d\n", __func__, nhip,
+					*outport_id);
+
+					/* Drop the pkt */
+					p_nat->invalid_packets |= pkt_mask;
+					p_nat->naptDroppedPktCount++;
+
+					#ifdef CGNAPT_DEBUGGING
+					p_nat->naptDroppedPktCount4++;
+					#endif
+					continue;
+				}
+
+				if (ret_arp_data->status == COMPLETE) {
+
+					if (ret_arp_data->num_pkts) {
+						p_nat->naptedPktCount +=
+							 ret_arp_data->num_pkts;
+						arp_send_buffered_pkts(ret_arp_data,
+							 &hw_addr, *outport_id);
+					}
 
-				if (ret == ARP_FOUND) {
 					memcpy(eth_dest, &hw_addr,
 						sizeof(struct ether_addr));
 					memcpy(eth_src, get_link_hw_addr(
 						dest_if),
 						sizeof(struct ether_addr));
-				#ifdef CGNAPT_DBG_PRNT
-				if (CGNAPT_DEBUG > 2) {
-				printf("MAC found for ip 0x%x, port %d - "
-				"%02x:%02x:%02x:%02x:%02x:%02x\n",
-				dest_address, *outport_id,
-				 hw_addr.addr_bytes[0], hw_addr.addr_bytes[1],
-				 hw_addr.addr_bytes[2], hw_addr.addr_bytes[3],
-				 hw_addr.addr_bytes[4], hw_addr.addr_bytes[5]);
+					#ifdef CGNAPT_DBG_PRNT
+					if (CGNAPT_DEBUG > 2) {
+					printf("MAC found for ip 0x%x, port %d - "
+					"%02x:%02x:%02x:%02x:%02x:%02x\n",
+					dest_address, *outport_id,
+				 	hw_addr.addr_bytes[0], hw_addr.addr_bytes[1],
+				 	hw_addr.addr_bytes[2], hw_addr.addr_bytes[3],
+				 	hw_addr.addr_bytes[4], hw_addr.addr_bytes[5]);
 
-				printf("Dest MAC before - "
-				"%02x:%02x:%02x:%02x:%02x:%02x\n",
+					printf("Dest MAC before - "
+					"%02x:%02x:%02x:%02x:%02x:%02x\n",
 					 eth_dest[0], eth_dest[1], eth_dest[2],
 					 eth_dest[3], eth_dest[4], eth_dest[5]);
-				}
-				#endif
+					}
+					#endif
 
-				#ifdef CGNAPT_DBG_PRNT
-				if (CGNAPT_DEBUG > 2) {
-				printf("Dest MAC after - "
-				"%02x:%02x:%02x:%02x:%02x:%02x\n",
+					#ifdef CGNAPT_DBG_PRNT
+					if (CGNAPT_DEBUG > 2) {
+					printf("Dest MAC after - "
+					"%02x:%02x:%02x:%02x:%02x:%02x\n",
 					 eth_dest[0], eth_dest[1], eth_dest[2],
 					 eth_dest[3], eth_dest[4], eth_dest[5]);
-				}
-				#endif
+					}
+					#endif
 
-				#ifdef CGNAPT_DBG_PRNT
-				if (CGNAPT_DEBUG > 4)
-					print_pkt(pkts[pkt_index]);
-				#endif
+					#ifdef CGNAPT_DBG_PRNT
+					if (CGNAPT_DEBUG > 4)
+						print_pkt(pkts[pkt_index]);
+					#endif
 
-				} else {
-					if (ret == ARP_NOT_FOUND) {
-					printf("%s: ARP Not Found, nhip: %x, "
-					"outport_id: %d\n", __func__, nhip,
-					*outport_id);
-					}
-						//request_arp(*outport_id,
-						//	nhip, p_nat->p.p);
-				p_nat->invalid_packets |= pkt_mask;
-				p_nat->naptDroppedPktCount++;
-				#ifdef CGNAPT_DEBUGGING
-				p_nat->naptDroppedPktCount4++;
-				#endif
-				continue;
-			}
+				} else if (ret_arp_data->status == INCOMPLETE || 
+					ret_arp_data->status == PROBE) {
+					arp_queue_unresolved_packet(ret_arp_data,
+						pkts[pkt_index]);
+					continue;
+				}
 
 			if (*protocol == IP_PROTOCOL_ICMP) {
 				// Query ID reverse translation done here
@@ -2547,7 +2619,7 @@ static int cgnapt_in_port_ah_ipv4_prv(struct rte_pipeline *rte_p,
 	p_nat->pkt_burst_cnt = 0;	/* for dynamic napt */
 	p_nat->valid_packets = rte_p->pkts_mask;	/*n_pkts; */
 	p_nat->invalid_packets = 0;
-
+	arp_pkts_mask = 0;
 	#ifdef CGNAPT_DBG_PRNT
 	if (CGNAPT_DEBUG > 1)
 		printf("cgnapt_key hit fn: %" PRIu32 "\n", n_pkts);
@@ -2567,6 +2639,7 @@ static int cgnapt_in_port_ah_ipv4_prv(struct rte_pipeline *rte_p,
 
 	if (unlikely(p_nat->valid_packets == 0)) {
 		/* no suitable packet for lookup */
+		printf("no suitable valid packets\n");
 		rte_pipeline_ah_packet_drop(rte_p, p_nat->invalid_packets);
 		return p_nat->valid_packets;
 	}
@@ -2600,12 +2673,20 @@ static int cgnapt_in_port_ah_ipv4_prv(struct rte_pipeline *rte_p,
 						[p_nat->lkup_indx[j]]);
 	}
 
+	//prefetch();
+
+
 	for (i = 0; i < (n_pkts & (~0x3LLU)); i += 4)
 		pkt4_work_cgnapt_ipv4_prv(pkts, i, arg, p_nat);
 
 	for (; i < n_pkts; i++)
 		pkt_work_cgnapt_ipv4_prv(pkts, i, arg, p_nat);
 
+	if (arp_pkts_mask) {
+		p_nat->valid_packets &= ~(arp_pkts_mask);
+		rte_pipeline_ah_packet_hijack(rte_p, arp_pkts_mask);
+	}
+
 	if (p_nat->invalid_packets) {
 		/* get rid of invalid packets */
 		rte_pipeline_ah_packet_drop(rte_p, p_nat->invalid_packets);
@@ -2682,7 +2763,7 @@ static int cgnapt_in_port_ah_ipv4_pub(struct rte_pipeline *rte_p,
 	p_nat->pkt_burst_cnt = 0;	/* for dynamic napt */
 	p_nat->valid_packets = rte_p->pkts_mask;	/*n_pkts; */
 	p_nat->invalid_packets = 0;
-
+	arp_pkts_mask = 0;
 	#ifdef CGNAPT_DBG_PRNT
 	if (CGNAPT_DEBUG > 1)
 		printf("cgnapt_key hit fn: %" PRIu32 "\n", n_pkts);
@@ -2701,6 +2782,7 @@ static int cgnapt_in_port_ah_ipv4_pub(struct rte_pipeline *rte_p,
 	p_nat->valid_packets &= ~(p_nat->invalid_packets);
 
 	if (unlikely(p_nat->valid_packets == 0)) {
+		printf("no valid packets in pub\n");
 		/* no suitable packet for lookup */
 		rte_pipeline_ah_packet_drop(rte_p, p_nat->invalid_packets);
 		return p_nat->valid_packets;
@@ -2741,6 +2823,11 @@ static int cgnapt_in_port_ah_ipv4_pub(struct rte_pipeline *rte_p,
 	for (; i < n_pkts; i++)
 		pkt_work_cgnapt_ipv4_pub(pkts, i, arg, p_nat);
 
+	if (arp_pkts_mask) {
+		rte_pipeline_ah_packet_hijack(rte_p, arp_pkts_mask);
+		p_nat->valid_packets &= ~(arp_pkts_mask);
+	}
+
 	if (p_nat->invalid_packets) {
 		/* get rid of invalid packets */
 		rte_pipeline_ah_packet_drop(rte_p, p_nat->invalid_packets);
@@ -3706,6 +3793,7 @@ pkt_work_cgnapt_key_ipv4_pub(
  *  A pointer to main CGNAPT structure
  *
  */
+uint64_t last_update;
 void
 pkt_work_cgnapt_ipv4_prv(
 	struct rte_mbuf **pkts,
@@ -3849,54 +3937,31 @@ pkt_work_cgnapt_ipv4_prv(
 		#endif
 		return;
 	}
-
+	last_update = rte_rdtsc();
 	dest_address = rte_bswap32(*dst_addr);
-	/*Multiport Changes */
 	uint32_t nhip = 0;
-	uint32_t ret;
-	ret = local_get_nh_ipv4(dest_address, &dest_if, &nhip, p_nat);
-	if (!ret) {
-		dest_if = get_prv_to_pub_port(&dest_address, IP_VERSION_4);
-
-		if (dest_if == INVALID_DESTIF) {
-			p_nat->invalid_packets |= pkt_mask;
-			p_nat->naptDroppedPktCount++;
-			#ifdef CGNAPT_DEBUGGING
-			p_nat->naptDroppedPktCount6++;
-			#endif
-			return;
-		}
-
-		do_local_nh_ipv4_cache(dest_if, p_nat);
-	}
-
+	struct arp_entry_data *ret_arp_data = NULL;
+	ret_arp_data = get_dest_mac_addr_port(dest_address, &dest_if, (struct ether_addr *)eth_dest);
 	*outport_id = p_nat->outport_id[dest_if];
 
-	#ifdef CGNAPT_DBG_PRNT
-	if (CGNAPT_DEBUG > 2)
-		printf("Egress: \tphy_port:%d\t get_prv_to_pub():%d "
-				"\tout_port:%d\n", pkt->port, dest_if,
-				*outport_id);
-	#endif
+	if (arp_cache_dest_mac_present(dest_if)) {
+		ether_addr_copy(get_link_hw_addr(dest_if),(struct ether_addr *)eth_src);
+		arp_data_ptr[dest_if]->n_last_update = time(NULL);
 
-	if (local_dest_mac_present(dest_if)) {
-		memcpy(eth_dest,
-				get_local_link_hw_addr(dest_if),
-				sizeof(struct ether_addr));
-		memcpy(eth_src, get_link_hw_addr(dest_if),
-				sizeof(struct ether_addr));
+		if (unlikely(ret_arp_data && ret_arp_data->num_pkts)) {
+			printf("sending buffered packets\n");
+			p_nat->naptedPktCount += ret_arp_data->num_pkts;
+			arp_send_buffered_pkts(ret_arp_data,
+				 (struct ether_addr *)eth_dest, *outport_id);
+
+		}
 	} else {
-		int ret;
-		ret = get_dest_mac_addr_port(dest_address, &dest_if, &hw_addr);
 
-		if (unlikely(ret != ARP_FOUND)) {
+		if (unlikely(ret_arp_data == NULL)) {
 
-			if (unlikely(ret == ARP_NOT_FOUND)) {
-				//request_arp(*outport_id, nhip, p_nat->p.p);
-				printf("%s: ARP Not Found, nhip: %x, "
-				"outport_id: %d\n", __func__, nhip,
-				*outport_id);
-			}
+			printf("%s: NHIP Not Found, nhip:%x , "
+			"outport_id: %d\n", __func__, nhip,
+			*outport_id);
 
 			/* Drop the pkt */
 			p_nat->invalid_packets |= pkt_mask;
@@ -3906,41 +3971,26 @@ pkt_work_cgnapt_ipv4_prv(
 			p_nat->naptDroppedPktCount4++;
 			#endif
 			return;
-
-		}
-
-		#ifdef CGNAPT_DBG_PRNT
-		if (CGNAPT_DEBUG > 2) {
-			printf("MAC found for ip 0x%x, port %d - %02x:%02x: "
-			"%02x:%02x:%02x:%02x\n", dest_address,
-			*outport_id,
-			hw_addr.addr_bytes[0], hw_addr.addr_bytes[1],
-			hw_addr.addr_bytes[2], hw_addr.addr_bytes[3],
-			hw_addr.addr_bytes[4], hw_addr.addr_bytes[5]);
-
-			printf("Dest MAC before - %02x:%02x:%02x: "
-			"%02x:%02x:%02x\n", eth_dest[0], eth_dest[1],
-			eth_dest[2], eth_dest[3], eth_dest[4], eth_dest[5]);
 		}
 
-		#endif
-
-		memcpy(eth_dest, &hw_addr, sizeof(struct ether_addr));
-
-		link_hw_laddr_valid[dest_if] = 1;
-		memcpy(&link_hw_laddr[dest_if], &hw_addr,
-				sizeof(struct ether_addr));
+		if (ret_arp_data->status == INCOMPLETE || 
+			   ret_arp_data->status == PROBE) {
+				if (ret_arp_data->num_pkts >= NUM_DESC) {
+					/* Drop the pkt */
+					p_nat->invalid_packets |= pkt_mask;
+					p_nat->naptDroppedPktCount++;
 
-		#ifdef CGNAPT_DBG_PRNT
-		if (CGNAPT_DEBUG > 2) {
-			printf("Dest MAC after - %02x:%02x:%02x:%02x:%02x"
-			":%02x\n", eth_dest[0], eth_dest[1], eth_dest[2],
-			eth_dest[3], eth_dest[4], eth_dest[5]);
+					#ifdef CGNAPT_DEBUGGING
+					p_nat->naptDroppedPktCount4++;
+					#endif
+					return;
+				} else {
+					arp_pkts_mask |= pkt_mask;
+					arp_queue_unresolved_packet(ret_arp_data, pkt);
+					return;
+				}
 		}
-		#endif
 
-		memcpy(eth_src, get_link_hw_addr(dest_if),
-				sizeof(struct ether_addr));
 	}
 
 	{
@@ -4234,55 +4284,33 @@ pkt_work_cgnapt_ipv4_pub(
 			#endif
 			return;
 		}
+	}
 
 	dest_address = entry->data.u.prv_ip;
+	struct arp_entry_data *ret_arp_data = NULL;
+	ret_arp_data = get_dest_mac_addr_port(dest_address, &dest_if, (struct ether_addr *)eth_dest);
+	*outport_id = p_nat->outport_id[dest_if];
 
-	ret = local_get_nh_ipv4(dest_address, &dest_if, &nhip, p_nat);
-	if (!ret) {
-		dest_if = get_prv_to_pub_port(&dest_address, IP_VERSION_4);
+	if (arp_cache_dest_mac_present(dest_if)) {
+		ether_addr_copy(get_link_hw_addr(dest_if), (struct ether_addr *)eth_src);
+		arp_data_ptr[dest_if]->n_last_update = time(NULL);
 
-		if (dest_if == INVALID_DESTIF) {
-			p_nat->invalid_packets |= pkt_mask;
-			p_nat->naptDroppedPktCount++;
-			#ifdef CGNAPT_DEBUGGING
-			p_nat->naptDroppedPktCount6++;
-			#endif
-			return;
+		if (ret_arp_data && ret_arp_data->num_pkts) {
+			printf("sending buffered packets\n");
+			p_nat->naptedPktCount += ret_arp_data->num_pkts;
+			arp_send_buffered_pkts(ret_arp_data,
+				 (struct ether_addr *)eth_dest, *outport_id);
 		}
 
-		do_local_nh_ipv4_cache(dest_if, p_nat);
-	}
-
-		*outport_id = p_nat->outport_id[dest_if];
-
-		#ifdef CGNAPT_DBG_PRNT
-		if (CGNAPT_DEBUG > 2)
-			printf("Ingress: \tphy_port:%d\t get_pub_to_prv():%d "
-			"\tout_port%d\n", pkt->port, dest_if, *outport_id);
-		#endif
-	}
-
-	if (local_dest_mac_present(dest_if)) {
-		memcpy(eth_dest,
-				 get_local_link_hw_addr(dest_if),
-				 sizeof(struct ether_addr));
-		memcpy(eth_src, get_link_hw_addr(dest_if),
-				 sizeof(struct ether_addr));
 	} else {
-		int ret;
-		ret = get_dest_mac_addr_port(dest_address, &dest_if, &hw_addr);
 
-		if (unlikely(ret != ARP_FOUND)) {
-
-			if (unlikely(ret == ARP_NOT_FOUND)) {
-				/* Commented code may be required for debug
-				 * and future use, Please keep it */
-				//request_arp(*outport_id, nhip, p_nat->p.p);
-				printf("%s: ARP Not Found, nhip: %x, "
-				"outport_id: %d\n", __func__, nhip,
-				*outport_id);
+		if (unlikely(ret_arp_data == NULL)) {
 
-			}
+			/* Commented code may be required for debug
+			 * and future use, Please keep it */
+			printf("%s: NHIP Not Found, nhip: %x, "
+			"outport_id: %d\n", __func__, nhip,
+			*outport_id);
 
 			/* Drop the pkt */
 			p_nat->invalid_packets |= pkt_mask;
@@ -4294,42 +4322,24 @@ pkt_work_cgnapt_ipv4_pub(
 			return;
 
 		}
-		#ifdef CGNAPT_DBG_PRNT
-		if (CGNAPT_DEBUG > 2) {
-			printf
-				("MAC found for ip 0x%x, port %d - %02x:%02x: "
-				"%02x:%02x:%02x:%02x\n", dest_address,
-				*outport_id,
-				hw_addr.addr_bytes[0], hw_addr.addr_bytes[1],
-				hw_addr.addr_bytes[2], hw_addr.addr_bytes[3],
-				hw_addr.addr_bytes[4], hw_addr.addr_bytes[5]
-				);
-
-			printf
-				("Dest MAC before - %02x:%02x:%02x:%02x "
-				":%02x:%02x\n", eth_dest[0], eth_dest[1],
-				eth_dest[2], eth_dest[3], eth_dest[4],
-				eth_dest[5]);
-		}
-		#endif
 
-		memcpy(eth_dest, &hw_addr, sizeof(struct ether_addr));
-
-		link_hw_laddr_valid[dest_if] = 1;
-		memcpy(&link_hw_laddr[dest_if], &hw_addr,
-				 sizeof(struct ether_addr));
+		if (ret_arp_data->status == INCOMPLETE || 
+			ret_arp_data->status == PROBE) {
+			if (ret_arp_data->num_pkts >= NUM_DESC) {
+				/* Drop the pkt */
+				p_nat->invalid_packets |= pkt_mask;
+				p_nat->naptDroppedPktCount++;
 
-		#ifdef CGNAPT_DBG_PRNT
-		if (CGNAPT_DEBUG > 2) {
-			printf("Dest MAC after - "
-			"%02x:%02x:%02x:%02x:%02x:%02x\n",
-			eth_dest[0], eth_dest[1], eth_dest[2], eth_dest[3],
-			eth_dest[4], eth_dest[5]);
+				#ifdef CGNAPT_DEBUGGING
+				p_nat->naptDroppedPktCount4++;
+				#endif
+				return;
+			} else {
+				arp_pkts_mask |= pkt_mask;
+				arp_queue_unresolved_packet(ret_arp_data, pkt);
+				return;
+			}
 		}
-		#endif
-
-		memcpy(eth_src, get_link_hw_addr(dest_if),
-				 sizeof(struct ether_addr));
 	}
 
 	{
@@ -4466,7 +4476,7 @@ pkt_work_cgnapt_ipv4_pub(
 		if (ct_position < 0){
 			p_nat->invalid_packets |= pkt_mask;
 
-                        p_nat->naptDroppedPktCount++;
+			p_nat->naptDroppedPktCount++;
 			return;
 		}
 			#ifdef ALGDBG
@@ -4708,101 +4718,60 @@ pkt4_work_cgnapt_ipv4_prv(
 				#endif
 				continue;
 			}
-
-			dest_address = rte_bswap32(*dst_addr);
-		ret = local_get_nh_ipv4(dest_address, &dest_if, &nhip, p_nat);
-		if (!ret) {
-			dest_if = get_prv_to_pub_port(&dest_address,
-					IP_VERSION_4);
-		if (dest_if == INVALID_DESTIF) {
-			p_nat->invalid_packets |= pkt_mask;
-			p_nat->naptDroppedPktCount++;
-			#ifdef CGNAPT_DEBUGGING
-			p_nat->naptDroppedPktCount6++;
-			#endif
-			continue;
-		}
-			do_local_nh_ipv4_cache(dest_if, p_nat);
 		}
-			*outport_id = p_nat->outport_id[dest_if];
 
-		#ifdef CGNAPT_DBG_PRNT
-		if (CGNAPT_DEBUG > 2)
-			printf("Egress: \tphy_port:%d\t "
-			"get_prv_to_pub():%d \tout_port:%d\n",
-				 pkt->port, dest_if, *outport_id);
-		#endif
-		}
+		dest_address = rte_bswap32(*dst_addr);
+		struct arp_entry_data *ret_arp_data = NULL;
+		uint64_t start, end;
+		ret_arp_data = get_dest_mac_addr_port(dest_address, &dest_if, (struct ether_addr *)eth_dest);
+		*outport_id = p_nat->outport_id[dest_if];
+		if (arp_cache_dest_mac_present(dest_if)) {
+			ether_addr_copy(get_link_hw_addr(dest_if), (struct ether_addr *)eth_src);
+			arp_data_ptr[dest_if]->n_last_update = time(NULL);
+		
+			if (ret_arp_data && ret_arp_data->num_pkts) {
+				printf("sending buffered packets\n");
+				p_nat->naptedPktCount += ret_arp_data->num_pkts;
+				arp_send_buffered_pkts(ret_arp_data,
+					 (struct ether_addr *)eth_dest, *outport_id);
+			}
 
-		if (local_dest_mac_present(dest_if)) {
-			memcpy(eth_dest,
-					 get_local_link_hw_addr(dest_if),
-					 sizeof(struct ether_addr));
-			memcpy(eth_src,
-					 get_link_hw_addr(dest_if),
-					 sizeof(struct ether_addr));
 		} else {
-		int ret;
-		ret = get_dest_mac_addr_port(dest_address, &dest_if, &hw_addr);
 
-		if (unlikely(ret != ARP_FOUND)) {
+			if (unlikely(ret_arp_data == NULL)) {
 
-			if (unlikely(ret == ARP_NOT_FOUND)) {
 				printf("%s: ARP Not Found, nhip: %x, "
 				"outport_id: %d\n", __func__, nhip,
 				*outport_id);
-				//request_arp(*outport_id, nhip, p_nat->p.p);
-			}
 
-			/* Drop the pkt */
-			p_nat->invalid_packets |= pkt_mask;
-			p_nat->naptDroppedPktCount++;
-
-			#ifdef CGNAPT_DEBUGGING
-			p_nat->naptDroppedPktCount4++;
-			#endif
-			continue;
+				/* Drop the pkt */
+				p_nat->invalid_packets |= pkt_mask;
+				p_nat->naptDroppedPktCount++;
 
-		}
-			#ifdef CGNAPT_DBG_PRNT
-			if (CGNAPT_DEBUG > 2) {
-				printf("MAC found for ip 0x%x, port %d - "
-				"%02x:%02x:%02x:%02x:%02x:%02x\n",
-					dest_address,
-					*outport_id,
-					hw_addr.addr_bytes[0],
-					hw_addr.addr_bytes[1],
-					hw_addr.addr_bytes[2],
-					hw_addr.addr_bytes[3],
-					hw_addr.addr_bytes[4],
-					hw_addr.addr_bytes[5]
-					);
+				#ifdef CGNAPT_DEBUGGING
+				p_nat->naptDroppedPktCount4++;
+				#endif
+				continue;
 
-				printf("Dest MAC before - "
-				"%02x:%02x:%02x:%02x:%02x:%02x\n",
-				eth_dest[0], eth_dest[1], eth_dest[2],
-				eth_dest[3], eth_dest[4], eth_dest[5]);
 			}
-			#endif
-
-			memcpy(eth_dest, &hw_addr, sizeof(struct ether_addr));
 
-			link_hw_laddr_valid[dest_if] = 1;
-			memcpy(&link_hw_laddr[dest_if], &hw_addr,
-					 sizeof(struct ether_addr));
+			if (ret_arp_data->status == INCOMPLETE || 
+				ret_arp_data->status == PROBE) {
+				if (ret_arp_data->num_pkts >= NUM_DESC) {
+					/* Drop the pkt */
+					p_nat->invalid_packets |= pkt_mask;
+					p_nat->naptDroppedPktCount++;
 
-			#ifdef CGNAPT_DBG_PRNT
-			if (CGNAPT_DEBUG > 2) {
-				printf("Dest MAC after - "
-				"%02x:%02x:%02x:%02x:%02x:%02x\n",
-				eth_dest[0], eth_dest[1], eth_dest[2],
-				eth_dest[3], eth_dest[4], eth_dest[5]);
+					#ifdef CGNAPT_DEBUGGING
+					p_nat->naptDroppedPktCount4++;
+					#endif
+					continue;
+				} else {
+					arp_pkts_mask |= pkt_mask;
+					arp_queue_unresolved_packet(ret_arp_data, pkt);
+					continue;
+				}
 			}
-			#endif
-
-			memcpy(eth_src,
-					 get_link_hw_addr(dest_if),
-					 sizeof(struct ether_addr));
 		}
 
 		{
@@ -5050,7 +5019,7 @@ pkt4_work_cgnapt_ipv4_pub(
 				#ifdef CGNAPT_DEBUGGING
 				p_nat->naptDroppedPktCount3++;
 				#endif
-
+				printf("causing p_nat->naptDroppedPktCount3\n");
 				continue;
 			}
 
@@ -5114,54 +5083,30 @@ pkt4_work_cgnapt_ipv4_pub(
 				#endif
 				continue;
 			}
-
-			dest_address = entry->data.u.prv_ip;
-	ret = local_get_nh_ipv4(dest_address, &dest_if, &nhip, p_nat);
-	if (!ret) {
-		dest_if = get_prv_to_pub_port(&dest_address, IP_VERSION_4);
-
-		if (dest_if == INVALID_DESTIF) {
-			p_nat->invalid_packets |= pkt_mask;
-			p_nat->naptDroppedPktCount++;
-			#ifdef CGNAPT_DEBUGGING
-			p_nat->naptDroppedPktCount6++;
-			#endif
-			continue;
 		}
+		dest_address = entry->data.u.prv_ip;
+		struct arp_entry_data *ret_arp_data = NULL;
+		ret_arp_data = get_dest_mac_addr_port(dest_address, &dest_if, (struct ether_addr *)eth_dest);
+		*outport_id = p_nat->outport_id[dest_if];
 
-		do_local_nh_ipv4_cache(dest_if, p_nat);
-	}
-
-			*outport_id = p_nat->outport_id[dest_if];
-
-			#ifdef CGNAPT_DBG_PRNT
-			if (CGNAPT_DEBUG > 2)
-				printf("Ingress: \tphy_port:%d\t "
-				"get_pub_to_prv():%d \tout_port%d\n",
-					 pkt->port, dest_if,
-					 *outport_id);
-			#endif
+	if (arp_cache_dest_mac_present(dest_if)) {
+		ether_addr_copy(get_link_hw_addr(dest_if), (struct ether_addr *)eth_src);
+		arp_data_ptr[dest_if]->n_last_update = time(NULL);
+		
+		if (ret_arp_data && ret_arp_data->num_pkts) {
+			printf("sending buffered packets\n");
+			p_nat->naptedPktCount += ret_arp_data->num_pkts;
+			arp_send_buffered_pkts(ret_arp_data,
+				 (struct ether_addr *)eth_dest, *outport_id);
 		}
 
-		if (local_dest_mac_present(dest_if)) {
-			memcpy(eth_dest,
-					 get_local_link_hw_addr(dest_if),
-					 sizeof(struct ether_addr));
-			memcpy(eth_src,
-					 get_link_hw_addr(dest_if),
-					 sizeof(struct ether_addr));
-		} else {
-		int ret;
-		ret = get_dest_mac_addr_port(dest_address, &dest_if, &hw_addr);
+	} else {
 
-		if (unlikely(ret != ARP_FOUND)) {
+		if (unlikely(ret_arp_data == NULL)) {
 
-			if (unlikely(ret == ARP_NOT_FOUND)) {
-				printf("%s: ARP Not Found, nhip: %x, "
-				"outport_id: %d\n", __func__, nhip,
-				*outport_id);
-				//request_arp(*outport_id, nhip, p_nat->p.p);
-			}
+			printf("%s: NHIP Not Found, nhip: %x, "
+			"outport_id: %d\n", __func__, nhip,
+			*outport_id);
 
 			/* Drop the pkt */
 			p_nat->invalid_packets |= pkt_mask;
@@ -5171,47 +5116,26 @@ pkt4_work_cgnapt_ipv4_pub(
 			p_nat->naptDroppedPktCount4++;
 			#endif
 			continue;
-
 		}
-			#ifdef CGNAPT_DBG_PRNT
-			if (CGNAPT_DEBUG > 2) {
-				printf("MAC found for ip 0x%x, port %d - "
-				"%02x:%02x:%02x:%02x:%02x:%02x\n",
-				dest_address, *outport_id,
-				hw_addr.addr_bytes[0],
-				hw_addr.addr_bytes[1],
-				hw_addr.addr_bytes[2],
-				hw_addr.addr_bytes[3],
-				hw_addr.addr_bytes[4],
-				hw_addr.addr_bytes[5]
-				);
-
-				printf("Dest MAC before - "
-				"%02x:%02x:%02x:%02x:%02x:%02x\n",
-				eth_dest[0], eth_dest[1], eth_dest[2],
-				eth_dest[3], eth_dest[4], eth_dest[5]);
-			}
-			#endif
-
-			memcpy(eth_dest, &hw_addr, sizeof(struct ether_addr));
 
-			link_hw_laddr_valid[dest_if] = 1;
-			memcpy(&link_hw_laddr[dest_if],
-					 &hw_addr, sizeof(struct ether_addr));
+		if (ret_arp_data->status == INCOMPLETE || 
+			ret_arp_data->status == PROBE) {
+			if (ret_arp_data->num_pkts >= NUM_DESC) {
+				/* Drop the pkt */
+				p_nat->invalid_packets |= pkt_mask;
+				p_nat->naptDroppedPktCount++;
 
-			#ifdef CGNAPT_DBG_PRNT
-			if (CGNAPT_DEBUG > 2) {
-				printf("Dest MAC after - %02x:%02x:%02x: "
-				"%02x:%02x:%02x\n",
-				eth_dest[0], eth_dest[1], eth_dest[2],
-				eth_dest[3], eth_dest[4], eth_dest[5]);
+				#ifdef CGNAPT_DEBUGGING
+				p_nat->naptDroppedPktCount4++;
+				#endif
+				continue;
+			} else {
+				arp_pkts_mask |= pkt_mask;
+				arp_queue_unresolved_packet(ret_arp_data, pkt);
+				continue;
 			}
-			#endif
-
-			memcpy(eth_src,
-					 get_link_hw_addr(dest_if),
-					 sizeof(struct ether_addr));
 		}
+	}
 
 		{
 			/* Ingress */
@@ -6159,7 +6083,6 @@ pkt_work_cgnapt_ipv6_prv(
 	__rte_unused void *arg,
 	struct pipeline_cgnapt *p_nat)
 {
-
 	/* index into hash table entries */
 	int hash_table_entry = p_nat->lkup_indx[pkt_num];
 
@@ -6351,23 +6274,17 @@ pkt_work_cgnapt_ipv6_prv(
 	#endif
 
 	if (local_dest_mac_present(dest_if)) {
-		memcpy(eth_dest,
-				get_local_link_hw_addr(dest_if),
-				sizeof(struct ether_addr));
 		memcpy(eth_src, get_link_hw_addr(dest_if),
 				sizeof(struct ether_addr));
 	} else {
-		int ret;
-		ret = get_dest_mac_addr_port(dest_address, &dest_if, &hw_addr);
+		struct arp_entry_data *ret_arp_data;
+		ret_arp_data = get_dest_mac_addr_port(dest_address, &dest_if, (struct ether_addr *)&hw_addr);
 
-		if (unlikely(ret != ARP_FOUND)) {
+		if (unlikely(ret_arp_data == NULL)) {
 
-			if (unlikely(ret == ARP_NOT_FOUND)) {
-				printf("%s: ARP Not Found, nhip: %x, "
-				"outport_id: %d\n", __func__, nhip,
-				*outport_id);
-				//request_arp(*outport_id, nhip, p_nat->p.p);
-			}
+			printf("%s: NHIP Not Found, nhip: %x, "
+			"outport_id: %d\n", __func__, nhip,
+			*outport_id);
 
 			/* Drop the pkt */
 			p_nat->invalid_packets |= pkt_mask;
@@ -6377,10 +6294,12 @@ pkt_work_cgnapt_ipv6_prv(
 			p_nat->naptDroppedPktCount4++;
 			#endif
 			return;
-
 		}
-		#ifdef CGNAPT_DBG_PRNT
-		if (CGNAPT_DEBUG > 2) {
+
+		if (ret_arp_data->status == COMPLETE) {
+
+			#ifdef CGNAPT_DBG_PRNT
+			if (CGNAPT_DEBUG > 2) {
 			printf("MAC found for ip 0x%x, port %d - %02x:%02x: "
 			"%02x:%02x:%02x:%02x\n", dest_address,
 			*outport_id,
@@ -6392,22 +6311,28 @@ pkt_work_cgnapt_ipv6_prv(
 			"%02x:%02x\n", eth_dest[0], eth_dest[1],
 			eth_dest[2], eth_dest[3],
 			eth_dest[4], eth_dest[5]);
-		}
-		#endif
+			}
+			#endif
 
-		memcpy(eth_dest, &hw_addr, sizeof(struct ether_addr));
+			//memcpy(eth_dest, &hw_addr, sizeof(struct ether_addr));
 
-		#ifdef CGNAPT_DBG_PRNT
-		if (CGNAPT_DEBUG > 2) {
-			printf("Dest MAC after - "
-			"%02x:%02x:%02x:%02x:%02x:%02x\n",
-			eth_dest[0], eth_dest[1], eth_dest[2], eth_dest[3],
-			eth_dest[4], eth_dest[5]);
-		}
-		#endif
+			#ifdef CGNAPT_DBG_PRNT
+			if (CGNAPT_DEBUG > 2) {
+				printf("Dest MAC after - "
+				"%02x:%02x:%02x:%02x:%02x:%02x\n",
+				eth_dest[0], eth_dest[1], eth_dest[2], eth_dest[3],
+				eth_dest[4], eth_dest[5]);
+			}
+			#endif
 
-		memcpy(eth_src, get_link_hw_addr(dest_if),
+			memcpy(eth_src, get_link_hw_addr(dest_if),
 				 sizeof(struct ether_addr));
+		} else if (ret_arp_data->status == INCOMPLETE || 
+			ret_arp_data->status == PROBE) {
+			arp_queue_unresolved_packet(ret_arp_data,
+				pkt);
+			return;
+		}
 	}
 
 	{
@@ -6874,20 +6799,15 @@ pkt4_work_cgnapt_ipv6_prv(
 
 		memset(nh_ipv6, 0, 16);
 
-		{
-		int ret;
-		ret = get_dest_mac_addr_port(dest_address, &dest_if, &hw_addr);
+	{
+		struct arp_entry_data *ret_arp_data;
+		ret_arp_data = get_dest_mac_addr_port(dest_address, &dest_if, (struct ether_addr *)&hw_addr);
 
-		if (unlikely(ret != ARP_FOUND)) {
+		if (unlikely(ret_arp_data == NULL)) {
 
-			if (unlikely(ret == ARP_NOT_FOUND)) {
-				/* Commented code may be required for debug
-				 * and future use, Please keep it */
-				//request_arp(*outport_id, nhip, p_nat->p.p);
-				printf("%s: ARP Not Found, nhip: %x, "
-				"outport_id: %d\n", __func__, nhip,
-				*outport_id);
-			}
+			printf("%s: NHIP Not Found, nhip: %x, "
+			"outport_id: %d\n", __func__, nhip,
+			*outport_id);
 
 			/* Drop the pkt */
 			p_nat->invalid_packets |= pkt_mask;
@@ -6897,9 +6817,10 @@ pkt4_work_cgnapt_ipv6_prv(
 			p_nat->naptDroppedPktCount4++;
 			#endif
 			continue;
-
 		}
 
+		if (ret_arp_data->status == COMPLETE) {
+
 			#ifdef CGNAPT_DBG_PRNT
 			if (CGNAPT_DEBUG > 2) {
 				printf("MAC found for ip 0x%x, port %d - "
@@ -6920,7 +6841,7 @@ pkt4_work_cgnapt_ipv6_prv(
 			}
 			#endif
 
-			memcpy(eth_dest, &hw_addr, sizeof(struct ether_addr));
+			//memcpy(eth_dest, &hw_addr, sizeof(struct ether_addr));
 
 			#ifdef CGNAPT_DBG_PRNT
 			if (CGNAPT_DEBUG > 2) {
@@ -6931,10 +6852,15 @@ pkt4_work_cgnapt_ipv6_prv(
 			}
 			#endif
 
-			memcpy(eth_src,
-					 get_link_hw_addr(dest_if),
+			memcpy(eth_src, get_link_hw_addr(dest_if),
 					 sizeof(struct ether_addr));
+		} else if (ret_arp_data->status == INCOMPLETE || 
+			ret_arp_data->status == PROBE) {
+			arp_queue_unresolved_packet(ret_arp_data,
+				pkt);
+			continue;
 		}
+	}
 
 		{
 			/* Egress */
@@ -7050,8 +6976,7 @@ pkt4_work_cgnapt_ipv6_pub(
 		uint8_t dest_addr_ipv6[16];
 		uint8_t nh_ipv6[16];
 		uint32_t dest_if = INVALID_DESTIF;
-		/* Ingress */
-		{
+		{ /*start of Ingress */
 
 			if (unlikely(protocol == IP_PROTOCOL_UDP
 				&& rte_be_to_cpu_16(*src_port) == 53)) {
@@ -7086,7 +7011,7 @@ pkt4_work_cgnapt_ipv6_pub(
 			}
 
 			*outport_id = p_nat->outport_id[dest_if];
-		}
+		}/* end of ingress */
 
 		#ifdef CGNAPT_DEBUGGING
 		static int static_count;
@@ -7147,7 +7072,7 @@ pkt4_work_cgnapt_ipv6_pub(
 		}
 
 		{
-			/* Ingress */
+		/* start of Ingress */
 
 			convert_ipv4_to_ipv6(pkt, &ipv4_hdr);
 
@@ -7175,7 +7100,7 @@ pkt4_work_cgnapt_ipv6_pub(
 			#endif
 
 			p_nat->inaptedPktCount++;
-		}
+		} /* end of ingress */
 
 		p_nat->naptedPktCount++;
 
@@ -7185,7 +7110,7 @@ pkt4_work_cgnapt_ipv6_pub(
 			else
 				sw_checksum(pkt, pkt_type);
 		#endif
-	}
+	} /* end of for loop */
 }
 
 /**
@@ -7891,12 +7816,11 @@ pkt_miss_cgnapt(struct pipeline_cgnapt_entry_key *key,
 					printf("Add Dynamic NAT entry failed "
 					"in pkt!!!\n");
 				#endif
-		} else {
+			} else {
 				#ifdef CGNAPT_DEBUGGING
 				p_nat->missedpktcount11++;
 				#endif
-		}
-
+			}
 		}
 
 	} else if (!is_phy_port_privte(phy_port)) {
@@ -8389,7 +8313,6 @@ pipeline_cgnapt_parse_args(struct pipeline_cgnapt *p,
 	return 0;
 
 }
-
 /**
  * Function to initialize the pipeline
  *
@@ -8417,6 +8340,7 @@ static void *pipeline_cgnapt_init(struct pipeline_params *params, void *arg)
 	size = RTE_CACHE_LINE_ROUNDUP(sizeof(struct pipeline_cgnapt));
 	p = rte_zmalloc(NULL, size, RTE_CACHE_LINE_SIZE);
 	p_nat = (struct pipeline_cgnapt *)p;
+	global_pnat = p_nat;
 	if (p == NULL)
 		return NULL;
 
@@ -9260,10 +9184,10 @@ pipeline_cgnapt_msg_req_entry_addm_pair(
 	}
 	#endif
 
-	if (CGNAPT_DEBUG > 2)
-		printf("key.ip %x, key.port %d", key.ip, key.port);
-		printf("key.pid %d, in_type %d,", key.pid, type);
-		printf("entry_type %d\n", entry.data.type);
+	//if (CGNAPT_DEBUG > 2)
+		//printf("key.ip %x, key.port %d", key.ip, key.port);
+		//printf("key.pid %d, in_type %d,", key.pid, type);
+		//printf("entry_type %d\n", entry.data.type);
 
 	int32_t position = rte_hash_add_key(napt_common_table, &key);
 
diff --git a/VNFs/vCGNAPT/pipeline/pipeline_cgnapt_be.h b/VNFs/vCGNAPT/pipeline/pipeline_cgnapt_be.h
index 34031192..a5c83bf1 100644
--- a/VNFs/vCGNAPT/pipeline/pipeline_cgnapt_be.h
+++ b/VNFs/vCGNAPT/pipeline/pipeline_cgnapt_be.h
@@ -780,7 +780,6 @@ void calculate_hw_checksum(
 	uint8_t protocol);
 
 uint64_t nextPowerOf2(uint64_t n);
-struct ether_addr *get_local_link_hw_addr(uint8_t out_port);
 uint8_t local_dest_mac_present(uint8_t out_port);
 
 enum PKT_TYPE {
diff --git a/VNFs/vFW/init.c b/VNFs/vFW/init.c
index 979e1050..f5d457a4 100644
--- a/VNFs/vFW/init.c
+++ b/VNFs/vFW/init.c
@@ -1368,12 +1368,13 @@ int app_init(struct app_params *app)
 	app_init_pipelines(app);
 	app_init_threads(app);
 
+	#ifdef L3_STACK_SUPPORT
 	l3fwd_init();
 	create_arp_table();
 	create_nd_table();
 	populate_lpm_routes();
 	print_interface_details();
-
+	#endif
 	return 0;
 }
 
diff --git a/VNFs/vFW/pipeline/pipeline_vfw_be.c b/VNFs/vFW/pipeline/pipeline_vfw_be.c
index 97508a77..a654c3fc 100644
--- a/VNFs/vFW/pipeline/pipeline_vfw_be.c
+++ b/VNFs/vFW/pipeline/pipeline_vfw_be.c
@@ -177,6 +177,7 @@ __rte_cache_aligned;
   * to avoid useless table lookup
   */
 /***** ARP local cache *****/
+#if 0
 uint8_t link_hw_laddr_valid[MAX_NUM_LOCAL_MAC_ADDRESS] = {
        0, 0, 0, 0, 0, 0, 0, 0,
        0, 0, 0, 0, 0, 0, 0, 0
@@ -200,6 +201,8 @@ static struct ether_addr link_hw_laddr[MAX_NUM_LOCAL_MAC_ADDRESS] = {
        {.addr_bytes = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00} },
        {.addr_bytes = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00} }
 };
+#endif
+uint64_t arp_pkts_mask;
 
 /* Start TSC measurement */
 /* Prefetch counters and pipe before this function */
@@ -228,11 +231,11 @@ static inline void end_tsc_measure(
        }
 }
 
-static struct ether_addr *get_local_link_hw_addr(uint8_t out_port)
-{
-       return &link_hw_laddr[out_port];
-}
-
+//static struct ether_addr *get_local_link_hw_addr(uint8_t out_port)
+//{
+//       return &link_hw_laddr[out_port];
+//}
+#if 0
 static uint8_t local_dest_mac_present(uint8_t out_port)
 {
        return link_hw_laddr_valid[out_port];
@@ -284,6 +287,7 @@ static void do_local_nh_ipv4_cache(uint32_t dest_if,
               }
        }
 }
+#endif
 static uint32_t local_get_nh_ipv6(
        uint8_t *ip,
        uint32_t *port,
@@ -895,15 +899,11 @@ pkt4_work_vfw_arp_ipv4_packets(struct rte_mbuf **pkts,
               struct pipeline_vfw *vfw_pipe)
 {
 
-       uint32_t ret;
-       int ret_mac;
        uint8_t i;
 
-       struct ether_addr hw_addr;
        struct mbuf_tcp_meta_data *meta_data_addr;
        struct ether_hdr *ehdr;
        struct rte_mbuf *pkt;
-       uint16_t phy_port;
 
        for (i = 0; i < 4; i++) {
               uint32_t dest_if = INVALID_DESTIF;
@@ -917,7 +917,6 @@ pkt4_work_vfw_arp_ipv4_packets(struct rte_mbuf **pkts,
 
               int must_reverse = ((synproxy_reply_mask & pkt_mask) != 0);
 
-              phy_port = pkt->port;
               meta_data_addr = (struct mbuf_tcp_meta_data *)
                      RTE_MBUF_METADATA_UINT32_PTR(pkt, META_DATA_OFFSET);
               ehdr = rte_vfw_get_ether_addr(pkt);
@@ -928,7 +927,9 @@ pkt4_work_vfw_arp_ipv4_packets(struct rte_mbuf **pkts,
               uint32_t nhip = 0;
 
               uint32_t dest_address = rte_bswap32(ihdr->dst_addr);
-
+              if (must_reverse)
+                     rte_sp_exchange_mac_addresses(ehdr);
+#if 0
               ret = local_get_nh_ipv4(dest_address, &dest_if,
                             &nhip, vfw_pipe);
               if (must_reverse) {
@@ -993,55 +994,52 @@ pkt4_work_vfw_arp_ipv4_packets(struct rte_mbuf **pkts,
                      ether_addr_copy(get_link_hw_addr(dest_if),
                                    &ehdr->s_addr);
               } else {
-                     ret_mac = get_dest_mac_addr_port(dest_address,
-                                   &dest_if, &hw_addr);
-                     if (ret_mac == ARP_FOUND) {
+#endif
+	struct arp_entry_data *ret_arp_data = NULL;
+        ret_arp_data = get_dest_mac_addr_port(dest_address,
+                       &dest_if, &ehdr->d_addr);
+        meta_data_addr->output_port =  vfw_pipe->outport_id[dest_if];
 
-                            link_hw_laddr_valid[dest_if] = 1;
-                            memcpy(&link_hw_laddr[dest_if], &hw_addr,
-                                          sizeof(struct ether_addr));
+        if (arp_cache_dest_mac_present(dest_if)) {
 
-                            ether_addr_copy(&hw_addr, &ehdr->d_addr);
-                            ether_addr_copy(get_link_hw_addr(dest_if),
-                                          &ehdr->s_addr);
+                ether_addr_copy(get_link_hw_addr(dest_if), &ehdr->s_addr);
+                arp_data_ptr[dest_if]->n_last_update = time(NULL);
 
-                            if (vfw_debug >= DEBUG_LEVEL_4) {
-                                   char buf[HW_ADDR_SIZE];
+                if (unlikely(ret_arp_data && ret_arp_data->num_pkts)) {
+
+                        printf("sending buffered packets\n");
+                        arp_send_buffered_pkts(ret_arp_data,
+                                 &ehdr->d_addr, vfw_pipe->outport_id[dest_if]);
 
-                                   ether_format_addr(buf, sizeof(buf),
-                                                 &hw_addr);
-                                   printf("MAC found for ip 0x%"
-                                          PRIx32", dest_if %d: %s, ",
-                                          dest_address,
-                                          dest_if, buf);
-                                   ether_format_addr(buf, sizeof(buf),
-                                                 &ehdr->s_addr);
-                                   printf("new eth hdr src: %s, ", buf);
-                                   ether_format_addr(buf, sizeof(buf),
-                                                 &ehdr->d_addr);
-                                   printf("new eth hdr dst: %s\n", buf);
                             }
 
                      } else {
+                if (unlikely(ret_arp_data == NULL)) {
 
-                            if (vfw_debug >= DEBUG_LEVEL_4) {
-                                   char buf[HW_ADDR_SIZE];
+                        printf("%s: NHIP Not Found, nhip:%x , "
+                        "outport_id: %d\n", __func__, nhip,
+                        vfw_pipe->outport_id[dest_if]);
 
-                                   ether_format_addr(buf, sizeof(buf),
-                                          &hw_addr);
-                                   printf("MAC NOT FOUND for ip 0x%"
-                                          PRIx32", dest_if %"
-                                          PRId16": %s, ",
-                                          dest_address,
-                                          dest_if, buf);
+                        /* Drop the pkt */
+                        vfw_pipe->counters->
+                                 pkts_drop_without_arp_entry++;
+                        continue;
                             }
+		if (ret_arp_data->status == INCOMPLETE ||
+                           ret_arp_data->status == PROBE) {
+                                if (ret_arp_data->num_pkts >= NUM_DESC) {
                             /* ICMP req sent, drop packet by
                              * changing the mask */
-                            *pkts_mask &= ~pkt_mask;
-                            vfw_pipe->
-                                   counters->pkts_drop_without_arp_entry++;
+                        	vfw_pipe->counters->
+                                	 pkts_drop_without_arp_entry++;
+                                        continue;
+                                } else {
+                                        arp_pkts_mask |= pkt_mask;
+                                        arp_queue_unresolved_packet(ret_arp_data, pkt);
+                                        continue;
                      }
               }
+	}
        }
 }
 
@@ -1072,15 +1070,11 @@ pkt_work_vfw_arp_ipv4_packets(struct rte_mbuf *pkts,
               struct pipeline_vfw *vfw_pipe)
 {
 
-       uint32_t ret;
        uint32_t dest_if = INVALID_DESTIF;
-       int ret_mac;
 
-       struct ether_addr hw_addr;
        struct mbuf_tcp_meta_data *meta_data_addr;
        struct ether_hdr *ehdr;
        struct rte_mbuf *pkt;
-       uint16_t phy_port;
        uint64_t pkt_mask = 1LLU << pkt_num;
 
        pkt = pkts;
@@ -1089,7 +1083,6 @@ pkt_work_vfw_arp_ipv4_packets(struct rte_mbuf *pkts,
 
               int must_reverse = ((synproxy_reply_mask & pkt_mask) != 0);
 
-              phy_port = pkt->port;
               meta_data_addr = (struct mbuf_tcp_meta_data *)
                      RTE_MBUF_METADATA_UINT32_PTR(pkt, META_DATA_OFFSET);
               ehdr = rte_vfw_get_ether_addr(pkt);
@@ -1100,7 +1093,10 @@ pkt_work_vfw_arp_ipv4_packets(struct rte_mbuf *pkts,
               uint32_t nhip = 0;
 
               uint32_t dest_address = rte_bswap32(ihdr->dst_addr);
+              if (must_reverse)
+                     rte_sp_exchange_mac_addresses(ehdr);
 
+#if 0
               ret = local_get_nh_ipv4(dest_address, &dest_if,
                             &nhip, vfw_pipe);
               if (must_reverse) {
@@ -1165,53 +1161,50 @@ pkt_work_vfw_arp_ipv4_packets(struct rte_mbuf *pkts,
                      ether_addr_copy(get_link_hw_addr(dest_if),
                                    &ehdr->s_addr);
               } else {
-                     ret_mac = get_dest_mac_addr_port(dest_address,
-                                   &dest_if, &hw_addr);
-                     if (ret_mac) {
-                            link_hw_laddr_valid[dest_if] = 1;
-                            memcpy(&link_hw_laddr[dest_if], &hw_addr,
-                                          sizeof(struct ether_addr));
-
-                            ether_addr_copy(&hw_addr, &ehdr->d_addr);
-                            ether_addr_copy(get_link_hw_addr(dest_if),
-                                          &ehdr->s_addr);
+#endif
+	struct arp_entry_data *ret_arp_data = NULL;
+                     ret_arp_data = get_dest_mac_addr_port(dest_address,
+                                   &dest_if, &ehdr->d_addr);
+              		meta_data_addr->output_port =  vfw_pipe->outport_id[dest_if];
+        if (arp_cache_dest_mac_present(dest_if)) {
 
-                            if (vfw_debug >= DEBUG_LEVEL_4) {
-                                   char buf[HW_ADDR_SIZE];
+                ether_addr_copy(get_link_hw_addr(dest_if), &ehdr->s_addr);
+                arp_data_ptr[dest_if]->n_last_update = time(NULL);
+
+                if (unlikely(ret_arp_data && ret_arp_data->num_pkts)) {
+
+                        printf("sending buffered packets\n");
+                        arp_send_buffered_pkts(ret_arp_data,
+                                 &ehdr->d_addr, vfw_pipe->outport_id[dest_if]);
 
-                                   ether_format_addr(buf, sizeof(buf),
-                                                 &hw_addr);
-                                   printf("MAC found for ip 0x%"
-                                          PRIx32", dest_if %d: %s, ",
-                                          dest_address,
-                                          dest_if, buf);
-                                   ether_format_addr(buf, sizeof(buf),
-                                                 &ehdr->s_addr);
-                                   printf("new eth hdr src: %s, ", buf);
-                                   ether_format_addr(buf, sizeof(buf),
-                                                 &ehdr->d_addr);
-                                   printf("new eth hdr dst: %s\n", buf);
                             }
 
                      } else {
-                            if (vfw_debug >= DEBUG_LEVEL_4) {
-                                   char buf[HW_ADDR_SIZE];
-
-                                   ether_format_addr(buf, sizeof(buf),
-                                                 &hw_addr);
-                                   printf("MAC NOT FOUND for ip 0x%"
-                                                 PRIx32", dest_if %"
-                                                 PRId16": %s, ",
-                                                 dest_address,
-                                                 dest_if, buf);
+                if (unlikely(ret_arp_data == NULL)) {
+
+                        printf("%s: NHIP Not Found, nhip:%x , "
+                        "outport_id: %d\n", __func__, nhip,
+                        vfw_pipe->outport_id[dest_if]);
+
+                        vfw_pipe->counters->
+                                pkts_drop_without_arp_entry++;
+                        return;
                             }
+		if (ret_arp_data->status == INCOMPLETE ||
+                           ret_arp_data->status == PROBE) {
+                                if (ret_arp_data->num_pkts >= NUM_DESC) {
                             /* ICMP req sent, drop packet by
                              * changing the mask */
-                            *pkts_mask &= ~pkt_mask;
-                            vfw_pipe->
-                                   counters->pkts_drop_without_arp_entry++;
+                        		vfw_pipe->counters->
+                                		pkts_drop_without_arp_entry++;
+                                        return;
+                                } else {
+                                        arp_pkts_mask |= pkt_mask;
+                                        arp_queue_unresolved_packet(ret_arp_data, pkt);
+                                        return;
                      }
               }
+	}
 
        }
 }
@@ -1567,7 +1560,6 @@ rte_vfw_arp_ipv4_packets(struct rte_mbuf **pkts,
 
        uint32_t ret;
        uint32_t dest_if = INVALID_DESTIF;
-       int ret_mac;
        for (; pkts_to_arp;) {
               struct ether_addr hw_addr;
               struct mbuf_tcp_meta_data *meta_data_addr;
@@ -1594,7 +1586,9 @@ rte_vfw_arp_ipv4_packets(struct rte_mbuf **pkts,
               uint32_t nhip = 0;
 
               uint32_t dest_address = rte_bswap32(ihdr->dst_addr);
-
+              if (must_reverse)
+                     rte_sp_exchange_mac_addresses(ehdr);
+#if 0
               ret = local_get_nh_ipv4(dest_address, &dest_if,
                             &nhip, vfw_pipe);
               if (must_reverse) {
@@ -1659,57 +1653,52 @@ rte_vfw_arp_ipv4_packets(struct rte_mbuf **pkts,
                      ether_addr_copy(get_link_hw_addr(dest_if),
                                    &ehdr->s_addr);
               } else {
-                     ret_mac = get_dest_mac_addr_port(dest_address,
-                                   &dest_if, &hw_addr);
-                     if (ret_mac) {
-                            link_hw_laddr_valid[dest_if] = 1;
-                            memcpy(&link_hw_laddr[dest_if], &hw_addr,
-                                          sizeof(struct ether_addr));
-
-                            ether_addr_copy(&hw_addr, &ehdr->d_addr);
-                            ether_addr_copy(get_link_hw_addr(dest_if),
-                                          &ehdr->s_addr);
+#endif
+		struct arp_entry_data *ret_arp_data = NULL;
+                     ret_arp_data = get_dest_mac_addr_port(dest_address,
+                                   &dest_if, &ehdr->d_addr);
+              	meta_data_addr->output_port =  vfw_pipe->outport_id[dest_if];
+        if (arp_cache_dest_mac_present(dest_if)) {
 
-                            if (vfw_debug >= DEBUG_LEVEL_4) {
-                                   char buf[HW_ADDR_SIZE];
+                ether_addr_copy(get_link_hw_addr(dest_if), &ehdr->s_addr);
+                arp_data_ptr[dest_if]->n_last_update = time(NULL);
+
+                if (unlikely(ret_arp_data && ret_arp_data->num_pkts)) {
+
+                        printf("sending buffered packets\n");
+                        p_nat->naptedPktCount += ret_arp_data->num_pkts;
+                        arp_send_buffered_pkts(ret_arp_data,
+                                 &ehdr->d_addr, vfw_pipe->outport_id[dest_if]);
 
-                                   ether_format_addr(buf, sizeof(buf),
-                                                 &hw_addr);
-                                   printf("MAC found for ip 0x%"
-                                          PRIx32", dest_if %d: %s, ",
-                                          dest_address,
-                                          dest_if, buf);
-                                   ether_format_addr(buf, sizeof(buf),
-                                                 &ehdr->s_addr);
-                                   printf("new eth hdr src: %s, ", buf);
-                                   ether_format_addr(buf, sizeof(buf),
-                                                 &ehdr->d_addr);
-                                   printf("new eth hdr dst: %s\n", buf);
                             }
 
                      } else {
-                            if (unlikely(ret_mac == 0))
-                                   request_arp(meta_data_addr->output_port,
-                                          nhip);
+                if (unlikely(ret_arp_data == NULL)) {
 
-                            if (vfw_debug >= DEBUG_LEVEL_4) {
-                                   char buf[HW_ADDR_SIZE];
+                        printf("%s: NHIP Not Found, nhip:%x , "
+                        "outport_id: %d\n", __func__, nhip,
+                        vfw_pipe->outport_id[dest_if]);
 
-                            ether_format_addr(buf, sizeof(buf),
-                                          &hw_addr);
-                            printf("MAC NOT FOUND for ip 0x%"
-                                          PRIx32", dest_if %"
-                                          PRId16": %s, ",
-                                          dest_address,
-                                          dest_if, buf);
+                        /* Drop the pkt */
+                                   vfw_pipe->counters->
+                                          pkts_drop_without_arp_entry++;
+                        continue;
                      }
+		if (ret_arp_data->status == INCOMPLETE ||
+                           ret_arp_data->status == PROBE) {
+                                if (ret_arp_data->num_pkts >= NUM_DESC) {
                      /* ICMP req sent, drop packet by
                       * changing the mask */
-                     pkts_mask &= ~pkt_mask;
-                     vfw_pipe->
-                            counters->pkts_drop_without_arp_entry++;
+                                   	vfw_pipe->counters->
+                                          pkts_drop_without_arp_entry++;
+                                        continue;
+                                } else {
+                                        arp_pkts_mask |= pkt_mask;
+                                        arp_queue_unresolved_packet(ret_arp_data, pkt);
+                                        continue;
               }
 }
+	}
 
        }
 
@@ -2070,6 +2059,7 @@ vfw_port_in_action_ipv4(struct rte_pipeline *p,
        uint64_t packet_mask_in = RTE_LEN2MASK(n_pkts, uint64_t);
        uint64_t pkts_drop_mask;
        uint64_t hijack_mask = 0;
+       arp_pkts_mask = 0;
        uint64_t synproxy_reply_mask = 0;       /* for synproxy */
        uint64_t keep_mask = packet_mask_in;
 
@@ -2204,6 +2194,9 @@ vfw_port_in_action_ipv4(struct rte_pipeline *p,
        }
 
        /* Update mask before returning, so that bad packets are dropped */
+        if (arp_pkts_mask) {
+                rte_pipeline_ah_packet_hijack(p, arp_pkts_mask);
+        }
 
        pkts_drop_mask = packet_mask_in & ~keep_mask;