Improve performance in l3 submode

Two cases where performance has been improved
- When using a gateway from a routing table (l3 submode),
  store the mac within the next hop table, to avoid a hash_lookup.
  This gives ~10% improvement.
- Read tsc only once per bulk (of up to 64 packets).
  This gives ~10% improvement to swap, 4% to gen.

In addition a small fix has been added, preventig "No route" Error
to be written too aften.

Change-Id: I8a7ab74a32f09c8ff47f751ee91e84afee1b2147
Signed-off-by: Xavier Simonart <xavier.simonart@intel.com>

3 files changed, 17 insertions, 13 deletions

diff --git a/VNFs/DPPD-PROX/packet_utils.c b/VNFs/DPPD-PROX/packet_utils.c
index 04746130..155d8c61 100644
--- a/VNFs/DPPD-PROX/packet_utils.c
+++ b/VNFs/DPPD-PROX/packet_utils.c
@@ -120,14 +120,14 @@ static inline int update_mac_and_send_mbuf(struct arp_table *entry, prox_rte_eth
 	return DROP_MBUF;
 }
 
-int write_dst_mac(struct task_base *tbase, struct rte_mbuf *mbuf, uint32_t *ip_dst, uint64_t **time)
+int write_dst_mac(struct task_base *tbase, struct rte_mbuf *mbuf, uint32_t *ip_dst, uint64_t **time, uint64_t tsc)
 {
 	const uint64_t hz = rte_get_tsc_hz();
 	struct ether_hdr_arp *packet = rte_pktmbuf_mtod(mbuf, struct ether_hdr_arp *);
 	prox_rte_ether_addr *mac = &packet->ether_hdr.d_addr;
 	prox_next_hop_index_type next_hop_index;
+	static uint64_t last_tsc = 0, n_no_route = 0;
 
-	uint64_t tsc = rte_rdtsc();
 	struct l3_base *l3 = &(tbase->l3);
 
 	// First find the next hop
@@ -141,27 +141,30 @@ int write_dst_mac(struct task_base *tbase, struct rte_mbuf *mbuf, uint32_t *ip_d
 			return SEND_MBUF;
 		}
 		if (unlikely(rte_lpm_lookup(l3->ipv4_lpm, rte_bswap32(*ip_dst), &next_hop_index) != 0)) {
-			plog_err("No route to IP "IPv4_BYTES_FMT"\n", IP4(*ip_dst));
+			// Prevent printing too many messages
+			n_no_route++;
+			if (tsc > last_tsc + rte_get_tsc_hz()) {
+				plog_err("No route to IP "IPv4_BYTES_FMT" (%ld times)\n", IP4(*ip_dst), n_no_route);
+				last_tsc = tsc;
+				n_no_route = 0;
+			}
 			return DROP_MBUF;
 		}
 		struct arp_table *entry = &l3->next_hops[next_hop_index];
 
 		if (entry->ip) {
 			*ip_dst = entry->ip;
-		} else {
-			// no next ip: this is a local route
-			next_hop_index = MAX_HOP_INDEX;
+			return update_mac_and_send_mbuf(entry, mac, tsc, hz, l3->arp_update_time, time);
 		}
+
+		// no next ip: this is a local route
 		// Find IP in lookup table. Send ARP if not found
 		int ret = rte_hash_lookup(l3->ip_hash, (const void *)ip_dst);
 		if (unlikely(ret < 0)) {
 			// IP not found, try to send an ARP
-			return add_key_and_send_arp(l3->ip_hash, ip_dst, l3->arp_table, tsc, hz, l3->arp_update_time, next_hop_index, time);
+			return add_key_and_send_arp(l3->ip_hash, ip_dst, l3->arp_table, tsc, hz, l3->arp_update_time, MAX_HOP_INDEX, time);
 		} else {
-			if (entry->ip)
-				return update_mac_and_send_mbuf(entry, mac, tsc, hz, l3->arp_update_time, time);
-			else
-				return update_mac_and_send_mbuf(&l3->arp_table[ret], mac, tsc, hz, l3->arp_update_time, time);
+			return update_mac_and_send_mbuf(&l3->arp_table[ret], mac, tsc, hz, l3->arp_update_time, time);
 		}
 		return 0;
 	}
diff --git a/VNFs/DPPD-PROX/packet_utils.h b/VNFs/DPPD-PROX/packet_utils.h
index 021528de..0a1ef9d8 100644
--- a/VNFs/DPPD-PROX/packet_utils.h
+++ b/VNFs/DPPD-PROX/packet_utils.h
@@ -69,7 +69,7 @@ struct l3_base {
 
 void task_init_l3(struct task_base *tbase, struct task_args *targ);
 void task_start_l3(struct task_base *tbase, struct task_args *targ);
-int write_dst_mac(struct task_base *tbase, struct rte_mbuf *mbuf, uint32_t *ip_dst, uint64_t **time);
+int write_dst_mac(struct task_base *tbase, struct rte_mbuf *mbuf, uint32_t *ip_dst, uint64_t **time, uint64_t tsc);
 void task_set_gateway_ip(struct task_base *tbase, uint32_t ip);
 void task_set_local_ip(struct task_base *tbase, uint32_t ip);
 void handle_ctrl_plane_pkts(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts);
diff --git a/VNFs/DPPD-PROX/tx_pkt.c b/VNFs/DPPD-PROX/tx_pkt.c
index 2a4f53b2..51c1afa7 100644
--- a/VNFs/DPPD-PROX/tx_pkt.c
+++ b/VNFs/DPPD-PROX/tx_pkt.c
@@ -57,11 +57,12 @@ int tx_pkt_l3(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts,
 	const struct port_queue *port_queue = &tbase->tx_params_hw.tx_port_queue[0];
 	struct rte_mbuf *arp_mbuf = NULL;       // used when one need to send both an ARP and a mbuf
 	uint64_t *time;
+	uint64_t tsc = rte_rdtsc();
 
 	for (int j = 0; j < n_pkts; j++) {
 		if ((out) && (out[j] >= OUT_HANDLED))
 			continue;
-		if (unlikely((rc = write_dst_mac(tbase, mbufs[j], &ip_dst, &time)) != SEND_MBUF)) {
+		if (unlikely((rc = write_dst_mac(tbase, mbufs[j], &ip_dst, &time, tsc)) != SEND_MBUF)) {
 			if (j - first) {
 				ret = tbase->aux->tx_pkt_l2(tbase, mbufs + first, j - first, out);
 				ok += ret;