Improve ctrlplane performance

The crlplane performance has been improved. This is necessary when
handling many IP in L3 mode (i.e. many ARP Requests/Replies).

Before this change, arp timer was updated as soon as an
arp request was tentativaly sent to the master. This means that,
if the request failed to be sent (e.g. ring full) we had to wait
arp_update_time (default 1sec) before trying again.

Now arp_update_time is only used when an arp reply is received.
If a request has been successfully sent, then the arp timer
is updated by 1 second (i.e. we will not send any arp reuest for this
IP within this second)
If we failed to send the request, then the timer is updated by 100 msec
which means we will wait 100 msec before trying again.
A too high value here would have meant that we have to wait a long time
before trying again. A too short value overload the master ring with
request to transmit ARP requests, so that master is unable to handle
arp replies.

arp_update_time is now also partly randomized to avoid all IPs to request
arp request at the beginning of the test. So if arp_update_time is 10 sec
then the configuration sets it to 5 to 15 sec.

As a final note the following parameter (already implemented before)
can be changed to improve ctrlplane performance (at the cost of dataplane)

ctrl path polling frequency=10000

Change-Id: I9660458a22e7442eaa0c83aaa0c9893a15069f98
Signed-off-by: Xavier Simonart <xavier.simonart@intel.com>

4 files changed, 36 insertions, 13 deletions

diff --git a/VNFs/DPPD-PROX/packet_utils.c b/VNFs/DPPD-PROX/packet_utils.c
index fb467555..e06529c4 100644
--- a/VNFs/DPPD-PROX/packet_utils.c
+++ b/VNFs/DPPD-PROX/packet_utils.c
@@ -24,6 +24,7 @@
 #include "defines.h"
 #include "handle_master.h"
 #include "prox_port_cfg.h"
+#include "packet_utils.h"
 
 static inline int find_ip(struct ether_hdr_arp *pkt, uint16_t len, uint32_t *ip_dst)
 {
@@ -76,7 +77,7 @@ static inline int find_ip(struct ether_hdr_arp *pkt, uint16_t len, uint32_t *ip_
    but this would require either thread safety, or the the exchange of information between master and generating core.
 */
 
-int write_dst_mac(struct task_base *tbase, struct rte_mbuf *mbuf, uint32_t *ip_dst)
+int write_dst_mac(struct task_base *tbase, struct rte_mbuf *mbuf, uint32_t *ip_dst, uint64_t **time)
 {
 	const uint64_t hz = rte_get_tsc_hz();
 	struct ether_hdr_arp *packet = rte_pktmbuf_mtod(mbuf, struct ether_hdr_arp *);
@@ -89,8 +90,9 @@ int write_dst_mac(struct task_base *tbase, struct rte_mbuf *mbuf, uint32_t *ip_d
 			memcpy(mac, &l3->gw.mac, sizeof(prox_rte_ether_addr));
 			return SEND_MBUF;
 		} else if (tsc > l3->gw.arp_update_time) {
-			// long time since we have sent an arp, send arp
-			l3->gw.arp_update_time = tsc + l3->arp_update_time * hz / 1000;
+			// long time since we have successfully sent an arp, send arp
+			// If sending ARP failed (ring full) then arp_update_time is not updated to avoid having to wait 1 sec to send ARP REQ again
+			*time = &l3->gw.arp_update_time;
 			*ip_dst = l3->gw.ip;
 			if ((l3->flags & FLAG_DST_MAC_KNOWN) && (tsc < l3->gw.arp_timeout)){
 				// MAC is valid in the table => send also the mbuf
@@ -121,7 +123,7 @@ int write_dst_mac(struct task_base *tbase, struct rte_mbuf *mbuf, uint32_t *ip_d
 					return SEND_MBUF;
 				} else if (tsc > l3->optimized_arp_table[idx].arp_update_time) {
 					// ARP not sent since a long time, send ARP
-					l3->optimized_arp_table[idx].arp_update_time = tsc + l3->arp_update_time * hz / 1000;
+						*time = &l3->optimized_arp_table[idx].arp_update_time;
 					if (tsc < l3->optimized_arp_table[idx].arp_timeout) {
 						// MAC still valid => also send mbuf
 						memcpy(mac, &l3->optimized_arp_table[idx].mac, sizeof(prox_rte_ether_addr));
@@ -138,7 +140,7 @@ int write_dst_mac(struct task_base *tbase, struct rte_mbuf *mbuf, uint32_t *ip_d
 		}
 		// IP address not found in table
 		l3->optimized_arp_table[l3->n_pkts].ip = *ip_dst;
-		l3->optimized_arp_table[l3->n_pkts].arp_update_time = tsc + l3->arp_update_time * hz / 1000;
+		*time = &l3->optimized_arp_table[l3->n_pkts].arp_update_time;
 		l3->n_pkts++;
 
 		if (l3->n_pkts < 4) {
@@ -172,7 +174,7 @@ int write_dst_mac(struct task_base *tbase, struct rte_mbuf *mbuf, uint32_t *ip_d
 				return DROP_MBUF;
 			} else {
 				l3->arp_table[ret].ip = *ip_dst;
-				l3->arp_table[ret].arp_update_time = tsc + l3->arp_update_time * hz / 1000;
+				*time = &l3->arp_table[ret].arp_update_time;
 			}
 			return SEND_ARP;
 		} else {
@@ -183,7 +185,7 @@ int write_dst_mac(struct task_base *tbase, struct rte_mbuf *mbuf, uint32_t *ip_d
 				return SEND_MBUF;
 			} else if (tsc > l3->arp_table[ret].arp_update_time) {
 				// ARP not sent since a long time, send ARP
-				l3->arp_table[ret].arp_update_time = tsc + l3->arp_update_time * hz / 1000;
+				*time = &l3->arp_table[ret].arp_update_time;
 				if (tsc < l3->arp_table[ret].arp_timeout) {
 					// MAC still valid => send also MBUF
 					memcpy(mac, &l3->arp_table[ret].mac, sizeof(prox_rte_ether_addr));
@@ -230,6 +232,7 @@ void task_init_l3(struct task_base *tbase, struct task_args *targ)
 	tbase->l3.core_id = targ->lconf->id;
 	tbase->l3.task_id = targ->id;
 	tbase->l3.tmaster = targ->tmaster;
+	tbase->l3.seed = (uint)rte_rdtsc();
 	if (targ->arp_timeout != 0)
 		tbase->l3.arp_timeout = targ->arp_timeout;
 	else
@@ -344,6 +347,7 @@ void handle_ctrl_plane_pkts(struct task_base *tbase, struct rte_mbuf **mbufs, ui
 				memcpy(&l3->gw.mac, &hdr->arp.data.sha, 6);
 				l3->flags |= FLAG_DST_MAC_KNOWN;
 				l3->gw.arp_timeout = tsc + update_time;
+				update_arp_update_time(l3, &l3->gw.arp_update_time, l3->arp_update_time);
 			} else if (l3->n_pkts < 4) {
 				// Few packets tracked - should be faster to loop through them thean using a hash table
 				for (idx = 0; idx < l3->n_pkts; idx++) {
@@ -354,6 +358,7 @@ void handle_ctrl_plane_pkts(struct task_base *tbase, struct rte_mbuf **mbufs, ui
 				if (idx < l3->n_pkts) {
 					memcpy(&l3->optimized_arp_table[idx].mac, &(hdr->arp.data.sha), sizeof(prox_rte_ether_addr));
 					l3->optimized_arp_table[idx].arp_timeout = tsc + update_time;
+					update_arp_update_time(l3, &l3->optimized_arp_table[idx].arp_update_time, l3->arp_update_time);
 				}
 			} else {
 				int ret = rte_hash_add_key(l3->ip_hash, (const void *)&ip);
@@ -362,6 +367,7 @@ void handle_ctrl_plane_pkts(struct task_base *tbase, struct rte_mbuf **mbufs, ui
 				} else {
 					memcpy(&l3->arp_table[ret].mac, &(hdr->arp.data.sha), sizeof(prox_rte_ether_addr));
 					l3->arp_table[ret].arp_timeout = tsc + update_time;
+					update_arp_update_time(l3, &l3->arp_table[ret].arp_update_time, l3->arp_update_time);
 				}
 			}
 			tx_drop(mbufs[j]);
diff --git a/VNFs/DPPD-PROX/packet_utils.h b/VNFs/DPPD-PROX/packet_utils.h
index d6e2dbb9..a111b944 100644
--- a/VNFs/DPPD-PROX/packet_utils.h
+++ b/VNFs/DPPD-PROX/packet_utils.h
@@ -60,13 +60,22 @@ struct l3_base {
 	struct rte_hash *ip_hash;
 	struct arp_table *arp_table;
 	struct rte_mempool *arp_pool;
+	uint seed;
 };
 
 void task_init_l3(struct task_base *tbase, struct task_args *targ);
 void task_start_l3(struct task_base *tbase, struct task_args *targ);
-int write_dst_mac(struct task_base *tbase, struct rte_mbuf *mbuf, uint32_t *ip_dst);
+int write_dst_mac(struct task_base *tbase, struct rte_mbuf *mbuf, uint32_t *ip_dst, uint64_t **time);
 void task_set_gateway_ip(struct task_base *tbase, uint32_t ip);
 void task_set_local_ip(struct task_base *tbase, uint32_t ip);
 void handle_ctrl_plane_pkts(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts);
+static inline void update_arp_update_time(struct l3_base *l3, uint64_t *ptr, uint32_t base)
+{
+	// randomize timers - from 0.5 to 1.5 * configured time
+	const uint64_t hz = rte_get_tsc_hz();
+	uint64_t tsc = rte_rdtsc();
+	uint64_t rand = 500 + (1000L * rand_r(&l3->seed)) / RAND_MAX;
+	*ptr = tsc + (base * rand / 1000) * hz / 1000;
+}
 
 #endif /* _PACKET_UTILS_H_ */
diff --git a/VNFs/DPPD-PROX/tx_pkt.c b/VNFs/DPPD-PROX/tx_pkt.c
index e789d305..8bf501f6 100644
--- a/VNFs/DPPD-PROX/tx_pkt.c
+++ b/VNFs/DPPD-PROX/tx_pkt.c
@@ -56,11 +56,12 @@ int tx_pkt_l3(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts,
 	int first = 0, ret, ok = 0, rc;
 	const struct port_queue *port_queue = &tbase->tx_params_hw.tx_port_queue[0];
 	struct rte_mbuf *arp_mbuf = NULL;       // used when one need to send both an ARP and a mbuf
+	uint64_t *time;
 
 	for (int j = 0; j < n_pkts; j++) {
 		if ((out) && (out[j] >= OUT_HANDLED))
 			continue;
-		if (unlikely((rc = write_dst_mac(tbase, mbufs[j], &ip_dst)) != SEND_MBUF)) {
+		if (unlikely((rc = write_dst_mac(tbase, mbufs[j], &ip_dst, &time)) != SEND_MBUF)) {
 			if (j - first) {
 				ret = tbase->aux->tx_pkt_l2(tbase, mbufs + first, j - first, out);
 				ok += ret;
@@ -70,14 +71,20 @@ int tx_pkt_l3(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts,
 			case SEND_ARP:
 				// We re-use the mbuf - no need to create a arp_mbuf and delete the existing mbuf
 				mbufs[j]->port = tbase->l3.reachable_port_id;
-				tx_ring_cti(tbase, tbase->l3.ctrl_plane_ring, REQ_MAC_TO_CTRL, mbufs[j], tbase->l3.core_id, tbase->l3.task_id, ip_dst);
+				if (tx_ring_cti(tbase, tbase->l3.ctrl_plane_ring, REQ_MAC_TO_CTRL, mbufs[j], tbase->l3.core_id, tbase->l3.task_id, ip_dst) == 0)
+					update_arp_update_time(&tbase->l3, time, 1000);
+				else
+					update_arp_update_time(&tbase->l3, time, 100);
 				break;
 			case SEND_MBUF_AND_ARP:
 				// We send the mbuf and an ARP - we need to allocate another mbuf for ARP
 				ret = rte_mempool_get(tbase->l3.arp_pool, (void **)&arp_mbuf);
 				if (likely(ret == 0))   {
 					arp_mbuf->port = tbase->l3.reachable_port_id;
-					tx_ring_cti(tbase, tbase->l3.ctrl_plane_ring, REQ_MAC_TO_CTRL, arp_mbuf, tbase->l3.core_id, tbase->l3.task_id, ip_dst);
+					if (tx_ring_cti(tbase, tbase->l3.ctrl_plane_ring, REQ_MAC_TO_CTRL, arp_mbuf, tbase->l3.core_id, tbase->l3.task_id, ip_dst) == 0)
+						update_arp_update_time(&tbase->l3, time, 1000);
+					else
+						update_arp_update_time(&tbase->l3, time, 100);
 				} else {
 					plog_err("Failed to get a mbuf from arp mempool\n");
 					// We still send the initial mbuf
@@ -805,7 +812,7 @@ static inline int tx_ring_all(struct task_base *tbase, struct rte_ring *ring, ui
 	return rte_ring_enqueue(ring, mbuf);
 }
 
-void tx_ring_cti(struct task_base *tbase, struct rte_ring *ring, uint16_t command,  struct rte_mbuf *mbuf, uint8_t core_id, uint8_t task_id, uint32_t ip)
+int tx_ring_cti(struct task_base *tbase, struct rte_ring *ring, uint16_t command,  struct rte_mbuf *mbuf, uint8_t core_id, uint8_t task_id, uint32_t ip)
 {
 	plogx_dbg("\tSending command %s with ip %d.%d.%d.%d to ring %p using mbuf %p, core %d and task %d - ring size now %d\n", actions_string[command], IP4(ip), ring, mbuf, core_id, task_id, rte_ring_free_count(ring));
 	int ret = tx_ring_all(tbase, ring, command,  mbuf, core_id, task_id, ip);
@@ -814,6 +821,7 @@ void tx_ring_cti(struct task_base *tbase, struct rte_ring *ring, uint16_t comman
 		TASK_STATS_ADD_DROP_DISCARD(&tbase->aux->stats, 1);
 		rte_pktmbuf_free(mbuf);
 	}
+	return ret;
 }
 
 void tx_ring_ip(struct task_base *tbase, struct rte_ring *ring, uint16_t command,  struct rte_mbuf *mbuf, uint32_t ip)
diff --git a/VNFs/DPPD-PROX/tx_pkt.h b/VNFs/DPPD-PROX/tx_pkt.h
index a6881531..708a9837 100644
--- a/VNFs/DPPD-PROX/tx_pkt.h
+++ b/VNFs/DPPD-PROX/tx_pkt.h
@@ -82,7 +82,7 @@ uint16_t tx_try_self(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t
 int tx_pkt_drop_all(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts, uint8_t *out);
 int tx_pkt_l3(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts, uint8_t *out);
 
-void tx_ring_cti(struct task_base *tbase, struct rte_ring *ring, uint16_t command, struct rte_mbuf *mbuf, uint8_t core_id, uint8_t task_id, uint32_t ip);
+int tx_ring_cti(struct task_base *tbase, struct rte_ring *ring, uint16_t command, struct rte_mbuf *mbuf, uint8_t core_id, uint8_t task_id, uint32_t ip);
 void tx_ring_ip(struct task_base *tbase, struct rte_ring *ring, uint16_t command, struct rte_mbuf *mbuf, uint32_t ip);
 void tx_ring(struct task_base *tbase, struct rte_ring *ring, uint16_t command, struct rte_mbuf *mbuf);