Added support for secondary process & virtual devices

A virtual device can be created adding the following option on the eal field within
the eal options section:
--vdev=net_ring0
This will cause a new (virtual) dpdk port to be created.

Such a device can then be used in a primary process as other DPDK devices i.e.
a [port x] section must be defined in the configuration file

For a secondary process, there is no need to add --vdev, but, of course
--proc-type=secondary (or --proc-type=auto) must be set within the eal field.

There are two ways to use such a virtual device in a secondary process
1) Using the DPDK port number. Note however that the DPDK port number of virtual devices
in the secondary process might not start from 0 and might hence have to be guessed
e.g. if using 2 virtual devices in primary process (--vdev=net_ring0 --vdev=net_ring1)
the port_id will be 0 and 1 in primary process and 2 & 3 in secondary process.

2) Using the port name, as defined in --vdev in the primary process. In this way,
no [port] section must be defined, and the virtual port names are directly used within
the "rx port" and "tx port" configuration e.g.:
rx port=net_ring0
tx port=net_ring1

Limitations
===========
There seems to be a PROX leak causing DPDK port id to increase when restarting the
secondary process (causing the secondary process configuration through port id more difficult).

As the primary process configures the port (including the number of rx and tx queues)
based on its config file, the secondary process can't setup its own queues.
Simple configurations (such as using gen or lat in primary, and swap in secondary) work as PROX
allocates a RX queue even in gen mode and a TX queue even in lat mode. Better configuration/support
for secondary process should be designed.

The use of dpdk port names is only supported by PROX within "rx port" and "tx port". It is not
supported by other configuration fields such as "tx ports from routing table".

The use of dpdk port names in "rx port" and "tx port" is limited to only 1 port.

Change-Id: Iaa606625da471403713a21df79d3ded4bb91b91e
Signed-off-by: Xavier Simonart <xavier.simonart@intel.com>

7 files changed, 118 insertions, 19 deletions

diff --git a/VNFs/DPPD-PROX/handle_swap.c b/VNFs/DPPD-PROX/handle_swap.c
index a7a153a4..b9029b6b 100644
--- a/VNFs/DPPD-PROX/handle_swap.c
+++ b/VNFs/DPPD-PROX/handle_swap.c
@@ -133,6 +133,15 @@ static inline void build_igmp_message(struct task_base *tbase, struct rte_mbuf *
 	prox_ip_udp_cksum(mbuf, ip_hdr, sizeof(prox_rte_ether_hdr), sizeof(prox_rte_ipv4_hdr), task->offload_crc);
 }
 
+static void stop_swap(struct task_base *tbase)
+{
+	struct task_swap *task = (struct task_swap *)tbase;
+	if (task->igmp_pool) {
+		rte_mempool_free(task->igmp_pool);
+		task->igmp_pool = NULL;
+	}
+}
+
 static int handle_swap_bulk(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts)
 {
 	struct task_swap *task = (struct task_swap *)tbase;
@@ -432,6 +441,7 @@ static struct task_init task_init_swap = {
 	.handle = handle_swap_bulk,
 	.flag_features = 0,
 	.size = sizeof(struct task_swap),
+	.stop_last = stop_swap
 };
 
 __attribute__((constructor)) static void reg_task_swap(void)
diff --git a/VNFs/DPPD-PROX/main.c b/VNFs/DPPD-PROX/main.c
index d87561d0..1af49b7d 100644
--- a/VNFs/DPPD-PROX/main.c
+++ b/VNFs/DPPD-PROX/main.c
@@ -748,19 +748,21 @@ static void setup_mempools_unique_per_socket(void)
 	for (int i = 0 ; i < MAX_SOCKETS; i++) {
 		if (mbuf_count[i] != 0) {
 			sprintf(name, "socket_%u_pool", i);
-			pool[i] = rte_mempool_create(name,
+			if ((pool[i] = rte_mempool_lookup(name)) == NULL) {
+				pool[i] = rte_mempool_create(name,
 						     mbuf_count[i] - 1, mbuf_size[i],
 						     nb_cache_mbuf[i],
 						     sizeof(struct rte_pktmbuf_pool_private),
 						     rte_pktmbuf_pool_init, NULL,
 						     prox_pktmbuf_init, NULL,
 						     i, flags);
-			PROX_PANIC(pool[i] == NULL, "\t\tError: cannot create mempool for socket %u\n", i);
-			plog_info("\t\tMempool %p size = %u * %u cache %u, socket %d\n", pool[i],
-				  mbuf_count[i], mbuf_size[i], nb_cache_mbuf[i], i);
+				PROX_PANIC(pool[i] == NULL, "\t\tError: cannot create mempool for socket %u\n", i);
+				plog_info("\tMempool %p size = %u * %u cache %u, socket %d\n", pool[i],
+				  	mbuf_count[i], mbuf_size[i], nb_cache_mbuf[i], i);
 
-			if (prox_cfg.flags & DSF_SHUFFLE) {
-				shuffle_mempool(pool[i], mbuf_count[i]);
+				if (prox_cfg.flags & DSF_SHUFFLE) {
+					shuffle_mempool(pool[i], mbuf_count[i]);
+				}
 			}
 		}
 	}
@@ -775,7 +777,7 @@ static void setup_mempools_unique_per_socket(void)
 			targ->pool = pool[socket];
 			/* Set the number of mbuf to the number of the unique mempool, so that the used and free work */
 			targ->nb_mbuf = mbuf_count[socket];
-			plog_info("\t\tMempool %p size = %u * %u cache %u, socket %d\n", targ->pool,
+			plog_info("\tMempool %p size = %u * %u cache %u, socket %d\n", targ->pool,
 				  targ->nb_mbuf, mbuf_size[socket], targ->nb_cache_mbuf, socket);
 		}
 	}
@@ -797,7 +799,7 @@ static void setup_mempool_for_rx_task(struct lcore_cfg *lconf, struct task_args
 	PROX_ASSERT(targ->nb_mbuf != 0);
 
 	if (targ->pool_name[0] == '\0') {
-		sprintf(name, "core_%u_port_%u_pool", lconf->id, targ->id);
+		sprintf(name, "core_%u_task_%u_pool", lconf->id, targ->id);
 	}
 
 	snprintf(memzone_name, sizeof(memzone_name)-1, "MP_%s", targ->pool_name);
@@ -830,7 +832,7 @@ static void setup_mempool_for_rx_task(struct lcore_cfg *lconf, struct task_args
 	   receiving from if one core receives from multiple
 	   ports, all the ports use the same mempool */
 	if (targ->pool == NULL) {
-		plog_info("\t\tCreating mempool with name '%s'\n", name);
+		plog_info("\tCreating mempool with name '%s' on socket %d\n", name, socket);
 		targ->pool = rte_mempool_create(name,
 						targ->nb_mbuf - 1, targ->mbuf_size,
 						targ->nb_cache_mbuf,
@@ -841,9 +843,9 @@ static void setup_mempool_for_rx_task(struct lcore_cfg *lconf, struct task_args
 	}
 
 	PROX_PANIC(targ->pool == NULL,
-		   "\t\tError: cannot create mempool for core %u port %u: %s\n", lconf->id, targ->id, rte_strerror(rte_errno));
+		   "\tError: cannot create mempool for core %u port %u: %s\n", lconf->id, targ->id, rte_strerror(rte_errno));
 
-	plog_info("\t\tMempool %p size = %u * %u cache %u, socket %d\n", targ->pool,
+	plog_info("\tMempool %p size = %u * %u cache %u, socket %d\n", targ->pool,
 		  targ->nb_mbuf, targ->mbuf_size, targ->nb_cache_mbuf, socket);
 	if (prox_cfg.flags & DSF_SHUFFLE) {
 		shuffle_mempool(targ->pool, targ->nb_mbuf);
@@ -1230,5 +1232,6 @@ int main(int argc, char **argv)
 	if (setup_prox(argc, argv) != 0)
 		return EXIT_FAILURE;
 	run(prox_cfg.flags);
+
 	return EXIT_SUCCESS;
 }
diff --git a/VNFs/DPPD-PROX/prox_args.c b/VNFs/DPPD-PROX/prox_args.c
index 9e12eb68..cb0dcb31 100644
--- a/VNFs/DPPD-PROX/prox_args.c
+++ b/VNFs/DPPD-PROX/prox_args.c
@@ -137,6 +137,15 @@ static struct cfg_section core_cfg = {
 	.error  = 0
 };
 
+struct deferred_port {
+	struct task_args *targ;
+	char name[256];
+	uint8_t is_rx_port;
+};
+
+static struct deferred_port deferred_port[PROX_MAX_PORTS];
+static int n_deferred_ports = 0;
+
 static void set_errf(const char *format, ...)
 {
 	va_list ap;
@@ -901,7 +910,17 @@ static int get_core_cfg(unsigned sindex, char *str, void *data)
 		uint32_t ports[PROX_MAX_PORTS];
 
 		if(parse_port_name_list(ports, &n_if, PROX_MAX_PORTS, pkey)) {
-			return -1;
+			// Port name not found, but could be a virtual device of a secondary process
+			// As DPDK not started yet, we can only check the config file to see whether we are a secondary process
+			if (rte_cfg.eal &&
+					(strstr(rte_cfg.eal, "secondary") || strstr(rte_cfg.eal, "auto")) &&
+					(n_deferred_ports < PROX_MAX_PORTS)) {
+				prox_strncpy(deferred_port[n_deferred_ports].name, pkey, sizeof(deferred_port[n_deferred_ports].name));
+				deferred_port[n_deferred_ports].is_rx_port = 0;
+				deferred_port[n_deferred_ports++].targ = targ;
+				return 0;
+			} else
+				return -1;
 		}
 
 		PROX_ASSERT(n_if-1 < PROX_MAX_PORTS);
@@ -1111,7 +1130,17 @@ static int get_core_cfg(unsigned sindex, char *str, void *data)
 		uint32_t n_if;
 
                 if (parse_port_name_list(vals, &n_if, PROX_MAX_PORTS, pkey)) {
-                        return -1;
+			// Port name not found, but could be a virtual device of a secondary process
+			// As DPDK not started yet, we can only check the config file to see whether we are a secondary process
+			if (rte_cfg.eal &&
+					(strstr(rte_cfg.eal, "secondary") || strstr(rte_cfg.eal, "auto")) &&
+					(n_deferred_ports < PROX_MAX_PORTS)) {
+				prox_strncpy(deferred_port[n_deferred_ports].name, pkey, sizeof(deferred_port[n_deferred_ports].name));
+				deferred_port[n_deferred_ports].is_rx_port = 1;
+				deferred_port[n_deferred_ports++].targ = targ;
+				return 0;
+			} else
+				return -1;
                 }
 
                 for (uint8_t i = 0; i < n_if; ++i) {
@@ -2171,5 +2200,20 @@ int prox_setup_rte(const char *prog_name)
 			return -1;
 		}
 	}
+	uint16_t port_id;
+	for (int i = 0; i < n_deferred_ports; i++) {
+		if (rte_eth_dev_get_port_by_name(deferred_port[i].name, &port_id) != 0) {
+			plog_err("Did not find port name %s used while reading %s\n", deferred_port[i].name, deferred_port[i].is_rx_port ? "rx port" : "tx_port");
+			return -1;
+		}
+		plog_info("\tport %s is port id %d\n", deferred_port[i].name, port_id);
+		if (deferred_port[i].is_rx_port) {
+			deferred_port[i].targ->rx_port_queue[0].port = port_id;
+			deferred_port[i].targ->nb_rxports = 1;
+		} else {
+			deferred_port[i].targ->tx_port_queue[0].port = port_id;
+			deferred_port[i].targ->nb_txports = 1;
+		}
+	}
 	return 0;
 }
diff --git a/VNFs/DPPD-PROX/prox_port_cfg.c b/VNFs/DPPD-PROX/prox_port_cfg.c
index 2abf4d58..098d973b 100644
--- a/VNFs/DPPD-PROX/prox_port_cfg.c
+++ b/VNFs/DPPD-PROX/prox_port_cfg.c
@@ -47,6 +47,7 @@
 #include "rte_ethdev.h"
 
 struct prox_port_cfg prox_port_cfg[PROX_MAX_PORTS];
+
 rte_atomic32_t lsc;
 
 int prox_nb_active_ports(void)
@@ -271,17 +272,33 @@ void init_rte_dev(int use_dummy_devices)
 
 		nb_ports = PROX_MAX_PORTS;
 	}
-	port_id_max = nb_ports - 1;
+	port_id_max = -1;
+	uint16_t id;
+	RTE_ETH_FOREACH_DEV(id) {
+		char name[256];
+		rte_eth_dev_get_name_by_port(id, name);
+		plog_info("\tFound DPDK port id %u %s\n", id, name);
+		if (id >= PROX_MAX_PORTS) {
+			plog_warn("\tWarning: I can deal with at most %u ports."
+				 " Please update PROX_MAX_PORTS and recompile.\n", PROX_MAX_PORTS);
+		} else {
+			prox_port_cfg[id].available = 1;
+			if (id > port_id_max)
+				port_id_max = id;
+		}
+	}
 	port_id_last = prox_last_port_active();
 	PROX_PANIC(port_id_last > port_id_max,
 		   "\tError: invalid port(s) specified, last port index active: %d (max index is %d)\n",
 		   port_id_last, port_id_max);
 
 	/* Assign ports to PROX interfaces & Read max RX/TX queues per port */
-	for (uint8_t port_id = 0; port_id < nb_ports; ++port_id) {
+	for (uint8_t port_id = 0; port_id <= port_id_last; ++port_id) {
 		/* skip ports that are not enabled */
 		if (!prox_port_cfg[port_id].active) {
 			continue;
+		} else if (prox_port_cfg[port_id].available == 0) {
+			PROX_PANIC(1, "port %u enabled but not available\n", port_id);
 		}
 		plog_info("\tGetting info for rte dev %u\n", port_id);
 		rte_eth_dev_info_get(port_id, &dev_info);
@@ -813,6 +830,12 @@ static void init_port(struct prox_port_cfg *port_cfg)
 
 void init_port_all(void)
 {
+	enum rte_proc_type_t proc_type;
+	proc_type = rte_eal_process_type();
+	if (proc_type == RTE_PROC_SECONDARY) {
+		plog_info("\tSkipping port initialization as secondary process\n");
+		return;
+	}
 	uint8_t max_port_idx = prox_last_port_active() + 1;
 
 	for (uint8_t portid = 0; portid < max_port_idx; ++portid) {
@@ -831,6 +854,7 @@ void close_ports_atexit(void)
 		if (!prox_port_cfg[portid].active) {
 			continue;
 		}
+		plog_info("Closing port %u\n", portid);
 		rte_eth_dev_close(portid);
 	}
 }
@@ -838,6 +862,7 @@ void close_ports_atexit(void)
 void init_port_addr(void)
 {
 	struct prox_port_cfg *port_cfg;
+	enum rte_proc_type_t proc_type;
 	int rc;
 
 	for (uint8_t port_id = 0; port_id < PROX_MAX_PORTS; ++port_id) {
@@ -854,9 +879,13 @@ void init_port_addr(void)
 			prox_rte_eth_random_addr(port_cfg->eth_addr.addr_bytes);
 			break;
 		case PROX_PORT_MAC_SET:
-			plog_info("Setting MAC to "MAC_BYTES_FMT"\n", MAC_BYTES(port_cfg->eth_addr.addr_bytes));
-			if ((rc = rte_eth_dev_default_mac_addr_set(port_id, &port_cfg->eth_addr)) != 0)
-				plog_warn("port %u: failed to set mac address. Error = %d\n", port_id, rc);
+			proc_type = rte_eal_process_type();
+			if (proc_type == RTE_PROC_SECONDARY) {
+				plog_warn("\tport %u: unable to change port mac address as secondary process\n", port_id);
+			} else if ((rc = rte_eth_dev_default_mac_addr_set(port_id, &port_cfg->eth_addr)) != 0)
+				plog_warn("\tport %u: failed to set mac address. Error = %d\n", port_id, rc);
+			else
+				plog_info("Setting MAC to "MAC_BYTES_FMT"\n", MAC_BYTES(port_cfg->eth_addr.addr_bytes));
 			break;
 		}
 	}
diff --git a/VNFs/DPPD-PROX/prox_port_cfg.h b/VNFs/DPPD-PROX/prox_port_cfg.h
index ad3d9380..94f2c41a 100644
--- a/VNFs/DPPD-PROX/prox_port_cfg.h
+++ b/VNFs/DPPD-PROX/prox_port_cfg.h
@@ -79,6 +79,7 @@ struct prox_port_cfg {
 	uint16_t min_tx_desc;
 	uint16_t max_tx_desc;
 	uint32_t nb_mc_addr;
+	uint8_t available;
 	prox_rte_ether_addr mc_addr[NB_MCAST_ADDR];
 	int dpdk_mapping;
 	uint32_t ip;
diff --git a/VNFs/DPPD-PROX/run.c b/VNFs/DPPD-PROX/run.c
index c05f0a9f..d568dba4 100644
--- a/VNFs/DPPD-PROX/run.c
+++ b/VNFs/DPPD-PROX/run.c
@@ -277,6 +277,15 @@ void __attribute__((noreturn)) run(uint32_t flags)
 		stop_core_all(-1);
 	}
 
+	struct lcore_cfg *lconf = NULL;
+	struct task_args *targ;
+	while (core_targ_next(&lconf, &targ, 0) == 0) {
+		if (targ->pool) {
+			rte_mempool_free(targ->pool);
+			plog_info("freeing pool %p\n", targ->pool);
+			targ->pool = NULL;
+		}
+	}
 	if (prox_cfg.logbuf) {
 		file_print(prox_cfg.logbuf);
 	}
diff --git a/VNFs/DPPD-PROX/stats_port.c b/VNFs/DPPD-PROX/stats_port.c
index e6210c33..124c849e 100644
--- a/VNFs/DPPD-PROX/stats_port.c
+++ b/VNFs/DPPD-PROX/stats_port.c
@@ -169,7 +169,10 @@ void stats_port_init(void)
 	for (uint8_t port_id = 0; port_id < nb_interface; ++port_id) {
 		if (prox_port_cfg[port_id].active) {
 #if RTE_VERSION >= RTE_VERSION_NUM(16,7,0,0)
-			num_xstats[port_id] = rte_eth_xstats_get_names(port_id, NULL, 0);
+			if ((num_xstats[port_id] = rte_eth_xstats_get_names(port_id, NULL, 0)) < 0) {
+				plog_err("\tport %u: rte_eth_xstats_get_names returns %d\n", port_id, num_xstats[port_id]);
+				continue;
+			}
 			eth_xstat_names[port_id] = prox_zmalloc(num_xstats[port_id] * sizeof(struct rte_eth_xstat_name), prox_port_cfg[port_id].socket);
 			PROX_PANIC(eth_xstat_names[port_id] == NULL, "Error allocating memory for xstats");
 			num_xstats[port_id] = rte_eth_xstats_get_names(port_id, eth_xstat_names[port_id], num_xstats[port_id]);