summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorXavier Simonart <xavier.simonart@intel.com>2020-03-24 00:31:13 +0100
committerXavier Simonart <xavier.simonart@intel.com>2020-05-29 23:40:28 +0200
commitc61fccde40dc6bb4a6ecd21c9d6dc8969df33400 (patch)
treeacdf0a31a6d1eea0e97f3ef6d1b70f4b97bc0dc0
parent4359ac786cc49d0ce20f745def415d46fbba177d (diff)
Added support for secondary process & virtual devices
A virtual device can be created adding the following option on the eal field within the eal options section: --vdev=net_ring0 This will cause a new (virtual) dpdk port to be created. Such a device can then be used in a primary process as other DPDK devices i.e. a [port x] section must be defined in the configuration file For a secondary process, there is no need to add --vdev, but, of course --proc-type=secondary (or --proc-type=auto) must be set within the eal field. There are two ways to use such a virtual device in a secondary process 1) Using the DPDK port number. Note however that the DPDK port number of virtual devices in the secondary process might not start from 0 and might hence have to be guessed e.g. if using 2 virtual devices in primary process (--vdev=net_ring0 --vdev=net_ring1) the port_id will be 0 and 1 in primary process and 2 & 3 in secondary process. 2) Using the port name, as defined in --vdev in the primary process. In this way, no [port] section must be defined, and the virtual port names are directly used within the "rx port" and "tx port" configuration e.g.: rx port=net_ring0 tx port=net_ring1 Limitations =========== There seems to be a PROX leak causing DPDK port id to increase when restarting the secondary process (causing the secondary process configuration through port id more difficult). As the primary process configures the port (including the number of rx and tx queues) based on its config file, the secondary process can't setup its own queues. Simple configurations (such as using gen or lat in primary, and swap in secondary) work as PROX allocates a RX queue even in gen mode and a TX queue even in lat mode. Better configuration/support for secondary process should be designed. The use of dpdk port names is only supported by PROX within "rx port" and "tx port". It is not supported by other configuration fields such as "tx ports from routing table". The use of dpdk port names in "rx port" and "tx port" is limited to only 1 port. Change-Id: Iaa606625da471403713a21df79d3ded4bb91b91e Signed-off-by: Xavier Simonart <xavier.simonart@intel.com>
-rw-r--r--VNFs/DPPD-PROX/handle_swap.c10
-rw-r--r--VNFs/DPPD-PROX/main.c25
-rw-r--r--VNFs/DPPD-PROX/prox_args.c48
-rw-r--r--VNFs/DPPD-PROX/prox_port_cfg.c39
-rw-r--r--VNFs/DPPD-PROX/prox_port_cfg.h1
-rw-r--r--VNFs/DPPD-PROX/run.c9
-rw-r--r--VNFs/DPPD-PROX/stats_port.c5
7 files changed, 118 insertions, 19 deletions
diff --git a/VNFs/DPPD-PROX/handle_swap.c b/VNFs/DPPD-PROX/handle_swap.c
index a7a153a..b9029b6 100644
--- a/VNFs/DPPD-PROX/handle_swap.c
+++ b/VNFs/DPPD-PROX/handle_swap.c
@@ -133,6 +133,15 @@ static inline void build_igmp_message(struct task_base *tbase, struct rte_mbuf *
prox_ip_udp_cksum(mbuf, ip_hdr, sizeof(prox_rte_ether_hdr), sizeof(prox_rte_ipv4_hdr), task->offload_crc);
}
+static void stop_swap(struct task_base *tbase)
+{
+ struct task_swap *task = (struct task_swap *)tbase;
+ if (task->igmp_pool) {
+ rte_mempool_free(task->igmp_pool);
+ task->igmp_pool = NULL;
+ }
+}
+
static int handle_swap_bulk(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts)
{
struct task_swap *task = (struct task_swap *)tbase;
@@ -432,6 +441,7 @@ static struct task_init task_init_swap = {
.handle = handle_swap_bulk,
.flag_features = 0,
.size = sizeof(struct task_swap),
+ .stop_last = stop_swap
};
__attribute__((constructor)) static void reg_task_swap(void)
diff --git a/VNFs/DPPD-PROX/main.c b/VNFs/DPPD-PROX/main.c
index d87561d..1af49b7 100644
--- a/VNFs/DPPD-PROX/main.c
+++ b/VNFs/DPPD-PROX/main.c
@@ -748,19 +748,21 @@ static void setup_mempools_unique_per_socket(void)
for (int i = 0 ; i < MAX_SOCKETS; i++) {
if (mbuf_count[i] != 0) {
sprintf(name, "socket_%u_pool", i);
- pool[i] = rte_mempool_create(name,
+ if ((pool[i] = rte_mempool_lookup(name)) == NULL) {
+ pool[i] = rte_mempool_create(name,
mbuf_count[i] - 1, mbuf_size[i],
nb_cache_mbuf[i],
sizeof(struct rte_pktmbuf_pool_private),
rte_pktmbuf_pool_init, NULL,
prox_pktmbuf_init, NULL,
i, flags);
- PROX_PANIC(pool[i] == NULL, "\t\tError: cannot create mempool for socket %u\n", i);
- plog_info("\t\tMempool %p size = %u * %u cache %u, socket %d\n", pool[i],
- mbuf_count[i], mbuf_size[i], nb_cache_mbuf[i], i);
+ PROX_PANIC(pool[i] == NULL, "\t\tError: cannot create mempool for socket %u\n", i);
+ plog_info("\tMempool %p size = %u * %u cache %u, socket %d\n", pool[i],
+ mbuf_count[i], mbuf_size[i], nb_cache_mbuf[i], i);
- if (prox_cfg.flags & DSF_SHUFFLE) {
- shuffle_mempool(pool[i], mbuf_count[i]);
+ if (prox_cfg.flags & DSF_SHUFFLE) {
+ shuffle_mempool(pool[i], mbuf_count[i]);
+ }
}
}
}
@@ -775,7 +777,7 @@ static void setup_mempools_unique_per_socket(void)
targ->pool = pool[socket];
/* Set the number of mbuf to the number of the unique mempool, so that the used and free work */
targ->nb_mbuf = mbuf_count[socket];
- plog_info("\t\tMempool %p size = %u * %u cache %u, socket %d\n", targ->pool,
+ plog_info("\tMempool %p size = %u * %u cache %u, socket %d\n", targ->pool,
targ->nb_mbuf, mbuf_size[socket], targ->nb_cache_mbuf, socket);
}
}
@@ -797,7 +799,7 @@ static void setup_mempool_for_rx_task(struct lcore_cfg *lconf, struct task_args
PROX_ASSERT(targ->nb_mbuf != 0);
if (targ->pool_name[0] == '\0') {
- sprintf(name, "core_%u_port_%u_pool", lconf->id, targ->id);
+ sprintf(name, "core_%u_task_%u_pool", lconf->id, targ->id);
}
snprintf(memzone_name, sizeof(memzone_name)-1, "MP_%s", targ->pool_name);
@@ -830,7 +832,7 @@ static void setup_mempool_for_rx_task(struct lcore_cfg *lconf, struct task_args
receiving from if one core receives from multiple
ports, all the ports use the same mempool */
if (targ->pool == NULL) {
- plog_info("\t\tCreating mempool with name '%s'\n", name);
+ plog_info("\tCreating mempool with name '%s' on socket %d\n", name, socket);
targ->pool = rte_mempool_create(name,
targ->nb_mbuf - 1, targ->mbuf_size,
targ->nb_cache_mbuf,
@@ -841,9 +843,9 @@ static void setup_mempool_for_rx_task(struct lcore_cfg *lconf, struct task_args
}
PROX_PANIC(targ->pool == NULL,
- "\t\tError: cannot create mempool for core %u port %u: %s\n", lconf->id, targ->id, rte_strerror(rte_errno));
+ "\tError: cannot create mempool for core %u port %u: %s\n", lconf->id, targ->id, rte_strerror(rte_errno));
- plog_info("\t\tMempool %p size = %u * %u cache %u, socket %d\n", targ->pool,
+ plog_info("\tMempool %p size = %u * %u cache %u, socket %d\n", targ->pool,
targ->nb_mbuf, targ->mbuf_size, targ->nb_cache_mbuf, socket);
if (prox_cfg.flags & DSF_SHUFFLE) {
shuffle_mempool(targ->pool, targ->nb_mbuf);
@@ -1230,5 +1232,6 @@ int main(int argc, char **argv)
if (setup_prox(argc, argv) != 0)
return EXIT_FAILURE;
run(prox_cfg.flags);
+
return EXIT_SUCCESS;
}
diff --git a/VNFs/DPPD-PROX/prox_args.c b/VNFs/DPPD-PROX/prox_args.c
index 9e12eb6..cb0dcb3 100644
--- a/VNFs/DPPD-PROX/prox_args.c
+++ b/VNFs/DPPD-PROX/prox_args.c
@@ -137,6 +137,15 @@ static struct cfg_section core_cfg = {
.error = 0
};
+struct deferred_port {
+ struct task_args *targ;
+ char name[256];
+ uint8_t is_rx_port;
+};
+
+static struct deferred_port deferred_port[PROX_MAX_PORTS];
+static int n_deferred_ports = 0;
+
static void set_errf(const char *format, ...)
{
va_list ap;
@@ -901,7 +910,17 @@ static int get_core_cfg(unsigned sindex, char *str, void *data)
uint32_t ports[PROX_MAX_PORTS];
if(parse_port_name_list(ports, &n_if, PROX_MAX_PORTS, pkey)) {
- return -1;
+ // Port name not found, but could be a virtual device of a secondary process
+ // As DPDK not started yet, we can only check the config file to see whether we are a secondary process
+ if (rte_cfg.eal &&
+ (strstr(rte_cfg.eal, "secondary") || strstr(rte_cfg.eal, "auto")) &&
+ (n_deferred_ports < PROX_MAX_PORTS)) {
+ prox_strncpy(deferred_port[n_deferred_ports].name, pkey, sizeof(deferred_port[n_deferred_ports].name));
+ deferred_port[n_deferred_ports].is_rx_port = 0;
+ deferred_port[n_deferred_ports++].targ = targ;
+ return 0;
+ } else
+ return -1;
}
PROX_ASSERT(n_if-1 < PROX_MAX_PORTS);
@@ -1111,7 +1130,17 @@ static int get_core_cfg(unsigned sindex, char *str, void *data)
uint32_t n_if;
if (parse_port_name_list(vals, &n_if, PROX_MAX_PORTS, pkey)) {
- return -1;
+ // Port name not found, but could be a virtual device of a secondary process
+ // As DPDK not started yet, we can only check the config file to see whether we are a secondary process
+ if (rte_cfg.eal &&
+ (strstr(rte_cfg.eal, "secondary") || strstr(rte_cfg.eal, "auto")) &&
+ (n_deferred_ports < PROX_MAX_PORTS)) {
+ prox_strncpy(deferred_port[n_deferred_ports].name, pkey, sizeof(deferred_port[n_deferred_ports].name));
+ deferred_port[n_deferred_ports].is_rx_port = 1;
+ deferred_port[n_deferred_ports++].targ = targ;
+ return 0;
+ } else
+ return -1;
}
for (uint8_t i = 0; i < n_if; ++i) {
@@ -2171,5 +2200,20 @@ int prox_setup_rte(const char *prog_name)
return -1;
}
}
+ uint16_t port_id;
+ for (int i = 0; i < n_deferred_ports; i++) {
+ if (rte_eth_dev_get_port_by_name(deferred_port[i].name, &port_id) != 0) {
+ plog_err("Did not find port name %s used while reading %s\n", deferred_port[i].name, deferred_port[i].is_rx_port ? "rx port" : "tx_port");
+ return -1;
+ }
+ plog_info("\tport %s is port id %d\n", deferred_port[i].name, port_id);
+ if (deferred_port[i].is_rx_port) {
+ deferred_port[i].targ->rx_port_queue[0].port = port_id;
+ deferred_port[i].targ->nb_rxports = 1;
+ } else {
+ deferred_port[i].targ->tx_port_queue[0].port = port_id;
+ deferred_port[i].targ->nb_txports = 1;
+ }
+ }
return 0;
}
diff --git a/VNFs/DPPD-PROX/prox_port_cfg.c b/VNFs/DPPD-PROX/prox_port_cfg.c
index 2abf4d5..098d973 100644
--- a/VNFs/DPPD-PROX/prox_port_cfg.c
+++ b/VNFs/DPPD-PROX/prox_port_cfg.c
@@ -47,6 +47,7 @@
#include "rte_ethdev.h"
struct prox_port_cfg prox_port_cfg[PROX_MAX_PORTS];
+
rte_atomic32_t lsc;
int prox_nb_active_ports(void)
@@ -271,17 +272,33 @@ void init_rte_dev(int use_dummy_devices)
nb_ports = PROX_MAX_PORTS;
}
- port_id_max = nb_ports - 1;
+ port_id_max = -1;
+ uint16_t id;
+ RTE_ETH_FOREACH_DEV(id) {
+ char name[256];
+ rte_eth_dev_get_name_by_port(id, name);
+ plog_info("\tFound DPDK port id %u %s\n", id, name);
+ if (id >= PROX_MAX_PORTS) {
+ plog_warn("\tWarning: I can deal with at most %u ports."
+ " Please update PROX_MAX_PORTS and recompile.\n", PROX_MAX_PORTS);
+ } else {
+ prox_port_cfg[id].available = 1;
+ if (id > port_id_max)
+ port_id_max = id;
+ }
+ }
port_id_last = prox_last_port_active();
PROX_PANIC(port_id_last > port_id_max,
"\tError: invalid port(s) specified, last port index active: %d (max index is %d)\n",
port_id_last, port_id_max);
/* Assign ports to PROX interfaces & Read max RX/TX queues per port */
- for (uint8_t port_id = 0; port_id < nb_ports; ++port_id) {
+ for (uint8_t port_id = 0; port_id <= port_id_last; ++port_id) {
/* skip ports that are not enabled */
if (!prox_port_cfg[port_id].active) {
continue;
+ } else if (prox_port_cfg[port_id].available == 0) {
+ PROX_PANIC(1, "port %u enabled but not available\n", port_id);
}
plog_info("\tGetting info for rte dev %u\n", port_id);
rte_eth_dev_info_get(port_id, &dev_info);
@@ -813,6 +830,12 @@ static void init_port(struct prox_port_cfg *port_cfg)
void init_port_all(void)
{
+ enum rte_proc_type_t proc_type;
+ proc_type = rte_eal_process_type();
+ if (proc_type == RTE_PROC_SECONDARY) {
+ plog_info("\tSkipping port initialization as secondary process\n");
+ return;
+ }
uint8_t max_port_idx = prox_last_port_active() + 1;
for (uint8_t portid = 0; portid < max_port_idx; ++portid) {
@@ -831,6 +854,7 @@ void close_ports_atexit(void)
if (!prox_port_cfg[portid].active) {
continue;
}
+ plog_info("Closing port %u\n", portid);
rte_eth_dev_close(portid);
}
}
@@ -838,6 +862,7 @@ void close_ports_atexit(void)
void init_port_addr(void)
{
struct prox_port_cfg *port_cfg;
+ enum rte_proc_type_t proc_type;
int rc;
for (uint8_t port_id = 0; port_id < PROX_MAX_PORTS; ++port_id) {
@@ -854,9 +879,13 @@ void init_port_addr(void)
prox_rte_eth_random_addr(port_cfg->eth_addr.addr_bytes);
break;
case PROX_PORT_MAC_SET:
- plog_info("Setting MAC to "MAC_BYTES_FMT"\n", MAC_BYTES(port_cfg->eth_addr.addr_bytes));
- if ((rc = rte_eth_dev_default_mac_addr_set(port_id, &port_cfg->eth_addr)) != 0)
- plog_warn("port %u: failed to set mac address. Error = %d\n", port_id, rc);
+ proc_type = rte_eal_process_type();
+ if (proc_type == RTE_PROC_SECONDARY) {
+ plog_warn("\tport %u: unable to change port mac address as secondary process\n", port_id);
+ } else if ((rc = rte_eth_dev_default_mac_addr_set(port_id, &port_cfg->eth_addr)) != 0)
+ plog_warn("\tport %u: failed to set mac address. Error = %d\n", port_id, rc);
+ else
+ plog_info("Setting MAC to "MAC_BYTES_FMT"\n", MAC_BYTES(port_cfg->eth_addr.addr_bytes));
break;
}
}
diff --git a/VNFs/DPPD-PROX/prox_port_cfg.h b/VNFs/DPPD-PROX/prox_port_cfg.h
index ad3d938..94f2c41 100644
--- a/VNFs/DPPD-PROX/prox_port_cfg.h
+++ b/VNFs/DPPD-PROX/prox_port_cfg.h
@@ -79,6 +79,7 @@ struct prox_port_cfg {
uint16_t min_tx_desc;
uint16_t max_tx_desc;
uint32_t nb_mc_addr;
+ uint8_t available;
prox_rte_ether_addr mc_addr[NB_MCAST_ADDR];
int dpdk_mapping;
uint32_t ip;
diff --git a/VNFs/DPPD-PROX/run.c b/VNFs/DPPD-PROX/run.c
index c05f0a9..d568dba 100644
--- a/VNFs/DPPD-PROX/run.c
+++ b/VNFs/DPPD-PROX/run.c
@@ -277,6 +277,15 @@ void __attribute__((noreturn)) run(uint32_t flags)
stop_core_all(-1);
}
+ struct lcore_cfg *lconf = NULL;
+ struct task_args *targ;
+ while (core_targ_next(&lconf, &targ, 0) == 0) {
+ if (targ->pool) {
+ rte_mempool_free(targ->pool);
+ plog_info("freeing pool %p\n", targ->pool);
+ targ->pool = NULL;
+ }
+ }
if (prox_cfg.logbuf) {
file_print(prox_cfg.logbuf);
}
diff --git a/VNFs/DPPD-PROX/stats_port.c b/VNFs/DPPD-PROX/stats_port.c
index e6210c3..124c849 100644
--- a/VNFs/DPPD-PROX/stats_port.c
+++ b/VNFs/DPPD-PROX/stats_port.c
@@ -169,7 +169,10 @@ void stats_port_init(void)
for (uint8_t port_id = 0; port_id < nb_interface; ++port_id) {
if (prox_port_cfg[port_id].active) {
#if RTE_VERSION >= RTE_VERSION_NUM(16,7,0,0)
- num_xstats[port_id] = rte_eth_xstats_get_names(port_id, NULL, 0);
+ if ((num_xstats[port_id] = rte_eth_xstats_get_names(port_id, NULL, 0)) < 0) {
+ plog_err("\tport %u: rte_eth_xstats_get_names returns %d\n", port_id, num_xstats[port_id]);
+ continue;
+ }
eth_xstat_names[port_id] = prox_zmalloc(num_xstats[port_id] * sizeof(struct rte_eth_xstat_name), prox_port_cfg[port_id].socket);
PROX_PANIC(eth_xstat_names[port_id] == NULL, "Error allocating memory for xstats");
num_xstats[port_id] = rte_eth_xstats_get_names(port_id, eth_xstat_names[port_id], num_xstats[port_id]);