X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Faf_xdp%2Frte_eth_af_xdp.c;h=b362ccdcd38c1bfe0ac1a1ebb6c27c39681f08dd;hb=53a80512644c8a12cb8efc903f77dd7b42263565;hp=a0edfc3cd392172817b2707319d8cf1b72035381;hpb=96d8ae9990c0a213255b4dc45c53368ea2c92bfa;p=dpdk.git diff --git a/drivers/net/af_xdp/rte_eth_af_xdp.c b/drivers/net/af_xdp/rte_eth_af_xdp.c index a0edfc3cd3..b362ccdcd3 100644 --- a/drivers/net/af_xdp/rte_eth_af_xdp.c +++ b/drivers/net/af_xdp/rte_eth_af_xdp.c @@ -1,11 +1,10 @@ /* SPDX-License-Identifier: BSD-3-Clause - * Copyright(c) 2019 Intel Corporation. + * Copyright(c) 2019-2020 Intel Corporation. */ #include #include #include #include -#include #include #include #include @@ -19,8 +18,8 @@ #include #include -#include -#include +#include +#include #include #include #include @@ -37,6 +36,18 @@ #include #include #include +#include +#include + +#include "compat.h" + +#ifndef SO_PREFER_BUSY_POLL +#define SO_PREFER_BUSY_POLL 69 +#endif +#ifndef SO_BUSY_POLL_BUDGET +#define SO_BUSY_POLL_BUDGET 70 +#endif + #ifndef SOL_XDP #define SOL_XDP 283 @@ -50,7 +61,7 @@ #define PF_XDP AF_XDP #endif -static int af_xdp_logtype; +RTE_LOG_REGISTER_DEFAULT(af_xdp_logtype, NOTICE); #define AF_XDP_LOG(level, fmt, args...) \ rte_log(RTE_LOG_ ## level, af_xdp_logtype, \ @@ -58,29 +69,24 @@ static int af_xdp_logtype; #define ETH_AF_XDP_FRAME_SIZE 2048 #define ETH_AF_XDP_NUM_BUFFERS 4096 -#ifdef XDP_UMEM_UNALIGNED_CHUNK_FLAG -#define ETH_AF_XDP_MBUF_OVERHEAD 128 /* sizeof(struct rte_mbuf) */ -#define ETH_AF_XDP_DATA_HEADROOM \ - (ETH_AF_XDP_MBUF_OVERHEAD + RTE_PKTMBUF_HEADROOM) -#else -#define ETH_AF_XDP_DATA_HEADROOM 0 -#endif #define ETH_AF_XDP_DFLT_NUM_DESCS XSK_RING_CONS__DEFAULT_NUM_DESCS #define ETH_AF_XDP_DFLT_START_QUEUE_IDX 0 #define ETH_AF_XDP_DFLT_QUEUE_COUNT 1 +#define ETH_AF_XDP_DFLT_BUSY_BUDGET 64 +#define ETH_AF_XDP_DFLT_BUSY_TIMEOUT 20 -#define ETH_AF_XDP_RX_BATCH_SIZE 32 -#define ETH_AF_XDP_TX_BATCH_SIZE 32 +#define ETH_AF_XDP_RX_BATCH_SIZE XSK_RING_CONS__DEFAULT_NUM_DESCS +#define ETH_AF_XDP_TX_BATCH_SIZE XSK_RING_CONS__DEFAULT_NUM_DESCS struct xsk_umem_info { - struct xsk_ring_prod fq; - struct xsk_ring_cons cq; struct xsk_umem *umem; struct rte_ring *buf_ring; const struct rte_memzone *mz; struct rte_mempool *mb_pool; void *buffer; + uint8_t refcnt; + uint32_t max_xsks; }; struct rx_stats { @@ -97,9 +103,13 @@ struct pkt_rx_queue { struct rx_stats stats; + struct xsk_ring_prod fq; + struct xsk_ring_cons cq; + struct pkt_tx_queue *pair; struct pollfd fds[1]; int xsk_queue_idx; + int busy_budget; }; struct tx_stats { @@ -125,6 +135,9 @@ struct pmd_internals { int queue_cnt; int max_queue_cnt; int combined_queue_cnt; + bool shared_umem; + char prog_path[PATH_MAX]; + bool custom_prog_configured; struct rte_ether_addr eth_addr; @@ -135,11 +148,17 @@ struct pmd_internals { #define ETH_AF_XDP_IFACE_ARG "iface" #define ETH_AF_XDP_START_QUEUE_ARG "start_queue" #define ETH_AF_XDP_QUEUE_COUNT_ARG "queue_count" +#define ETH_AF_XDP_SHARED_UMEM_ARG "shared_umem" +#define ETH_AF_XDP_PROG_ARG "xdp_prog" +#define ETH_AF_XDP_BUDGET_ARG "busy_budget" static const char * const valid_arguments[] = { ETH_AF_XDP_IFACE_ARG, ETH_AF_XDP_START_QUEUE_ARG, ETH_AF_XDP_QUEUE_COUNT_ARG, + ETH_AF_XDP_SHARED_UMEM_ARG, + ETH_AF_XDP_PROG_ARG, + ETH_AF_XDP_BUDGET_ARG, NULL }; @@ -150,12 +169,23 @@ static const struct rte_eth_link pmd_link = { .link_autoneg = ETH_LINK_AUTONEG }; +/* List which tracks PMDs to facilitate sharing UMEMs across them. */ +struct internal_list { + TAILQ_ENTRY(internal_list) next; + struct rte_eth_dev *eth_dev; +}; + +TAILQ_HEAD(internal_list_head, internal_list); +static struct internal_list_head internal_list = + TAILQ_HEAD_INITIALIZER(internal_list); + +static pthread_mutex_t internal_list_lock = PTHREAD_MUTEX_INITIALIZER; + #if defined(XDP_UMEM_UNALIGNED_CHUNK_FLAG) static inline int reserve_fill_queue_zc(struct xsk_umem_info *umem, uint16_t reserve_size, - struct rte_mbuf **bufs) + struct rte_mbuf **bufs, struct xsk_ring_prod *fq) { - struct xsk_ring_prod *fq = &umem->fq; uint32_t idx; uint16_t i; @@ -183,9 +213,9 @@ reserve_fill_queue_zc(struct xsk_umem_info *umem, uint16_t reserve_size, #else static inline int reserve_fill_queue_cp(struct xsk_umem_info *umem, uint16_t reserve_size, - struct rte_mbuf **bufs __rte_unused) + struct rte_mbuf **bufs __rte_unused, + struct xsk_ring_prod *fq) { - struct xsk_ring_prod *fq = &umem->fq; void *addrs[reserve_size]; uint32_t idx; uint16_t i; @@ -218,12 +248,12 @@ reserve_fill_queue_cp(struct xsk_umem_info *umem, uint16_t reserve_size, static inline int reserve_fill_queue(struct xsk_umem_info *umem, uint16_t reserve_size, - struct rte_mbuf **bufs) + struct rte_mbuf **bufs, struct xsk_ring_prod *fq) { #if defined(XDP_UMEM_UNALIGNED_CHUNK_FLAG) - return reserve_fill_queue_zc(umem, reserve_size, bufs); + return reserve_fill_queue_zc(umem, reserve_size, bufs, fq); #else - return reserve_fill_queue_cp(umem, reserve_size, bufs); + return reserve_fill_queue_cp(umem, reserve_size, bufs, fq); #endif } @@ -233,31 +263,43 @@ af_xdp_rx_zc(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) { struct pkt_rx_queue *rxq = queue; struct xsk_ring_cons *rx = &rxq->rx; + struct xsk_ring_prod *fq = &rxq->fq; struct xsk_umem_info *umem = rxq->umem; uint32_t idx_rx = 0; unsigned long rx_bytes = 0; - int rcvd, i; + int i; struct rte_mbuf *fq_bufs[ETH_AF_XDP_RX_BATCH_SIZE]; + nb_pkts = xsk_ring_cons__peek(rx, nb_pkts, &idx_rx); + + if (nb_pkts == 0) { + /* we can assume a kernel >= 5.11 is in use if busy polling is + * enabled and thus we can safely use the recvfrom() syscall + * which is only supported for AF_XDP sockets in kernels >= + * 5.11. + */ + if (rxq->busy_budget) { + (void)recvfrom(xsk_socket__fd(rxq->xsk), NULL, 0, + MSG_DONTWAIT, NULL, NULL); + } else if (xsk_ring_prod__needs_wakeup(fq)) { + (void)poll(&rxq->fds[0], 1, 1000); + } + + return 0; + } + /* allocate bufs for fill queue replenishment after rx */ if (rte_pktmbuf_alloc_bulk(umem->mb_pool, fq_bufs, nb_pkts)) { AF_XDP_LOG(DEBUG, "Failed to get enough buffers for fq.\n"); - return -1; - } - - rcvd = xsk_ring_cons__peek(rx, nb_pkts, &idx_rx); - - if (rcvd == 0) { -#if defined(XDP_USE_NEED_WAKEUP) - if (xsk_ring_prod__needs_wakeup(&umem->fq)) - (void)poll(rxq->fds, 1, 1000); -#endif - - goto out; + /* rollback cached_cons which is added by + * xsk_ring_cons__peek + */ + rx->cached_cons -= nb_pkts; + return 0; } - for (i = 0; i < rcvd; i++) { + for (i = 0; i < nb_pkts; i++) { const struct xdp_desc *desc; uint64_t addr; uint32_t len; @@ -282,20 +324,14 @@ af_xdp_rx_zc(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) rx_bytes += len; } - xsk_ring_cons__release(rx, rcvd); - - (void)reserve_fill_queue(umem, rcvd, fq_bufs); + xsk_ring_cons__release(rx, nb_pkts); + (void)reserve_fill_queue(umem, nb_pkts, fq_bufs, fq); /* statistics */ - rxq->stats.rx_pkts += rcvd; + rxq->stats.rx_pkts += nb_pkts; rxq->stats.rx_bytes += rx_bytes; -out: - if (rcvd != nb_pkts) - rte_mempool_put_bulk(umem->mb_pool, (void **)&fq_bufs[rcvd], - nb_pkts - rcvd); - - return rcvd; + return nb_pkts; } #else static uint16_t @@ -304,30 +340,34 @@ af_xdp_rx_cp(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) struct pkt_rx_queue *rxq = queue; struct xsk_ring_cons *rx = &rxq->rx; struct xsk_umem_info *umem = rxq->umem; - struct xsk_ring_prod *fq = &umem->fq; + struct xsk_ring_prod *fq = &rxq->fq; uint32_t idx_rx = 0; unsigned long rx_bytes = 0; - int rcvd, i; + int i; uint32_t free_thresh = fq->size >> 1; struct rte_mbuf *mbufs[ETH_AF_XDP_RX_BATCH_SIZE]; - if (unlikely(rte_pktmbuf_alloc_bulk(rxq->mb_pool, mbufs, nb_pkts) != 0)) - return 0; + if (xsk_prod_nb_free(fq, free_thresh) >= free_thresh) + (void)reserve_fill_queue(umem, nb_pkts, NULL, fq); - rcvd = xsk_ring_cons__peek(rx, nb_pkts, &idx_rx); - if (rcvd == 0) { + nb_pkts = xsk_ring_cons__peek(rx, nb_pkts, &idx_rx); + if (nb_pkts == 0) { #if defined(XDP_USE_NEED_WAKEUP) if (xsk_ring_prod__needs_wakeup(fq)) (void)poll(rxq->fds, 1, 1000); #endif - - goto out; + return 0; } - if (xsk_prod_nb_free(fq, free_thresh) >= free_thresh) - (void)reserve_fill_queue(umem, ETH_AF_XDP_RX_BATCH_SIZE, NULL); + if (unlikely(rte_pktmbuf_alloc_bulk(rxq->mb_pool, mbufs, nb_pkts))) { + /* rollback cached_cons which is added by + * xsk_ring_cons__peek + */ + rx->cached_cons -= nb_pkts; + return 0; + } - for (i = 0; i < rcvd; i++) { + for (i = 0; i < nb_pkts; i++) { const struct xdp_desc *desc; uint64_t addr; uint32_t len; @@ -346,26 +386,19 @@ af_xdp_rx_cp(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) bufs[i] = mbufs[i]; } - xsk_ring_cons__release(rx, rcvd); + xsk_ring_cons__release(rx, nb_pkts); /* statistics */ - rxq->stats.rx_pkts += rcvd; + rxq->stats.rx_pkts += nb_pkts; rxq->stats.rx_bytes += rx_bytes; -out: - if (rcvd != nb_pkts) - rte_mempool_put_bulk(rxq->mb_pool, (void **)&mbufs[rcvd], - nb_pkts - rcvd); - - return rcvd; + return nb_pkts; } #endif static uint16_t -eth_af_xdp_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) +af_xdp_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) { - nb_pkts = RTE_MIN(nb_pkts, ETH_AF_XDP_RX_BATCH_SIZE); - #if defined(XDP_UMEM_UNALIGNED_CHUNK_FLAG) return af_xdp_rx_zc(queue, bufs, nb_pkts); #else @@ -373,10 +406,35 @@ eth_af_xdp_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) #endif } +static uint16_t +eth_af_xdp_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) +{ + uint16_t nb_rx; + + if (likely(nb_pkts <= ETH_AF_XDP_RX_BATCH_SIZE)) + return af_xdp_rx(queue, bufs, nb_pkts); + + /* Split larger batch into smaller batches of size + * ETH_AF_XDP_RX_BATCH_SIZE or less. + */ + nb_rx = 0; + while (nb_pkts) { + uint16_t ret, n; + + n = (uint16_t)RTE_MIN(nb_pkts, ETH_AF_XDP_RX_BATCH_SIZE); + ret = af_xdp_rx(queue, &bufs[nb_rx], n); + nb_rx = (uint16_t)(nb_rx + ret); + nb_pkts = (uint16_t)(nb_pkts - ret); + if (ret < n) + break; + } + + return nb_rx; +} + static void -pull_umem_cq(struct xsk_umem_info *umem, int size) +pull_umem_cq(struct xsk_umem_info *umem, int size, struct xsk_ring_cons *cq) { - struct xsk_ring_cons *cq = &umem->cq; size_t i, n; uint32_t idx_cq = 0; @@ -399,13 +457,13 @@ pull_umem_cq(struct xsk_umem_info *umem, int size) } static void -kick_tx(struct pkt_tx_queue *txq) +kick_tx(struct pkt_tx_queue *txq, struct xsk_ring_cons *cq) { struct xsk_umem_info *umem = txq->umem; -#if defined(XDP_USE_NEED_WAKEUP) - if (xsk_ring_prod__needs_wakeup(&txq->tx)) -#endif + pull_umem_cq(umem, XSK_RING_CONS__DEFAULT_NUM_DESCS, cq); + + if (tx_syscall_needed(&txq->tx)) while (send(xsk_socket__fd(txq->pair->xsk), NULL, 0, MSG_DONTWAIT) < 0) { /* some thing unexpected */ @@ -414,11 +472,10 @@ kick_tx(struct pkt_tx_queue *txq) /* pull from completion queue to leave more space */ if (errno == EAGAIN) - pull_umem_cq(umem, ETH_AF_XDP_TX_BATCH_SIZE); + pull_umem_cq(umem, + XSK_RING_CONS__DEFAULT_NUM_DESCS, + cq); } -#ifndef XDP_UMEM_UNALIGNED_CHUNK_FLAG - pull_umem_cq(umem, ETH_AF_XDP_TX_BATCH_SIZE); -#endif } #if defined(XDP_UMEM_UNALIGNED_CHUNK_FLAG) @@ -434,16 +491,21 @@ af_xdp_tx_zc(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) uint16_t count = 0; struct xdp_desc *desc; uint64_t addr, offset; + struct xsk_ring_cons *cq = &txq->pair->cq; + uint32_t free_thresh = cq->size >> 1; - pull_umem_cq(umem, nb_pkts); + if (xsk_cons_nb_avail(cq, free_thresh) >= free_thresh) + pull_umem_cq(umem, XSK_RING_CONS__DEFAULT_NUM_DESCS, cq); for (i = 0; i < nb_pkts; i++) { mbuf = bufs[i]; if (mbuf->pool == umem->mb_pool) { if (!xsk_ring_prod__reserve(&txq->tx, 1, &idx_tx)) { - kick_tx(txq); - goto out; + kick_tx(txq, cq); + if (!xsk_ring_prod__reserve(&txq->tx, 1, + &idx_tx)) + goto out; } desc = xsk_ring_prod__tx_desc(&txq->tx, idx_tx); desc->len = mbuf->pkt_len; @@ -465,7 +527,6 @@ af_xdp_tx_zc(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) if (!xsk_ring_prod__reserve(&txq->tx, 1, &idx_tx)) { rte_pktmbuf_free(local_mbuf); - kick_tx(txq); goto out; } @@ -489,10 +550,9 @@ af_xdp_tx_zc(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) tx_bytes += mbuf->pkt_len; } - kick_tx(txq); - out: xsk_ring_prod__submit(&txq->tx, count); + kick_tx(txq, cq); txq->stats.tx_pkts += count; txq->stats.tx_bytes += tx_bytes; @@ -511,10 +571,9 @@ af_xdp_tx_cp(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) unsigned long tx_bytes = 0; int i; uint32_t idx_tx; + struct xsk_ring_cons *cq = &txq->pair->cq; - nb_pkts = RTE_MIN(nb_pkts, ETH_AF_XDP_TX_BATCH_SIZE); - - pull_umem_cq(umem, nb_pkts); + pull_umem_cq(umem, nb_pkts, cq); nb_pkts = rte_ring_dequeue_bulk(umem->buf_ring, addrs, nb_pkts, NULL); @@ -522,7 +581,7 @@ af_xdp_tx_cp(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) return 0; if (xsk_ring_prod__reserve(&txq->tx, nb_pkts, &idx_tx) != nb_pkts) { - kick_tx(txq); + kick_tx(txq, cq); rte_ring_enqueue_bulk(umem->buf_ring, addrs, nb_pkts, NULL); return 0; } @@ -545,13 +604,39 @@ af_xdp_tx_cp(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) xsk_ring_prod__submit(&txq->tx, nb_pkts); - kick_tx(txq); + kick_tx(txq, cq); txq->stats.tx_pkts += nb_pkts; txq->stats.tx_bytes += tx_bytes; return nb_pkts; } + +static uint16_t +af_xdp_tx_cp_batch(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) +{ + uint16_t nb_tx; + + if (likely(nb_pkts <= ETH_AF_XDP_TX_BATCH_SIZE)) + return af_xdp_tx_cp(queue, bufs, nb_pkts); + + nb_tx = 0; + while (nb_pkts) { + uint16_t ret, n; + + /* Split larger batch into smaller batches of size + * ETH_AF_XDP_TX_BATCH_SIZE or less. + */ + n = (uint16_t)RTE_MIN(nb_pkts, ETH_AF_XDP_TX_BATCH_SIZE); + ret = af_xdp_tx_cp(queue, &bufs[nb_tx], n); + nb_tx = (uint16_t)(nb_tx + ret); + nb_pkts = (uint16_t)(nb_pkts - ret); + if (ret < n) + break; + } + + return nb_tx; +} #endif static uint16_t @@ -560,7 +645,7 @@ eth_af_xdp_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) #if defined(XDP_UMEM_UNALIGNED_CHUNK_FLAG) return af_xdp_tx_zc(queue, bufs, nb_pkts); #else - return af_xdp_tx_cp(queue, bufs, nb_pkts); + return af_xdp_tx_cp_batch(queue, bufs, nb_pkts); #endif } @@ -573,19 +658,164 @@ eth_dev_start(struct rte_eth_dev *dev) } /* This function gets called when the current port gets stopped. */ -static void +static int eth_dev_stop(struct rte_eth_dev *dev) { dev->data->dev_link.link_status = ETH_LINK_DOWN; + return 0; +} + +/* Find ethdev in list */ +static inline struct internal_list * +find_internal_resource(struct pmd_internals *port_int) +{ + int found = 0; + struct internal_list *list = NULL; + + if (port_int == NULL) + return NULL; + + pthread_mutex_lock(&internal_list_lock); + + TAILQ_FOREACH(list, &internal_list, next) { + struct pmd_internals *list_int = + list->eth_dev->data->dev_private; + if (list_int == port_int) { + found = 1; + break; + } + } + + pthread_mutex_unlock(&internal_list_lock); + + if (!found) + return NULL; + + return list; +} + +/* Check if the netdev,qid context already exists */ +static inline bool +ctx_exists(struct pkt_rx_queue *rxq, const char *ifname, + struct pkt_rx_queue *list_rxq, const char *list_ifname) +{ + bool exists = false; + + if (rxq->xsk_queue_idx == list_rxq->xsk_queue_idx && + !strncmp(ifname, list_ifname, IFNAMSIZ)) { + AF_XDP_LOG(ERR, "ctx %s,%i already exists, cannot share umem\n", + ifname, rxq->xsk_queue_idx); + exists = true; + } + + return exists; +} + +/* Get a pointer to an existing UMEM which overlays the rxq's mb_pool */ +static inline int +get_shared_umem(struct pkt_rx_queue *rxq, const char *ifname, + struct xsk_umem_info **umem) +{ + struct internal_list *list; + struct pmd_internals *internals; + int i = 0, ret = 0; + struct rte_mempool *mb_pool = rxq->mb_pool; + + if (mb_pool == NULL) + return ret; + + pthread_mutex_lock(&internal_list_lock); + + TAILQ_FOREACH(list, &internal_list, next) { + internals = list->eth_dev->data->dev_private; + for (i = 0; i < internals->queue_cnt; i++) { + struct pkt_rx_queue *list_rxq = + &internals->rx_queues[i]; + if (rxq == list_rxq) + continue; + if (mb_pool == internals->rx_queues[i].mb_pool) { + if (ctx_exists(rxq, ifname, list_rxq, + internals->if_name)) { + ret = -1; + goto out; + } + if (__atomic_load_n( + &internals->rx_queues[i].umem->refcnt, + __ATOMIC_ACQUIRE)) { + *umem = internals->rx_queues[i].umem; + goto out; + } + } + } + } + +out: + pthread_mutex_unlock(&internal_list_lock); + + return ret; } static int eth_dev_configure(struct rte_eth_dev *dev) { + struct pmd_internals *internal = dev->data->dev_private; + /* rx/tx must be paired */ if (dev->data->nb_rx_queues != dev->data->nb_tx_queues) return -EINVAL; + if (internal->shared_umem) { + struct internal_list *list = NULL; + const char *name = dev->device->name; + + /* Ensure PMD is not already inserted into the list */ + list = find_internal_resource(internal); + if (list) + return 0; + + list = rte_zmalloc_socket(name, sizeof(*list), 0, + dev->device->numa_node); + if (list == NULL) + return -1; + + list->eth_dev = dev; + pthread_mutex_lock(&internal_list_lock); + TAILQ_INSERT_TAIL(&internal_list, list, next); + pthread_mutex_unlock(&internal_list_lock); + } + + return 0; +} + +#define CLB_VAL_IDX 0 +static int +eth_monitor_callback(const uint64_t value, + const uint64_t opaque[RTE_POWER_MONITOR_OPAQUE_SZ]) +{ + const uint64_t v = opaque[CLB_VAL_IDX]; + const uint64_t m = (uint32_t)~0; + + /* if the value has changed, abort entering power optimized state */ + return (value & m) == v ? 0 : -1; +} + +static int +eth_get_monitor_addr(void *rx_queue, struct rte_power_monitor_cond *pmc) +{ + struct pkt_rx_queue *rxq = rx_queue; + unsigned int *prod = rxq->rx.producer; + const uint32_t cur_val = rxq->rx.cached_prod; /* use cached value */ + + /* watch for changes in producer ring */ + pmc->addr = (void *)prod; + + /* store current value */ + pmc->opaque[CLB_VAL_IDX] = cur_val; + pmc->fn = eth_monitor_callback; + + /* AF_XDP producer ring index is 32-bit */ + pmc->size = sizeof(uint32_t); + return 0; } @@ -601,8 +831,17 @@ eth_dev_info(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) dev_info->max_tx_queues = internals->queue_cnt; dev_info->min_mtu = RTE_ETHER_MIN_MTU; - dev_info->max_mtu = ETH_AF_XDP_FRAME_SIZE - ETH_AF_XDP_DATA_HEADROOM; +#if defined(XDP_UMEM_UNALIGNED_CHUNK_FLAG) + dev_info->max_mtu = getpagesize() - + sizeof(struct rte_mempool_objhdr) - + sizeof(struct rte_mbuf) - + RTE_PKTMBUF_HEADROOM - XDP_PACKET_HEADROOM; +#else + dev_info->max_mtu = ETH_AF_XDP_FRAME_SIZE - XDP_PACKET_HEADROOM; +#endif + dev_info->default_rxportconf.burst_size = ETH_AF_XDP_DFLT_BUSY_BUDGET; + dev_info->default_txportconf.burst_size = ETH_AF_XDP_DFLT_BUSY_BUDGET; dev_info->default_rxportconf.nb_queues = 1; dev_info->default_txportconf.nb_queues = 1; dev_info->default_rxportconf.ring_size = ETH_AF_XDP_DFLT_NUM_DESCS; @@ -684,7 +923,6 @@ static void xdp_umem_destroy(struct xsk_umem_info *umem) { #if defined(XDP_UMEM_UNALIGNED_CHUNK_FLAG) - rte_mempool_free(umem->mb_pool); umem->mb_pool = NULL; #else rte_memzone_free(umem->mz); @@ -695,16 +933,18 @@ xdp_umem_destroy(struct xsk_umem_info *umem) #endif rte_free(umem); - umem = NULL; } -static void +static int eth_dev_close(struct rte_eth_dev *dev) { struct pmd_internals *internals = dev->data->dev_private; struct pkt_rx_queue *rxq; int i; + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return 0; + AF_XDP_LOG(INFO, "Closing AF_XDP ethdev on numa socket %u\n", rte_socket_id()); @@ -713,8 +953,12 @@ eth_dev_close(struct rte_eth_dev *dev) if (rxq->umem == NULL) break; xsk_socket__delete(rxq->xsk); - (void)xsk_umem__delete(rxq->umem->umem); - xdp_umem_destroy(rxq->umem); + + if (__atomic_sub_fetch(&rxq->umem->refcnt, 1, __ATOMIC_ACQUIRE) + == 0) { + (void)xsk_umem__delete(rxq->umem->umem); + xdp_umem_destroy(rxq->umem); + } /* free pkt_tx_queue */ rte_free(rxq->pair); @@ -728,11 +972,21 @@ eth_dev_close(struct rte_eth_dev *dev) dev->data->mac_addrs = NULL; remove_xdp_program(internals); -} -static void -eth_queue_release(void *q __rte_unused) -{ + if (internals->shared_umem) { + struct internal_list *list; + + /* Remove ethdev from list used to track and share UMEMs */ + list = find_internal_resource(internals); + if (list) { + pthread_mutex_lock(&internal_list_lock); + TAILQ_REMOVE(&internal_list, list, next); + pthread_mutex_unlock(&internal_list_lock); + rte_free(list); + } + } + + return 0; } static int @@ -743,54 +997,85 @@ eth_link_update(struct rte_eth_dev *dev __rte_unused, } #if defined(XDP_UMEM_UNALIGNED_CHUNK_FLAG) -static inline uint64_t get_base_addr(struct rte_mempool *mp) +static inline uintptr_t get_base_addr(struct rte_mempool *mp, uint64_t *align) { struct rte_mempool_memhdr *memhdr; + uintptr_t memhdr_addr, aligned_addr; memhdr = STAILQ_FIRST(&mp->mem_list); - return (uint64_t)memhdr->addr & ~(getpagesize() - 1); + memhdr_addr = (uintptr_t)memhdr->addr; + aligned_addr = memhdr_addr & ~(getpagesize() - 1); + *align = memhdr_addr - aligned_addr; + + return aligned_addr; } static struct -xsk_umem_info *xdp_umem_configure(struct pmd_internals *internals __rte_unused, +xsk_umem_info *xdp_umem_configure(struct pmd_internals *internals, struct pkt_rx_queue *rxq) { - struct xsk_umem_info *umem; + struct xsk_umem_info *umem = NULL; int ret; struct xsk_umem_config usr_config = { - .fill_size = ETH_AF_XDP_DFLT_NUM_DESCS, + .fill_size = ETH_AF_XDP_DFLT_NUM_DESCS * 2, .comp_size = ETH_AF_XDP_DFLT_NUM_DESCS, .flags = XDP_UMEM_UNALIGNED_CHUNK_FLAG}; void *base_addr = NULL; struct rte_mempool *mb_pool = rxq->mb_pool; + uint64_t umem_size, align = 0; + + if (internals->shared_umem) { + if (get_shared_umem(rxq, internals->if_name, &umem) < 0) + return NULL; + + if (umem != NULL && + __atomic_load_n(&umem->refcnt, __ATOMIC_ACQUIRE) < + umem->max_xsks) { + AF_XDP_LOG(INFO, "%s,qid%i sharing UMEM\n", + internals->if_name, rxq->xsk_queue_idx); + __atomic_fetch_add(&umem->refcnt, 1, __ATOMIC_ACQUIRE); + } + } - usr_config.frame_size = rte_mempool_calc_obj_size(mb_pool->elt_size, - mb_pool->flags, - NULL); - usr_config.frame_headroom = mb_pool->header_size + - sizeof(struct rte_mbuf) + - rte_pktmbuf_priv_size(mb_pool) + - RTE_PKTMBUF_HEADROOM; - - umem = rte_zmalloc_socket("umem", sizeof(*umem), 0, rte_socket_id()); if (umem == NULL) { - AF_XDP_LOG(ERR, "Failed to allocate umem info"); - return NULL; - } + usr_config.frame_size = + rte_mempool_calc_obj_size(mb_pool->elt_size, + mb_pool->flags, NULL); + usr_config.frame_headroom = mb_pool->header_size + + sizeof(struct rte_mbuf) + + rte_pktmbuf_priv_size(mb_pool) + + RTE_PKTMBUF_HEADROOM; + + umem = rte_zmalloc_socket("umem", sizeof(*umem), 0, + rte_socket_id()); + if (umem == NULL) { + AF_XDP_LOG(ERR, "Failed to allocate umem info"); + return NULL; + } - umem->mb_pool = mb_pool; - base_addr = (void *)get_base_addr(mb_pool); + umem->mb_pool = mb_pool; + base_addr = (void *)get_base_addr(mb_pool, &align); + umem_size = (uint64_t)mb_pool->populated_size * + (uint64_t)usr_config.frame_size + + align; + + ret = xsk_umem__create(&umem->umem, base_addr, umem_size, + &rxq->fq, &rxq->cq, &usr_config); + if (ret) { + AF_XDP_LOG(ERR, "Failed to create umem"); + goto err; + } + umem->buffer = base_addr; - ret = xsk_umem__create(&umem->umem, base_addr, - mb_pool->populated_size * usr_config.frame_size, - &umem->fq, &umem->cq, - &usr_config); + if (internals->shared_umem) { + umem->max_xsks = mb_pool->populated_size / + ETH_AF_XDP_NUM_BUFFERS; + AF_XDP_LOG(INFO, "Max xsks for UMEM %s: %u\n", + mb_pool->name, umem->max_xsks); + } - if (ret) { - AF_XDP_LOG(ERR, "Failed to create umem"); - goto err; + __atomic_store_n(&umem->refcnt, 1, __ATOMIC_RELEASE); } - umem->buffer = base_addr; #else static struct @@ -803,7 +1088,7 @@ xsk_umem_info *xdp_umem_configure(struct pmd_internals *internals, .fill_size = ETH_AF_XDP_DFLT_NUM_DESCS, .comp_size = ETH_AF_XDP_DFLT_NUM_DESCS, .frame_size = ETH_AF_XDP_FRAME_SIZE, - .frame_headroom = ETH_AF_XDP_DATA_HEADROOM }; + .frame_headroom = 0 }; char ring_name[RTE_RING_NAMESIZE]; char mz_name[RTE_MEMZONE_NAMESIZE]; int ret; @@ -820,7 +1105,7 @@ xsk_umem_info *xdp_umem_configure(struct pmd_internals *internals, umem->buf_ring = rte_ring_create(ring_name, ETH_AF_XDP_NUM_BUFFERS, rte_socket_id(), - RING_F_SP_ENQ | RING_F_SC_DEQ); + 0x0); if (umem->buf_ring == NULL) { AF_XDP_LOG(ERR, "Failed to create rte_ring\n"); goto err; @@ -828,8 +1113,7 @@ xsk_umem_info *xdp_umem_configure(struct pmd_internals *internals, for (i = 0; i < ETH_AF_XDP_NUM_BUFFERS; i++) rte_ring_enqueue(umem->buf_ring, - (void *)(i * ETH_AF_XDP_FRAME_SIZE + - ETH_AF_XDP_DATA_HEADROOM)); + (void *)(i * ETH_AF_XDP_FRAME_SIZE)); snprintf(mz_name, sizeof(mz_name), "af_xdp_umem_%s_%u", internals->if_name, rxq->xsk_queue_idx); @@ -844,7 +1128,7 @@ xsk_umem_info *xdp_umem_configure(struct pmd_internals *internals, ret = xsk_umem__create(&umem->umem, mz->addr, ETH_AF_XDP_NUM_BUFFERS * ETH_AF_XDP_FRAME_SIZE, - &umem->fq, &umem->cq, + &rxq->fq, &rxq->cq, &usr_config); if (ret) { @@ -861,6 +1145,104 @@ err: return NULL; } +static int +load_custom_xdp_prog(const char *prog_path, int if_index) +{ + int ret, prog_fd = -1; + struct bpf_object *obj; + struct bpf_map *map; + + ret = bpf_prog_load(prog_path, BPF_PROG_TYPE_XDP, &obj, &prog_fd); + if (ret) { + AF_XDP_LOG(ERR, "Failed to load program %s\n", prog_path); + return ret; + } + + /* + * The loaded program must provision for a map of xsks, such that some + * traffic can be redirected to userspace. When the xsk is created, + * libbpf inserts it into the map. + */ + map = bpf_object__find_map_by_name(obj, "xsks_map"); + if (!map) { + AF_XDP_LOG(ERR, "Failed to find xsks_map in %s\n", prog_path); + return -1; + } + + /* Link the program with the given network device */ + ret = bpf_set_link_xdp_fd(if_index, prog_fd, + XDP_FLAGS_UPDATE_IF_NOEXIST); + if (ret) { + AF_XDP_LOG(ERR, "Failed to set prog fd %d on interface\n", + prog_fd); + return -1; + } + + AF_XDP_LOG(INFO, "Successfully loaded XDP program %s with fd %d\n", + prog_path, prog_fd); + + return 0; +} + +/* Detect support for busy polling through setsockopt(). */ +static int +configure_preferred_busy_poll(struct pkt_rx_queue *rxq) +{ + int sock_opt = 1; + int fd = xsk_socket__fd(rxq->xsk); + int ret = 0; + + ret = setsockopt(fd, SOL_SOCKET, SO_PREFER_BUSY_POLL, + (void *)&sock_opt, sizeof(sock_opt)); + if (ret < 0) { + AF_XDP_LOG(DEBUG, "Failed to set SO_PREFER_BUSY_POLL\n"); + goto err_prefer; + } + + sock_opt = ETH_AF_XDP_DFLT_BUSY_TIMEOUT; + ret = setsockopt(fd, SOL_SOCKET, SO_BUSY_POLL, (void *)&sock_opt, + sizeof(sock_opt)); + if (ret < 0) { + AF_XDP_LOG(DEBUG, "Failed to set SO_BUSY_POLL\n"); + goto err_timeout; + } + + sock_opt = rxq->busy_budget; + ret = setsockopt(fd, SOL_SOCKET, SO_BUSY_POLL_BUDGET, + (void *)&sock_opt, sizeof(sock_opt)); + if (ret < 0) { + AF_XDP_LOG(DEBUG, "Failed to set SO_BUSY_POLL_BUDGET\n"); + } else { + AF_XDP_LOG(INFO, "Busy polling budget set to: %u\n", + rxq->busy_budget); + return 0; + } + + /* setsockopt failure - attempt to restore xsk to default state and + * proceed without busy polling support. + */ + sock_opt = 0; + ret = setsockopt(fd, SOL_SOCKET, SO_BUSY_POLL, (void *)&sock_opt, + sizeof(sock_opt)); + if (ret < 0) { + AF_XDP_LOG(ERR, "Failed to unset SO_BUSY_POLL\n"); + return -1; + } + +err_timeout: + sock_opt = 0; + ret = setsockopt(fd, SOL_SOCKET, SO_PREFER_BUSY_POLL, + (void *)&sock_opt, sizeof(sock_opt)); + if (ret < 0) { + AF_XDP_LOG(ERR, "Failed to unset SO_PREFER_BUSY_POLL\n"); + return -1; + } + +err_prefer: + rxq->busy_budget = 0; + return 0; +} + static int xsk_configure(struct pmd_internals *internals, struct pkt_rx_queue *rxq, int ring_size) @@ -868,7 +1250,7 @@ xsk_configure(struct pmd_internals *internals, struct pkt_rx_queue *rxq, struct xsk_socket_config cfg; struct pkt_tx_queue *txq = rxq->pair; int ret = 0; - int reserve_size = ETH_AF_XDP_DFLT_NUM_DESCS / 2; + int reserve_size = ETH_AF_XDP_DFLT_NUM_DESCS; struct rte_mbuf *fq_bufs[reserve_size]; rxq->umem = xdp_umem_configure(internals, rxq); @@ -886,21 +1268,49 @@ xsk_configure(struct pmd_internals *internals, struct pkt_rx_queue *rxq, cfg.bind_flags |= XDP_USE_NEED_WAKEUP; #endif - ret = xsk_socket__create(&rxq->xsk, internals->if_name, - rxq->xsk_queue_idx, rxq->umem->umem, &rxq->rx, - &txq->tx, &cfg); + if (strnlen(internals->prog_path, PATH_MAX) && + !internals->custom_prog_configured) { + ret = load_custom_xdp_prog(internals->prog_path, + internals->if_index); + if (ret) { + AF_XDP_LOG(ERR, "Failed to load custom XDP program %s\n", + internals->prog_path); + goto err; + } + internals->custom_prog_configured = 1; + } + + if (internals->shared_umem) + ret = create_shared_socket(&rxq->xsk, internals->if_name, + rxq->xsk_queue_idx, rxq->umem->umem, &rxq->rx, + &txq->tx, &rxq->fq, &rxq->cq, &cfg); + else + ret = xsk_socket__create(&rxq->xsk, internals->if_name, + rxq->xsk_queue_idx, rxq->umem->umem, &rxq->rx, + &txq->tx, &cfg); + if (ret) { AF_XDP_LOG(ERR, "Failed to create xsk socket.\n"); goto err; } #if defined(XDP_UMEM_UNALIGNED_CHUNK_FLAG) - if (rte_pktmbuf_alloc_bulk(rxq->umem->mb_pool, fq_bufs, reserve_size)) { + ret = rte_pktmbuf_alloc_bulk(rxq->umem->mb_pool, fq_bufs, reserve_size); + if (ret) { AF_XDP_LOG(DEBUG, "Failed to get enough buffers for fq.\n"); goto err; } #endif - ret = reserve_fill_queue(rxq->umem, reserve_size, fq_bufs); + + if (rxq->busy_budget) { + ret = configure_preferred_busy_poll(rxq); + if (ret) { + AF_XDP_LOG(ERR, "Failed configure busy polling.\n"); + goto err; + } + } + + ret = reserve_fill_queue(rxq->umem, reserve_size, fq_bufs, &rxq->fq); if (ret) { xsk_socket__delete(rxq->xsk); AF_XDP_LOG(ERR, "Failed to reserve fill queue.\n"); @@ -910,7 +1320,8 @@ xsk_configure(struct pmd_internals *internals, struct pkt_rx_queue *rxq, return 0; err: - xdp_umem_destroy(rxq->umem); + if (__atomic_sub_fetch(&rxq->umem->refcnt, 1, __ATOMIC_ACQUIRE) == 0) + xdp_umem_destroy(rxq->umem); return ret; } @@ -938,7 +1349,7 @@ eth_rx_queue_setup(struct rte_eth_dev *dev, /* Now get the space available for data in the mbuf */ buf_size = rte_pktmbuf_data_room_size(mb_pool) - RTE_PKTMBUF_HEADROOM; - data_size = ETH_AF_XDP_FRAME_SIZE - ETH_AF_XDP_DATA_HEADROOM; + data_size = ETH_AF_XDP_FRAME_SIZE; if (data_size > buf_size) { AF_XDP_LOG(ERR, "%s: %d bytes will not fit in mbuf (%d bytes)\n", @@ -956,6 +1367,9 @@ eth_rx_queue_setup(struct rte_eth_dev *dev, goto err; } + if (!rxq->busy_budget) + AF_XDP_LOG(DEBUG, "Preferred busy polling not enabled\n"); + rxq->fds[0].fd = xsk_socket__fd(rxq->xsk); rxq->fds[0].events = POLLIN; @@ -1055,13 +1469,30 @@ static const struct eth_dev_ops ops = { .promiscuous_disable = eth_dev_promiscuous_disable, .rx_queue_setup = eth_rx_queue_setup, .tx_queue_setup = eth_tx_queue_setup, - .rx_queue_release = eth_queue_release, - .tx_queue_release = eth_queue_release, .link_update = eth_link_update, .stats_get = eth_stats_get, .stats_reset = eth_stats_reset, + .get_monitor_addr = eth_get_monitor_addr, }; +/** parse busy_budget argument */ +static int +parse_budget_arg(const char *key __rte_unused, + const char *value, void *extra_args) +{ + int *i = (int *)extra_args; + char *end; + + *i = strtol(value, &end, 10); + if (*i < 0 || *i > UINT16_MAX) { + AF_XDP_LOG(ERR, "Invalid busy_budget %i, must be >= 0 and <= %u\n", + *i, UINT16_MAX); + return -EINVAL; + } + + return 0; +} + /** parse integer from integer argument */ static int parse_integer_arg(const char *key __rte_unused, @@ -1097,6 +1528,30 @@ parse_name_arg(const char *key __rte_unused, return 0; } +/** parse xdp prog argument */ +static int +parse_prog_arg(const char *key __rte_unused, + const char *value, void *extra_args) +{ + char *path = extra_args; + + if (strnlen(value, PATH_MAX) == PATH_MAX) { + AF_XDP_LOG(ERR, "Invalid path %s, should be less than %u bytes.\n", + value, PATH_MAX); + return -EINVAL; + } + + if (access(value, F_OK) != 0) { + AF_XDP_LOG(ERR, "Error accessing %s: %s\n", + value, strerror(errno)); + return -EINVAL; + } + + strlcpy(path, value, PATH_MAX); + + return 0; +} + static int xdp_get_channels_info(const char *if_name, int *max_queues, int *combined_queues) @@ -1111,7 +1566,7 @@ xdp_get_channels_info(const char *if_name, int *max_queues, channels.cmd = ETHTOOL_GCHANNELS; ifr.ifr_data = (void *)&channels; - strncpy(ifr.ifr_name, if_name, IFNAMSIZ); + strlcpy(ifr.ifr_name, if_name, IFNAMSIZ); ret = ioctl(fd, SIOCETHTOOL, &ifr); if (ret) { if (errno == EOPNOTSUPP) { @@ -1140,7 +1595,8 @@ xdp_get_channels_info(const char *if_name, int *max_queues, static int parse_parameters(struct rte_kvargs *kvlist, char *if_name, int *start_queue, - int *queue_cnt) + int *queue_cnt, int *shared_umem, char *prog_path, + int *busy_budget) { int ret; @@ -1161,6 +1617,21 @@ parse_parameters(struct rte_kvargs *kvlist, char *if_name, int *start_queue, goto free_kvlist; } + ret = rte_kvargs_process(kvlist, ETH_AF_XDP_SHARED_UMEM_ARG, + &parse_integer_arg, shared_umem); + if (ret < 0) + goto free_kvlist; + + ret = rte_kvargs_process(kvlist, ETH_AF_XDP_PROG_ARG, + &parse_prog_arg, prog_path); + if (ret < 0) + goto free_kvlist; + + ret = rte_kvargs_process(kvlist, ETH_AF_XDP_BUDGET_ARG, + &parse_budget_arg, busy_budget); + if (ret < 0) + goto free_kvlist; + free_kvlist: rte_kvargs_free(kvlist); return ret; @@ -1198,7 +1669,8 @@ error: static struct rte_eth_dev * init_internals(struct rte_vdev_device *dev, const char *if_name, - int start_queue_idx, int queue_cnt) + int start_queue_idx, int queue_cnt, int shared_umem, + const char *prog_path, int busy_budget) { const char *name = rte_vdev_device_name(dev); const unsigned int numa_node = dev->device.numa_node; @@ -1214,6 +1686,17 @@ init_internals(struct rte_vdev_device *dev, const char *if_name, internals->start_queue_idx = start_queue_idx; internals->queue_cnt = queue_cnt; strlcpy(internals->if_name, if_name, IFNAMSIZ); + strlcpy(internals->prog_path, prog_path, PATH_MAX); + internals->custom_prog_configured = 0; + +#ifndef ETH_AF_XDP_SHARED_UMEM + if (shared_umem) { + AF_XDP_LOG(ERR, "Shared UMEM feature not available. " + "Check kernel and libbpf version\n"); + goto err_free_internals; + } +#endif + internals->shared_umem = shared_umem; if (xdp_get_channels_info(if_name, &internals->max_queue_cnt, &internals->combined_queue_cnt)) { @@ -1248,6 +1731,7 @@ init_internals(struct rte_vdev_device *dev, const char *if_name, internals->rx_queues[i].pair = &internals->tx_queues[i]; internals->rx_queues[i].xsk_queue_idx = start_queue_idx + i; internals->tx_queues[i].xsk_queue_idx = start_queue_idx + i; + internals->rx_queues[i].busy_budget = busy_budget; } ret = get_iface_info(if_name, &internals->eth_addr, @@ -1262,11 +1746,10 @@ init_internals(struct rte_vdev_device *dev, const char *if_name, eth_dev->data->dev_private = internals; eth_dev->data->dev_link = pmd_link; eth_dev->data->mac_addrs = &internals->eth_addr; + eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS; eth_dev->dev_ops = &ops; eth_dev->rx_pkt_burst = eth_af_xdp_rx; eth_dev->tx_pkt_burst = eth_af_xdp_tx; - /* Let rte_eth_dev_close() release the port resources. */ - eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE; #if defined(XDP_UMEM_UNALIGNED_CHUNK_FLAG) AF_XDP_LOG(INFO, "Zero copy between umem and mbuf enabled.\n"); @@ -1290,6 +1773,9 @@ rte_pmd_af_xdp_probe(struct rte_vdev_device *dev) char if_name[IFNAMSIZ] = {'\0'}; int xsk_start_queue_idx = ETH_AF_XDP_DFLT_START_QUEUE_IDX; int xsk_queue_cnt = ETH_AF_XDP_DFLT_QUEUE_COUNT; + int shared_umem = 0; + char prog_path[PATH_MAX] = {'\0'}; + int busy_budget = -1; struct rte_eth_dev *eth_dev = NULL; const char *name; @@ -1297,16 +1783,11 @@ rte_pmd_af_xdp_probe(struct rte_vdev_device *dev) rte_vdev_device_name(dev)); name = rte_vdev_device_name(dev); - if (rte_eal_process_type() == RTE_PROC_SECONDARY && - strlen(rte_vdev_device_args(dev)) == 0) { - eth_dev = rte_eth_dev_attach_secondary(name); - if (eth_dev == NULL) { - AF_XDP_LOG(ERR, "Failed to probe %s\n", name); - return -EINVAL; - } - eth_dev->dev_ops = &ops; - rte_eth_dev_probing_finish(eth_dev); - return 0; + if (rte_eal_process_type() == RTE_PROC_SECONDARY) { + AF_XDP_LOG(ERR, "Failed to probe %s. " + "AF_XDP PMD does not support secondary processes.\n", + name); + return -ENOTSUP; } kvlist = rte_kvargs_parse(rte_vdev_device_args(dev), valid_arguments); @@ -1319,7 +1800,8 @@ rte_pmd_af_xdp_probe(struct rte_vdev_device *dev) dev->device.numa_node = rte_socket_id(); if (parse_parameters(kvlist, if_name, &xsk_start_queue_idx, - &xsk_queue_cnt) < 0) { + &xsk_queue_cnt, &shared_umem, prog_path, + &busy_budget) < 0) { AF_XDP_LOG(ERR, "Invalid kvargs value\n"); return -EINVAL; } @@ -1329,8 +1811,12 @@ rte_pmd_af_xdp_probe(struct rte_vdev_device *dev) return -EINVAL; } + busy_budget = busy_budget == -1 ? ETH_AF_XDP_DFLT_BUSY_BUDGET : + busy_budget; + eth_dev = init_internals(dev, if_name, xsk_start_queue_idx, - xsk_queue_cnt); + xsk_queue_cnt, shared_umem, prog_path, + busy_budget); if (eth_dev == NULL) { AF_XDP_LOG(ERR, "Failed to init internals\n"); return -1; @@ -1373,11 +1859,7 @@ RTE_PMD_REGISTER_VDEV(net_af_xdp, pmd_af_xdp_drv); RTE_PMD_REGISTER_PARAM_STRING(net_af_xdp, "iface= " "start_queue= " - "queue_count= "); - -RTE_INIT(af_xdp_init_log) -{ - af_xdp_logtype = rte_log_register("pmd.net.af_xdp"); - if (af_xdp_logtype >= 0) - rte_log_set_level(af_xdp_logtype, RTE_LOG_NOTICE); -} + "queue_count= " + "shared_umem= " + "xdp_prog= " + "busy_budget=");