#include <linux/ethtool.h>
#include <linux/sockios.h>
#include "af_xdp_deps.h"
-#include <bpf/bpf.h>
-#include <bpf/xsk.h>
#include <rte_ethdev.h>
#include <ethdev_driver.h>
#define ETH_AF_XDP_ETH_OVERHEAD (RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN)
+#define ETH_AF_XDP_MP_KEY "afxdp_mp_send_fds"
+
+static int afxdp_dev_count;
+
+/* Message header to synchronize fds via IPC */
+struct ipc_hdr {
+ char port_name[RTE_DEV_NAME_MAX_LEN];
+ /* The file descriptors are in the dedicated part
+ * of the Unix message to be translated by the kernel.
+ */
+};
+
struct xsk_umem_info {
struct xsk_umem *umem;
struct rte_ring *buf_ring;
struct pkt_tx_queue *tx_queues;
};
+struct pmd_process_private {
+ int rxq_xsk_fds[RTE_MAX_QUEUES_PER_PORT];
+};
+
#define ETH_AF_XDP_IFACE_ARG "iface"
#define ETH_AF_XDP_START_QUEUE_ARG "start_queue"
#define ETH_AF_XDP_QUEUE_COUNT_ARG "queue_count"
return list;
}
-/* Check if the netdev,qid context already exists */
-static inline bool
-ctx_exists(struct pkt_rx_queue *rxq, const char *ifname,
- struct pkt_rx_queue *list_rxq, const char *list_ifname)
-{
- bool exists = false;
-
- if (rxq->xsk_queue_idx == list_rxq->xsk_queue_idx &&
- !strncmp(ifname, list_ifname, IFNAMSIZ)) {
- AF_XDP_LOG(ERR, "ctx %s,%i already exists, cannot share umem\n",
- ifname, rxq->xsk_queue_idx);
- exists = true;
- }
-
- return exists;
-}
-
-/* Get a pointer to an existing UMEM which overlays the rxq's mb_pool */
-static inline int
-get_shared_umem(struct pkt_rx_queue *rxq, const char *ifname,
- struct xsk_umem_info **umem)
-{
- struct internal_list *list;
- struct pmd_internals *internals;
- int i = 0, ret = 0;
- struct rte_mempool *mb_pool = rxq->mb_pool;
-
- if (mb_pool == NULL)
- return ret;
-
- pthread_mutex_lock(&internal_list_lock);
-
- TAILQ_FOREACH(list, &internal_list, next) {
- internals = list->eth_dev->data->dev_private;
- for (i = 0; i < internals->queue_cnt; i++) {
- struct pkt_rx_queue *list_rxq =
- &internals->rx_queues[i];
- if (rxq == list_rxq)
- continue;
- if (mb_pool == internals->rx_queues[i].mb_pool) {
- if (ctx_exists(rxq, ifname, list_rxq,
- internals->if_name)) {
- ret = -1;
- goto out;
- }
- if (__atomic_load_n(
- &internals->rx_queues[i].umem->refcnt,
- __ATOMIC_ACQUIRE)) {
- *umem = internals->rx_queues[i].umem;
- goto out;
- }
- }
- }
- }
-
-out:
- pthread_mutex_unlock(&internal_list_lock);
-
- return ret;
-}
-
static int
eth_dev_configure(struct rte_eth_dev *dev)
{
eth_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
{
struct pmd_internals *internals = dev->data->dev_private;
+ struct pmd_process_private *process_private = dev->process_private;
struct xdp_statistics xdp_stats;
struct pkt_rx_queue *rxq;
struct pkt_tx_queue *txq;
socklen_t optlen;
- int i, ret;
+ int i, ret, fd;
for (i = 0; i < dev->data->nb_rx_queues; i++) {
optlen = sizeof(struct xdp_statistics);
stats->ibytes += stats->q_ibytes[i];
stats->imissed += rxq->stats.rx_dropped;
stats->oerrors += txq->stats.tx_dropped;
- ret = getsockopt(xsk_socket__fd(rxq->xsk), SOL_XDP,
- XDP_STATISTICS, &xdp_stats, &optlen);
+ fd = process_private->rxq_xsk_fds[i];
+ ret = fd >= 0 ? getsockopt(fd, SOL_XDP, XDP_STATISTICS,
+ &xdp_stats, &optlen) : -1;
if (ret != 0) {
AF_XDP_LOG(ERR, "getsockopt() failed for XDP_STATISTICS.\n");
return -1;
int i;
if (rte_eal_process_type() != RTE_PROC_PRIMARY)
- return 0;
+ goto out;
AF_XDP_LOG(INFO, "Closing AF_XDP ethdev on numa socket %u\n",
rte_socket_id());
}
}
+out:
+ rte_free(dev->process_private);
+
return 0;
}
return aligned_addr;
}
+/* Check if the netdev,qid context already exists */
+static inline bool
+ctx_exists(struct pkt_rx_queue *rxq, const char *ifname,
+ struct pkt_rx_queue *list_rxq, const char *list_ifname)
+{
+ bool exists = false;
+
+ if (rxq->xsk_queue_idx == list_rxq->xsk_queue_idx &&
+ !strncmp(ifname, list_ifname, IFNAMSIZ)) {
+ AF_XDP_LOG(ERR, "ctx %s,%i already exists, cannot share umem\n",
+ ifname, rxq->xsk_queue_idx);
+ exists = true;
+ }
+
+ return exists;
+}
+
+/* Get a pointer to an existing UMEM which overlays the rxq's mb_pool */
+static inline int
+get_shared_umem(struct pkt_rx_queue *rxq, const char *ifname,
+ struct xsk_umem_info **umem)
+{
+ struct internal_list *list;
+ struct pmd_internals *internals;
+ int i = 0, ret = 0;
+ struct rte_mempool *mb_pool = rxq->mb_pool;
+
+ if (mb_pool == NULL)
+ return ret;
+
+ pthread_mutex_lock(&internal_list_lock);
+
+ TAILQ_FOREACH(list, &internal_list, next) {
+ internals = list->eth_dev->data->dev_private;
+ for (i = 0; i < internals->queue_cnt; i++) {
+ struct pkt_rx_queue *list_rxq =
+ &internals->rx_queues[i];
+ if (rxq == list_rxq)
+ continue;
+ if (mb_pool == internals->rx_queues[i].mb_pool) {
+ if (ctx_exists(rxq, ifname, list_rxq,
+ internals->if_name)) {
+ ret = -1;
+ goto out;
+ }
+ if (__atomic_load_n(&internals->rx_queues[i].umem->refcnt,
+ __ATOMIC_ACQUIRE)) {
+ *umem = internals->rx_queues[i].umem;
+ goto out;
+ }
+ }
+ }
+ }
+
+out:
+ pthread_mutex_unlock(&internal_list_lock);
+
+ return ret;
+}
+
static struct
xsk_umem_info *xdp_umem_configure(struct pmd_internals *internals,
struct pkt_rx_queue *rxq)
umem = rte_zmalloc_socket("umem", sizeof(*umem), 0,
rte_socket_id());
if (umem == NULL) {
- AF_XDP_LOG(ERR, "Failed to allocate umem info");
+ AF_XDP_LOG(ERR, "Failed to allocate umem info\n");
return NULL;
}
ret = xsk_umem__create(&umem->umem, base_addr, umem_size,
&rxq->fq, &rxq->cq, &usr_config);
if (ret) {
- AF_XDP_LOG(ERR, "Failed to create umem");
+ AF_XDP_LOG(ERR, "Failed to create umem\n");
goto err;
}
umem->buffer = base_addr;
__atomic_store_n(&umem->refcnt, 1, __ATOMIC_RELEASE);
}
+ return umem;
+
+err:
+ xdp_umem_destroy(umem);
+ return NULL;
+}
#else
static struct
xsk_umem_info *xdp_umem_configure(struct pmd_internals *internals,
umem = rte_zmalloc_socket("umem", sizeof(*umem), 0, rte_socket_id());
if (umem == NULL) {
- AF_XDP_LOG(ERR, "Failed to allocate umem info");
+ AF_XDP_LOG(ERR, "Failed to allocate umem info\n");
return NULL;
}
&usr_config);
if (ret) {
- AF_XDP_LOG(ERR, "Failed to create umem");
+ AF_XDP_LOG(ERR, "Failed to create umem\n");
goto err;
}
umem->mz = mz;
-#endif
return umem;
err:
xdp_umem_destroy(umem);
return NULL;
}
+#endif
static int
load_custom_xdp_prog(const char *prog_path, int if_index, struct bpf_map **map)
{
- int ret, prog_fd = -1;
+ int ret, prog_fd;
struct bpf_object *obj;
- ret = bpf_prog_load(prog_path, BPF_PROG_TYPE_XDP, &obj, &prog_fd);
- if (ret) {
+ prog_fd = load_program(prog_path, &obj);
+ if (prog_fd < 0) {
AF_XDP_LOG(ERR, "Failed to load program %s\n", prog_path);
- return ret;
+ return -1;
}
/*
int ret = 0;
int reserve_size = ETH_AF_XDP_DFLT_NUM_DESCS;
struct rte_mbuf *fq_bufs[reserve_size];
+ bool reserve_before;
rxq->umem = xdp_umem_configure(internals, rxq);
if (rxq->umem == NULL)
return -ENOMEM;
txq->umem = rxq->umem;
+ reserve_before = __atomic_load_n(&rxq->umem->refcnt, __ATOMIC_ACQUIRE) <= 1;
+
+#if defined(XDP_UMEM_UNALIGNED_CHUNK_FLAG)
+ ret = rte_pktmbuf_alloc_bulk(rxq->umem->mb_pool, fq_bufs, reserve_size);
+ if (ret) {
+ AF_XDP_LOG(DEBUG, "Failed to get enough buffers for fq.\n");
+ goto out_umem;
+ }
+#endif
+
+ /* reserve fill queue of queues not (yet) sharing UMEM */
+ if (reserve_before) {
+ ret = reserve_fill_queue(rxq->umem, reserve_size, fq_bufs, &rxq->fq);
+ if (ret) {
+ AF_XDP_LOG(ERR, "Failed to reserve fill queue.\n");
+ goto out_umem;
+ }
+ }
cfg.rx_size = ring_size;
cfg.tx_size = ring_size;
cfg.bind_flags |= XDP_USE_NEED_WAKEUP;
#endif
- if (strnlen(internals->prog_path, PATH_MAX) &&
- !internals->custom_prog_configured) {
- ret = load_custom_xdp_prog(internals->prog_path,
- internals->if_index,
- &internals->map);
- if (ret) {
- AF_XDP_LOG(ERR, "Failed to load custom XDP program %s\n",
- internals->prog_path);
- goto err;
+ if (strnlen(internals->prog_path, PATH_MAX)) {
+ if (!internals->custom_prog_configured) {
+ ret = load_custom_xdp_prog(internals->prog_path,
+ internals->if_index,
+ &internals->map);
+ if (ret) {
+ AF_XDP_LOG(ERR, "Failed to load custom XDP program %s\n",
+ internals->prog_path);
+ goto out_umem;
+ }
+ internals->custom_prog_configured = 1;
}
- internals->custom_prog_configured = 1;
- cfg.libbpf_flags = XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD;
+ cfg.libbpf_flags |= XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD;
}
if (internals->shared_umem)
if (ret) {
AF_XDP_LOG(ERR, "Failed to create xsk socket.\n");
- goto err;
+ goto out_umem;
+ }
+
+ if (!reserve_before) {
+ /* reserve fill queue of queues sharing UMEM */
+ ret = reserve_fill_queue(rxq->umem, reserve_size, fq_bufs, &rxq->fq);
+ if (ret) {
+ AF_XDP_LOG(ERR, "Failed to reserve fill queue.\n");
+ goto out_xsk;
+ }
}
/* insert the xsk into the xsks_map */
&rxq->xsk_queue_idx, &fd, 0);
if (err) {
AF_XDP_LOG(ERR, "Failed to insert xsk in map.\n");
- goto err;
+ goto out_xsk;
}
}
-#if defined(XDP_UMEM_UNALIGNED_CHUNK_FLAG)
- ret = rte_pktmbuf_alloc_bulk(rxq->umem->mb_pool, fq_bufs, reserve_size);
- if (ret) {
- AF_XDP_LOG(DEBUG, "Failed to get enough buffers for fq.\n");
- goto err;
- }
-#endif
-
if (rxq->busy_budget) {
ret = configure_preferred_busy_poll(rxq);
if (ret) {
AF_XDP_LOG(ERR, "Failed configure busy polling.\n");
- goto err;
+ goto out_xsk;
}
}
- ret = reserve_fill_queue(rxq->umem, reserve_size, fq_bufs, &rxq->fq);
- if (ret) {
- xsk_socket__delete(rxq->xsk);
- AF_XDP_LOG(ERR, "Failed to reserve fill queue.\n");
- goto err;
- }
-
return 0;
-err:
+out_xsk:
+ xsk_socket__delete(rxq->xsk);
+out_umem:
if (__atomic_sub_fetch(&rxq->umem->refcnt, 1, __ATOMIC_ACQUIRE) == 0)
xdp_umem_destroy(rxq->umem);
struct rte_mempool *mb_pool)
{
struct pmd_internals *internals = dev->data->dev_private;
+ struct pmd_process_private *process_private = dev->process_private;
struct pkt_rx_queue *rxq;
int ret;
rxq->fds[0].fd = xsk_socket__fd(rxq->xsk);
rxq->fds[0].events = POLLIN;
+ process_private->rxq_xsk_fds[rx_queue_id] = rxq->fds[0].fd;
+
dev->data->rx_queues[rx_queue_id] = rxq;
return 0;
{
const char *name = rte_vdev_device_name(dev);
const unsigned int numa_node = dev->device.numa_node;
+ struct pmd_process_private *process_private;
struct pmd_internals *internals;
struct rte_eth_dev *eth_dev;
int ret;
if (ret)
goto err_free_tx;
+ process_private = (struct pmd_process_private *)
+ rte_zmalloc_socket(name, sizeof(struct pmd_process_private),
+ RTE_CACHE_LINE_SIZE, numa_node);
+ if (process_private == NULL) {
+ AF_XDP_LOG(ERR, "Failed to alloc memory for process private\n");
+ goto err_free_tx;
+ }
+
eth_dev = rte_eth_vdev_allocate(dev, 0);
if (eth_dev == NULL)
- goto err_free_tx;
+ goto err_free_pp;
eth_dev->data->dev_private = internals;
eth_dev->data->dev_link = pmd_link;
eth_dev->dev_ops = &ops;
eth_dev->rx_pkt_burst = eth_af_xdp_rx;
eth_dev->tx_pkt_burst = eth_af_xdp_tx;
+ eth_dev->process_private = process_private;
+
+ for (i = 0; i < queue_cnt; i++)
+ process_private->rxq_xsk_fds[i] = -1;
#if defined(XDP_UMEM_UNALIGNED_CHUNK_FLAG)
AF_XDP_LOG(INFO, "Zero copy between umem and mbuf enabled.\n");
return eth_dev;
+err_free_pp:
+ rte_free(process_private);
err_free_tx:
rte_free(internals->tx_queues);
err_free_rx:
return NULL;
}
+/* Secondary process requests rxq fds from primary. */
+static int
+afxdp_mp_request_fds(const char *name, struct rte_eth_dev *dev)
+{
+ struct pmd_process_private *process_private = dev->process_private;
+ struct timespec timeout = {.tv_sec = 1, .tv_nsec = 0};
+ struct rte_mp_msg request, *reply;
+ struct rte_mp_reply replies;
+ struct ipc_hdr *request_param = (struct ipc_hdr *)request.param;
+ int i, ret;
+
+ /* Prepare the request */
+ memset(&request, 0, sizeof(request));
+ strlcpy(request.name, ETH_AF_XDP_MP_KEY, sizeof(request.name));
+ strlcpy(request_param->port_name, name,
+ sizeof(request_param->port_name));
+ request.len_param = sizeof(*request_param);
+
+ /* Send the request and receive the reply */
+ AF_XDP_LOG(DEBUG, "Sending multi-process IPC request for %s\n", name);
+ ret = rte_mp_request_sync(&request, &replies, &timeout);
+ if (ret < 0 || replies.nb_received != 1) {
+ AF_XDP_LOG(ERR, "Failed to request fds from primary: %d\n",
+ rte_errno);
+ return -1;
+ }
+ reply = replies.msgs;
+ AF_XDP_LOG(DEBUG, "Received multi-process IPC reply for %s\n", name);
+ if (dev->data->nb_rx_queues != reply->num_fds) {
+ AF_XDP_LOG(ERR, "Incorrect number of fds received: %d != %d\n",
+ reply->num_fds, dev->data->nb_rx_queues);
+ return -EINVAL;
+ }
+
+ for (i = 0; i < reply->num_fds; i++)
+ process_private->rxq_xsk_fds[i] = reply->fds[i];
+
+ free(reply);
+ return 0;
+}
+
+/* Primary process sends rxq fds to secondary. */
+static int
+afxdp_mp_send_fds(const struct rte_mp_msg *request, const void *peer)
+{
+ struct rte_eth_dev *dev;
+ struct pmd_process_private *process_private;
+ struct rte_mp_msg reply;
+ const struct ipc_hdr *request_param =
+ (const struct ipc_hdr *)request->param;
+ struct ipc_hdr *reply_param =
+ (struct ipc_hdr *)reply.param;
+ const char *request_name = request_param->port_name;
+ int i;
+
+ AF_XDP_LOG(DEBUG, "Received multi-process IPC request for %s\n",
+ request_name);
+
+ /* Find the requested port */
+ dev = rte_eth_dev_get_by_name(request_name);
+ if (!dev) {
+ AF_XDP_LOG(ERR, "Failed to get port id for %s\n", request_name);
+ return -1;
+ }
+ process_private = dev->process_private;
+
+ /* Populate the reply with the xsk fd for each queue */
+ reply.num_fds = 0;
+ if (dev->data->nb_rx_queues > RTE_MP_MAX_FD_NUM) {
+ AF_XDP_LOG(ERR, "Number of rx queues (%d) exceeds max number of fds (%d)\n",
+ dev->data->nb_rx_queues, RTE_MP_MAX_FD_NUM);
+ return -EINVAL;
+ }
+
+ for (i = 0; i < dev->data->nb_rx_queues; i++)
+ reply.fds[reply.num_fds++] = process_private->rxq_xsk_fds[i];
+
+ /* Send the reply */
+ strlcpy(reply.name, request->name, sizeof(reply.name));
+ strlcpy(reply_param->port_name, request_name,
+ sizeof(reply_param->port_name));
+ reply.len_param = sizeof(*reply_param);
+ AF_XDP_LOG(DEBUG, "Sending multi-process IPC reply for %s\n",
+ reply_param->port_name);
+ if (rte_mp_reply(&reply, peer) < 0) {
+ AF_XDP_LOG(ERR, "Failed to reply to multi-process IPC request\n");
+ return -1;
+ }
+ return 0;
+}
+
static int
rte_pmd_af_xdp_probe(struct rte_vdev_device *dev)
{
int xsk_queue_cnt = ETH_AF_XDP_DFLT_QUEUE_COUNT;
int shared_umem = 0;
char prog_path[PATH_MAX] = {'\0'};
- int busy_budget = -1;
+ int busy_budget = -1, ret;
struct rte_eth_dev *eth_dev = NULL;
- const char *name;
+ const char *name = rte_vdev_device_name(dev);
- AF_XDP_LOG(INFO, "Initializing pmd_af_xdp for %s\n",
- rte_vdev_device_name(dev));
+ AF_XDP_LOG(INFO, "Initializing pmd_af_xdp for %s\n", name);
- name = rte_vdev_device_name(dev);
if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
- AF_XDP_LOG(ERR, "Failed to probe %s. "
- "AF_XDP PMD does not support secondary processes.\n",
- name);
- return -ENOTSUP;
+ eth_dev = rte_eth_dev_attach_secondary(name);
+ if (eth_dev == NULL) {
+ AF_XDP_LOG(ERR, "Failed to probe %s\n", name);
+ return -EINVAL;
+ }
+ eth_dev->dev_ops = &ops;
+ eth_dev->device = &dev->device;
+ eth_dev->rx_pkt_burst = rte_eth_pkt_burst_dummy;
+ eth_dev->tx_pkt_burst = rte_eth_pkt_burst_dummy;
+ eth_dev->process_private = (struct pmd_process_private *)
+ rte_zmalloc_socket(name,
+ sizeof(struct pmd_process_private),
+ RTE_CACHE_LINE_SIZE,
+ eth_dev->device->numa_node);
+ if (eth_dev->process_private == NULL) {
+ AF_XDP_LOG(ERR,
+ "Failed to alloc memory for process private\n");
+ return -ENOMEM;
+ }
+
+ /* Obtain the xsk fds from the primary process. */
+ if (afxdp_mp_request_fds(name, eth_dev))
+ return -1;
+
+ rte_eth_dev_probing_finish(eth_dev);
+ return 0;
}
kvlist = rte_kvargs_parse(rte_vdev_device_args(dev), valid_arguments);
return -1;
}
+ /* Register IPC callback which shares xsk fds from primary to secondary */
+ if (!afxdp_dev_count) {
+ ret = rte_mp_action_register(ETH_AF_XDP_MP_KEY, afxdp_mp_send_fds);
+ if (ret < 0 && rte_errno != ENOTSUP) {
+ AF_XDP_LOG(ERR, "%s: Failed to register multi-process IPC callback: %s\n",
+ name, strerror(rte_errno));
+ return -1;
+ }
+ }
+ afxdp_dev_count++;
+
rte_eth_dev_probing_finish(eth_dev);
return 0;
return 0;
eth_dev_close(eth_dev);
+ if (afxdp_dev_count == 1)
+ rte_mp_action_unregister(ETH_AF_XDP_MP_KEY);
+ afxdp_dev_count--;
rte_eth_dev_release_port(eth_dev);
-
return 0;
}
git.droids-corp.org / dpdk.git / blobdiff