1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation
11 #include <sys/types.h>
12 #include <sys/resource.h>
15 #include <rte_common.h>
16 #include <rte_errno.h>
17 #include <rte_ethdev.h>
19 #include <rte_malloc.h>
21 #include <rte_spinlock.h>
22 #include <rte_string_fns.h>
24 #include "compat_netmap.h"
27 struct rte_mempool *pool;
28 struct netmap_if *nmif;
29 struct rte_eth_conf eth_conf;
30 struct rte_eth_txconf tx_conf;
31 struct rte_eth_rxconf rx_conf;
47 #define POLLRDNORM 0x0040
51 #define POLLWRNORM 0x0100
54 #define FD_PORT_FREE UINT32_MAX
55 #define FD_PORT_RSRV (FD_PORT_FREE - 1)
58 struct rte_netmap_conf conf;
66 #define COMPAT_NETMAP_MAX_NOFILE (2 * RTE_MAX_ETHPORTS)
67 #define COMPAT_NETMAP_MAX_BURST 64
68 #define COMPAT_NETMAP_MAX_PKT_PER_SYNC (2 * COMPAT_NETMAP_MAX_BURST)
70 static struct netmap_port ports[RTE_MAX_ETHPORTS];
71 static struct netmap_state netmap;
73 static struct fd_port fd_port[COMPAT_NETMAP_MAX_NOFILE];
74 static const int next_fd_start = RLIMIT_NOFILE + 1;
75 static rte_spinlock_t netmap_lock;
77 #define IDX_TO_FD(x) ((x) + next_fd_start)
78 #define FD_TO_IDX(x) ((x) - next_fd_start)
79 #define FD_VALID(x) ((x) >= next_fd_start && \
80 (x) < (typeof (x))(RTE_DIM(fd_port) + next_fd_start))
82 #define PORT_NUM_RINGS (2 * netmap.conf.max_rings)
83 #define PORT_NUM_SLOTS (PORT_NUM_RINGS * netmap.conf.max_slots)
85 #define BUF_IDX(port, ring, slot) \
86 (((port) * PORT_NUM_RINGS + (ring)) * netmap.conf.max_slots + \
89 #define NETMAP_IF_RING_OFS(rid, rings, slots) ({\
90 struct netmap_if *_if; \
91 struct netmap_ring *_rg; \
93 (rings) * sizeof(_if->ring_ofs[0]) + \
94 (rid) * sizeof(*_rg) + \
95 (slots) * sizeof(_rg->slot[0]); \
98 static void netmap_unregif(uint32_t idx, uint32_t port);
102 ifname_to_portid(const char *ifname, uint16_t *port)
108 portid = strtoul(ifname, &endptr, 10);
109 if (endptr == ifname || *endptr != '\0' ||
110 portid >= RTE_DIM(ports) || errno != 0)
118 * Given a dpdk mbuf, fill in the Netmap slot in ring r and its associated
119 * buffer with the data held by the mbuf.
120 * Note that mbuf chains are not supported.
123 mbuf_to_slot(struct rte_mbuf *mbuf, struct netmap_ring *r, uint32_t index)
128 data = rte_pktmbuf_mtod(mbuf, char *);
129 length = rte_pktmbuf_data_len(mbuf);
131 if (length > r->nr_buf_size)
134 r->slot[index].len = length;
135 rte_memcpy(NETMAP_BUF(r, r->slot[index].buf_idx), data, length);
139 * Given a Netmap ring and a slot index for that ring, construct a dpdk mbuf
140 * from the data held in the buffer associated with the slot.
141 * Allocation/deallocation of the dpdk mbuf are the responsibility of the
143 * Note that mbuf chains are not supported.
146 slot_to_mbuf(struct netmap_ring *r, uint32_t index, struct rte_mbuf *mbuf)
151 rte_pktmbuf_reset(mbuf);
152 length = r->slot[index].len;
153 data = rte_pktmbuf_append(mbuf, length);
156 rte_memcpy(data, NETMAP_BUF(r, r->slot[index].buf_idx), length);
164 for (i = 0; i != RTE_DIM(fd_port) && fd_port[i].port != FD_PORT_FREE;
168 if (i == RTE_DIM(fd_port))
171 fd_port[i].port = FD_PORT_RSRV;
176 fd_release(int32_t fd)
182 if (!FD_VALID(fd) || (port = fd_port[idx].port) == FD_PORT_FREE)
185 /* if we still have a valid port attached, release the port */
186 if (port < RTE_DIM(ports) && ports[port].fd == idx) {
187 netmap_unregif(idx, port);
190 fd_port[idx].port = FD_PORT_FREE;
195 check_nmreq(struct nmreq *req, uint16_t *port)
203 if (req->nr_version != NETMAP_API) {
204 req->nr_version = NETMAP_API;
208 if ((rc = ifname_to_portid(req->nr_name, &portid)) != 0) {
209 RTE_LOG(ERR, USER1, "Invalid interface name:\"%s\" "
210 "in NIOCGINFO call\n", req->nr_name);
214 if (ports[portid].pool == NULL) {
215 RTE_LOG(ERR, USER1, "Misconfigured portid %u\n", portid);
224 * Simulate a Netmap NIOCGINFO ioctl: given a struct nmreq holding an interface
225 * name (a port number in our case), fill the struct nmreq in with advisory
226 * information about the interface: number of rings and their size, total memory
227 * required in the map, ...
228 * Those are preconfigured using rte_eth_{,tx,rx}conf and
229 * rte_netmap_port_conf structures
230 * and calls to rte_netmap_init_port() in the Netmap application.
233 ioctl_niocginfo(__rte_unused int fd, void * param)
239 req = (struct nmreq *)param;
240 if ((rc = check_nmreq(req, &portid)) != 0)
243 req->nr_tx_rings = (uint16_t)(ports[portid].nr_tx_rings - 1);
244 req->nr_rx_rings = (uint16_t)(ports[portid].nr_rx_rings - 1);
245 req->nr_tx_slots = ports[portid].nr_tx_slots;
246 req->nr_rx_slots = ports[portid].nr_rx_slots;
248 /* in current implementation we have all NETIFs shared aone region. */
249 req->nr_memsize = netmap.mem_sz;
256 netmap_ring_setup(struct netmap_ring *ring, uint16_t port, uint32_t ringid,
261 ring->buf_ofs = netmap.buf_start - (uintptr_t)ring;
262 ring->num_slots = num_slots;
265 ring->nr_buf_size = netmap.conf.max_bufsz;
268 ring->ts.tv_usec = 0;
270 for (j = 0; j < ring->num_slots; j++) {
271 ring->slot[j].buf_idx = BUF_IDX(port, ringid, j);
272 ring->slot[j].len = 0;
278 netmap_regif(struct nmreq *req, uint32_t idx, uint16_t port)
280 struct netmap_if *nmif;
281 struct netmap_ring *ring;
282 uint32_t i, slots, start_ring;
285 if (ports[port].fd < RTE_DIM(fd_port)) {
286 RTE_LOG(ERR, USER1, "port %u already in use by fd: %u\n",
287 port, IDX_TO_FD(ports[port].fd));
290 if (fd_port[idx].port != FD_PORT_RSRV) {
291 RTE_LOG(ERR, USER1, "fd: %u is misconfigured\n",
296 nmif = ports[port].nmif;
298 /* setup netmap_if fields. */
299 memset(nmif, 0, netmap.netif_memsz);
301 /* only ALL rings supported right now. */
302 if (req->nr_ringid != 0)
305 strlcpy(nmif->ni_name, req->nr_name, sizeof(nmif->ni_name));
306 nmif->ni_version = req->nr_version;
308 /* Netmap uses ni_(r|t)x_rings + 1 */
309 nmif->ni_rx_rings = ports[port].nr_rx_rings - 1;
310 nmif->ni_tx_rings = ports[port].nr_tx_rings - 1;
313 * Setup TX rings and slots.
314 * Refer to the comments in netmap.h for details
318 for (i = 0; i < nmif->ni_tx_rings + 1; i++) {
320 nmif->ring_ofs[i] = NETMAP_IF_RING_OFS(i,
321 PORT_NUM_RINGS, slots);
323 ring = NETMAP_TXRING(nmif, i);
324 netmap_ring_setup(ring, port, i, ports[port].nr_tx_slots);
325 ring->avail = ring->num_slots;
327 slots += ports[port].nr_tx_slots;
331 * Setup RX rings and slots.
332 * Refer to the comments in netmap.h for details
337 for (; i < nmif->ni_rx_rings + 1 + start_ring; i++) {
339 nmif->ring_ofs[i] = NETMAP_IF_RING_OFS(i,
340 PORT_NUM_RINGS, slots);
342 ring = NETMAP_RXRING(nmif, (i - start_ring));
343 netmap_ring_setup(ring, port, i, ports[port].nr_rx_slots);
346 slots += ports[port].nr_rx_slots;
349 if ((rc = rte_eth_dev_start(port)) < 0) {
351 "Couldn't start ethernet device %s (error %d)\n",
356 /* setup fdi <--> port relationtip. */
357 ports[port].fd = idx;
358 fd_port[idx].port = port;
360 req->nr_memsize = netmap.mem_sz;
361 req->nr_offset = (uintptr_t)nmif - (uintptr_t)netmap.mem;
367 * Simulate a Netmap NIOCREGIF ioctl:
370 ioctl_niocregif(int32_t fd, void * param)
377 req = (struct nmreq *)param;
378 if ((rc = check_nmreq(req, &portid)) != 0)
383 rte_spinlock_lock(&netmap_lock);
384 rc = netmap_regif(req, idx, portid);
385 rte_spinlock_unlock(&netmap_lock);
391 netmap_unregif(uint32_t idx, uint32_t port)
393 fd_port[idx].port = FD_PORT_RSRV;
394 ports[port].fd = UINT32_MAX;
395 rte_eth_dev_stop(port);
399 * Simulate a Netmap NIOCUNREGIF ioctl: put an interface running in Netmap
400 * mode back in "normal" mode. In our case, we just stop the port associated
401 * with this file descriptor.
404 ioctl_niocunregif(int fd)
411 rte_spinlock_lock(&netmap_lock);
413 port = fd_port[idx].port;
414 if (port < RTE_DIM(ports) && ports[port].fd == idx) {
415 netmap_unregif(idx, port);
419 "%s: %d is not associated with valid port\n",
424 rte_spinlock_unlock(&netmap_lock);
429 * A call to rx_sync_ring will try to fill a Netmap RX ring with as many
430 * packets as it can hold coming from its dpdk port.
433 rx_sync_ring(struct netmap_ring *ring, uint16_t port, uint16_t ring_number,
438 uint32_t cur_slot, n_free_slots;
439 struct rte_mbuf *rx_mbufs[COMPAT_NETMAP_MAX_BURST];
441 n_free_slots = ring->num_slots - (ring->avail + ring->reserved);
442 n_free_slots = RTE_MIN(n_free_slots, max_burst);
443 cur_slot = (ring->cur + ring->avail) & (ring->num_slots - 1);
445 while (n_free_slots) {
446 burst_size = (uint16_t)RTE_MIN(n_free_slots, RTE_DIM(rx_mbufs));
448 /* receive up to burst_size packets from the NIC's queue */
449 n_rx = rte_eth_rx_burst(port, ring_number, rx_mbufs,
454 if (unlikely(n_rx < 0))
457 /* Put those n_rx packets in the Netmap structures */
458 for (i = 0; i < n_rx ; i++) {
459 mbuf_to_slot(rx_mbufs[i], ring, cur_slot);
460 rte_pktmbuf_free(rx_mbufs[i]);
461 cur_slot = NETMAP_RING_NEXT(ring, cur_slot);
464 /* Update the Netmap ring structure to reflect the change */
466 n_free_slots -= n_rx;
473 rx_sync_if(uint32_t port)
477 struct netmap_if *nifp;
478 struct netmap_ring *r;
480 nifp = ports[port].nmif;
481 burst = ports[port].rx_burst;
484 for (i = 0; i < nifp->ni_rx_rings + 1; i++) {
485 r = NETMAP_RXRING(nifp, i);
486 rx_sync_ring(r, port, (uint16_t)i, burst);
494 * Simulate a Netmap NIOCRXSYNC ioctl:
497 ioctl_niocrxsync(int fd)
502 if ((port = fd_port[idx].port) < RTE_DIM(ports) &&
503 ports[port].fd == idx) {
504 return rx_sync_if(fd_port[idx].port);
511 * A call to tx_sync_ring will try to empty a Netmap TX ring by converting its
512 * buffers into rte_mbufs and sending them out on the rings's dpdk port.
515 tx_sync_ring(struct netmap_ring *ring, uint16_t port, uint16_t ring_number,
516 struct rte_mempool *pool, uint16_t max_burst)
520 uint32_t cur_slot, n_used_slots;
521 struct rte_mbuf *tx_mbufs[COMPAT_NETMAP_MAX_BURST];
523 n_used_slots = ring->num_slots - ring->avail;
524 n_used_slots = RTE_MIN(n_used_slots, max_burst);
525 cur_slot = (ring->cur + ring->avail) & (ring->num_slots - 1);
527 while (n_used_slots) {
528 burst_size = (uint16_t)RTE_MIN(n_used_slots, RTE_DIM(tx_mbufs));
530 for (i = 0; i < burst_size; i++) {
531 tx_mbufs[i] = rte_pktmbuf_alloc(pool);
532 if (tx_mbufs[i] == NULL)
535 slot_to_mbuf(ring, cur_slot, tx_mbufs[i]);
536 cur_slot = NETMAP_RING_NEXT(ring, cur_slot);
539 n_tx = rte_eth_tx_burst(port, ring_number, tx_mbufs,
542 /* Update the Netmap ring structure to reflect the change */
544 n_used_slots -= n_tx;
546 /* Return the mbufs that failed to transmit to their pool */
547 if (unlikely(n_tx != burst_size)) {
548 for (i = n_tx; i < burst_size; i++)
549 rte_pktmbuf_free(tx_mbufs[i]);
558 rte_pktmbuf_free(tx_mbufs[i]);
561 "Couldn't get mbuf from mempool is the mempool too small?\n");
566 tx_sync_if(uint32_t port)
570 struct netmap_if *nifp;
571 struct netmap_ring *r;
572 struct rte_mempool *mp;
574 nifp = ports[port].nmif;
575 mp = ports[port].pool;
576 burst = ports[port].tx_burst;
579 for (i = 0; i < nifp->ni_tx_rings + 1; i++) {
580 r = NETMAP_TXRING(nifp, i);
581 tx_sync_ring(r, port, (uint16_t)i, mp, burst);
589 * Simulate a Netmap NIOCTXSYNC ioctl:
592 ioctl_nioctxsync(int fd)
597 if ((port = fd_port[idx].port) < RTE_DIM(ports) &&
598 ports[port].fd == idx) {
599 return tx_sync_if(fd_port[idx].port);
606 * Give the library a mempool of rte_mbufs with which it can do the
607 * rte_mbuf <--> netmap slot conversions.
610 rte_netmap_init(const struct rte_netmap_conf *conf)
612 size_t buf_ofs, nmif_sz, sz;
613 size_t port_rings, port_slots, port_bufs;
614 uint32_t i, port_num;
616 port_num = RTE_MAX_ETHPORTS;
617 port_rings = 2 * conf->max_rings;
618 port_slots = port_rings * conf->max_slots;
619 port_bufs = port_slots;
621 nmif_sz = NETMAP_IF_RING_OFS(port_rings, port_rings, port_slots);
622 sz = nmif_sz * port_num;
624 buf_ofs = RTE_ALIGN_CEIL(sz, RTE_CACHE_LINE_SIZE);
625 sz = buf_ofs + port_bufs * conf->max_bufsz * port_num;
627 if (sz > UINT32_MAX ||
628 (netmap.mem = rte_zmalloc_socket(__func__, sz,
629 RTE_CACHE_LINE_SIZE, conf->socket_id)) == NULL) {
630 RTE_LOG(ERR, USER1, "%s: failed to allocate %zu bytes\n",
636 netmap.netif_memsz = nmif_sz;
637 netmap.buf_start = (uintptr_t)netmap.mem + buf_ofs;
640 rte_spinlock_init(&netmap_lock);
642 /* Mark all ports as unused and set NETIF pointer. */
643 for (i = 0; i != RTE_DIM(ports); i++) {
644 ports[i].fd = UINT32_MAX;
645 ports[i].nmif = (struct netmap_if *)
646 ((uintptr_t)netmap.mem + nmif_sz * i);
649 /* Mark all fd_ports as unused. */
650 for (i = 0; i != RTE_DIM(fd_port); i++) {
651 fd_port[i].port = FD_PORT_FREE;
659 rte_netmap_init_port(uint16_t portid, const struct rte_netmap_port_conf *conf)
663 uint16_t rx_slots, tx_slots;
664 struct rte_eth_rxconf rxq_conf;
665 struct rte_eth_txconf txq_conf;
666 struct rte_eth_dev_info dev_info;
669 portid >= RTE_DIM(ports) ||
670 conf->nr_tx_rings > netmap.conf.max_rings ||
671 conf->nr_rx_rings > netmap.conf.max_rings) {
672 RTE_LOG(ERR, USER1, "%s(%u): invalid parameters\n",
677 rx_slots = (uint16_t)rte_align32pow2(conf->nr_rx_slots);
678 tx_slots = (uint16_t)rte_align32pow2(conf->nr_tx_slots);
680 if (tx_slots > netmap.conf.max_slots ||
681 rx_slots > netmap.conf.max_slots) {
682 RTE_LOG(ERR, USER1, "%s(%u): invalid parameters\n",
687 ret = rte_eth_dev_info_get(portid, &dev_info);
690 "Error during getting device (port %u) info: %s\n",
691 portid, strerror(-ret));
695 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
696 conf->eth_conf->txmode.offloads |=
697 DEV_TX_OFFLOAD_MBUF_FAST_FREE;
698 ret = rte_eth_dev_configure(portid, conf->nr_rx_rings,
699 conf->nr_tx_rings, conf->eth_conf);
702 RTE_LOG(ERR, USER1, "Couldn't configure port %u\n", portid);
706 ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &rx_slots, &tx_slots);
710 "Couldn't ot adjust number of descriptors for port %u\n",
715 rxq_conf = dev_info.default_rxconf;
716 rxq_conf.offloads = conf->eth_conf->rxmode.offloads;
717 txq_conf = dev_info.default_txconf;
718 txq_conf.offloads = conf->eth_conf->txmode.offloads;
719 for (i = 0; i < conf->nr_tx_rings; i++) {
720 ret = rte_eth_tx_queue_setup(portid, i, tx_slots,
721 conf->socket_id, &txq_conf);
725 "fail to configure TX queue %u of port %u\n",
730 ret = rte_eth_rx_queue_setup(portid, i, rx_slots,
731 conf->socket_id, &rxq_conf, conf->pool);
735 "fail to configure RX queue %u of port %u\n",
741 /* copy config to the private storage. */
742 ports[portid].eth_conf = conf->eth_conf[0];
743 ports[portid].pool = conf->pool;
744 ports[portid].socket_id = conf->socket_id;
745 ports[portid].nr_tx_rings = conf->nr_tx_rings;
746 ports[portid].nr_rx_rings = conf->nr_rx_rings;
747 ports[portid].nr_tx_slots = tx_slots;
748 ports[portid].nr_rx_slots = rx_slots;
749 ports[portid].tx_burst = conf->tx_burst;
750 ports[portid].rx_burst = conf->rx_burst;
756 rte_netmap_close(__rte_unused int fd)
760 rte_spinlock_lock(&netmap_lock);
762 rte_spinlock_unlock(&netmap_lock);
771 int rte_netmap_ioctl(int fd, uint32_t op, void *param)
783 ret = ioctl_niocginfo(fd, param);
787 ret = ioctl_niocregif(fd, param);
791 ret = ioctl_niocunregif(fd);
795 ret = ioctl_niocrxsync(fd);
799 ret = ioctl_nioctxsync(fd);
817 rte_netmap_mmap(void *addr, size_t length,
818 int prot, int flags, int fd, off_t offset)
820 static const int cprot = PROT_WRITE | PROT_READ;
822 if (!FD_VALID(fd) || length + offset > netmap.mem_sz ||
823 (prot & cprot) != cprot ||
824 ((flags & MAP_FIXED) != 0 && addr != NULL)) {
830 return (void *)((uintptr_t)netmap.mem + (uintptr_t)offset);
834 * Return a "fake" file descriptor with a value above RLIMIT_NOFILE so that
835 * any attempt to use that file descriptor with the usual API will fail.
838 rte_netmap_open(__rte_unused const char *pathname, __rte_unused int flags)
842 rte_spinlock_lock(&netmap_lock);
844 rte_spinlock_unlock(&netmap_lock);
854 * Doesn't support timeout other than 0 or infinite (negative) timeout
857 rte_netmap_poll(struct pollfd *fds, nfds_t nfds, int timeout)
859 int32_t count_it, ret;
860 uint32_t i, idx, port;
861 uint32_t want_rx, want_tx;
868 for (i = 0; i < nfds; i++) {
872 if (!FD_VALID(fds[i].fd) || fds[i].events == 0) {
877 idx = FD_TO_IDX(fds[i].fd);
878 if ((port = fd_port[idx].port) >= RTE_DIM(ports) ||
879 ports[port].fd != idx) {
881 fds[i].revents |= POLLERR;
886 want_rx = fds[i].events & (POLLIN | POLLRDNORM);
887 want_tx = fds[i].events & (POLLOUT | POLLWRNORM);
889 if (want_rx && rx_sync_if(port) > 0) {
890 fds[i].revents = (uint16_t)
891 (fds[i].revents | want_rx);
894 if (want_tx && tx_sync_if(port) > 0) {
895 fds[i].revents = (uint16_t)
896 (fds[i].revents | want_tx);
903 while ((ret == 0 && timeout < 0) || timeout);