1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018-2021 Beijing WangXun Technology Co., Ltd.
3 * Copyright(c) 2010-2017 Intel Corporation
9 #include <rte_ethdev.h>
10 #include <ethdev_driver.h>
11 #include <rte_malloc.h>
13 #include "ngbe_logs.h"
14 #include "base/ngbe.h"
15 #include "ngbe_ethdev.h"
16 #include "ngbe_rxtx.h"
19 * Prefetch a cache line into all cache levels.
21 #define rte_ngbe_prefetch(p) rte_prefetch0(p)
23 /*********************************************************************
27 **********************************************************************/
30 * Check for descriptors with their DD bit set and free mbufs.
31 * Return the total number of buffers freed.
33 static __rte_always_inline int
34 ngbe_tx_free_bufs(struct ngbe_tx_queue *txq)
36 struct ngbe_tx_entry *txep;
39 struct rte_mbuf *m, *free[RTE_NGBE_TX_MAX_FREE_BUF_SZ];
41 /* check DD bit on threshold descriptor */
42 status = txq->tx_ring[txq->tx_next_dd].dw3;
43 if (!(status & rte_cpu_to_le_32(NGBE_TXD_DD))) {
44 if (txq->nb_tx_free >> 1 < txq->tx_free_thresh)
45 ngbe_set32_masked(txq->tdc_reg_addr,
46 NGBE_TXCFG_FLUSH, NGBE_TXCFG_FLUSH);
51 * first buffer to free from S/W ring is at index
52 * tx_next_dd - (tx_free_thresh-1)
54 txep = &txq->sw_ring[txq->tx_next_dd - (txq->tx_free_thresh - 1)];
55 for (i = 0; i < txq->tx_free_thresh; ++i, ++txep) {
56 /* free buffers one at a time */
57 m = rte_pktmbuf_prefree_seg(txep->mbuf);
60 if (unlikely(m == NULL))
63 if (nb_free >= RTE_NGBE_TX_MAX_FREE_BUF_SZ ||
64 (nb_free > 0 && m->pool != free[0]->pool)) {
65 rte_mempool_put_bulk(free[0]->pool,
66 (void **)free, nb_free);
74 rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free);
76 /* buffers were freed, update counters */
77 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_free_thresh);
78 txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_free_thresh);
79 if (txq->tx_next_dd >= txq->nb_tx_desc)
80 txq->tx_next_dd = (uint16_t)(txq->tx_free_thresh - 1);
82 return txq->tx_free_thresh;
85 /* Populate 4 descriptors with data from 4 mbufs */
87 tx4(volatile struct ngbe_tx_desc *txdp, struct rte_mbuf **pkts)
89 uint64_t buf_dma_addr;
93 for (i = 0; i < 4; ++i, ++txdp, ++pkts) {
94 buf_dma_addr = rte_mbuf_data_iova(*pkts);
95 pkt_len = (*pkts)->data_len;
97 /* write data to descriptor */
98 txdp->qw0 = rte_cpu_to_le_64(buf_dma_addr);
99 txdp->dw2 = cpu_to_le32(NGBE_TXD_FLAGS |
100 NGBE_TXD_DATLEN(pkt_len));
101 txdp->dw3 = cpu_to_le32(NGBE_TXD_PAYLEN(pkt_len));
103 rte_prefetch0(&(*pkts)->pool);
107 /* Populate 1 descriptor with data from 1 mbuf */
109 tx1(volatile struct ngbe_tx_desc *txdp, struct rte_mbuf **pkts)
111 uint64_t buf_dma_addr;
114 buf_dma_addr = rte_mbuf_data_iova(*pkts);
115 pkt_len = (*pkts)->data_len;
117 /* write data to descriptor */
118 txdp->qw0 = cpu_to_le64(buf_dma_addr);
119 txdp->dw2 = cpu_to_le32(NGBE_TXD_FLAGS |
120 NGBE_TXD_DATLEN(pkt_len));
121 txdp->dw3 = cpu_to_le32(NGBE_TXD_PAYLEN(pkt_len));
123 rte_prefetch0(&(*pkts)->pool);
127 * Fill H/W descriptor ring with mbuf data.
128 * Copy mbuf pointers to the S/W ring.
131 ngbe_tx_fill_hw_ring(struct ngbe_tx_queue *txq, struct rte_mbuf **pkts,
134 volatile struct ngbe_tx_desc *txdp = &txq->tx_ring[txq->tx_tail];
135 struct ngbe_tx_entry *txep = &txq->sw_ring[txq->tx_tail];
136 const int N_PER_LOOP = 4;
137 const int N_PER_LOOP_MASK = N_PER_LOOP - 1;
138 int mainpart, leftover;
142 * Process most of the packets in chunks of N pkts. Any
143 * leftover packets will get processed one at a time.
145 mainpart = (nb_pkts & ((uint32_t)~N_PER_LOOP_MASK));
146 leftover = (nb_pkts & ((uint32_t)N_PER_LOOP_MASK));
147 for (i = 0; i < mainpart; i += N_PER_LOOP) {
148 /* Copy N mbuf pointers to the S/W ring */
149 for (j = 0; j < N_PER_LOOP; ++j)
150 (txep + i + j)->mbuf = *(pkts + i + j);
151 tx4(txdp + i, pkts + i);
154 if (unlikely(leftover > 0)) {
155 for (i = 0; i < leftover; ++i) {
156 (txep + mainpart + i)->mbuf = *(pkts + mainpart + i);
157 tx1(txdp + mainpart + i, pkts + mainpart + i);
162 static inline uint16_t
163 tx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
166 struct ngbe_tx_queue *txq = (struct ngbe_tx_queue *)tx_queue;
170 * Begin scanning the H/W ring for done descriptors when the
171 * number of available descriptors drops below tx_free_thresh.
172 * For each done descriptor, free the associated buffer.
174 if (txq->nb_tx_free < txq->tx_free_thresh)
175 ngbe_tx_free_bufs(txq);
177 /* Only use descriptors that are available */
178 nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts);
179 if (unlikely(nb_pkts == 0))
182 /* Use exactly nb_pkts descriptors */
183 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts);
186 * At this point, we know there are enough descriptors in the
187 * ring to transmit all the packets. This assumes that each
188 * mbuf contains a single segment, and that no new offloads
189 * are expected, which would require a new context descriptor.
193 * See if we're going to wrap-around. If so, handle the top
194 * of the descriptor ring first, then do the bottom. If not,
195 * the processing looks just like the "bottom" part anyway...
197 if ((txq->tx_tail + nb_pkts) > txq->nb_tx_desc) {
198 n = (uint16_t)(txq->nb_tx_desc - txq->tx_tail);
199 ngbe_tx_fill_hw_ring(txq, tx_pkts, n);
203 /* Fill H/W descriptor ring with mbuf data */
204 ngbe_tx_fill_hw_ring(txq, tx_pkts + n, (uint16_t)(nb_pkts - n));
205 txq->tx_tail = (uint16_t)(txq->tx_tail + (nb_pkts - n));
208 * Check for wrap-around. This would only happen if we used
209 * up to the last descriptor in the ring, no more, no less.
211 if (txq->tx_tail >= txq->nb_tx_desc)
214 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u",
215 (uint16_t)txq->port_id, (uint16_t)txq->queue_id,
216 (uint16_t)txq->tx_tail, (uint16_t)nb_pkts);
218 /* update tail pointer */
220 ngbe_set32_relaxed(txq->tdt_reg_addr, txq->tx_tail);
226 ngbe_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts,
231 /* Try to transmit at least chunks of TX_MAX_BURST pkts */
232 if (likely(nb_pkts <= RTE_PMD_NGBE_TX_MAX_BURST))
233 return tx_xmit_pkts(tx_queue, tx_pkts, nb_pkts);
235 /* transmit more than the max burst, in chunks of TX_MAX_BURST */
237 while (nb_pkts != 0) {
240 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_NGBE_TX_MAX_BURST);
241 ret = tx_xmit_pkts(tx_queue, &tx_pkts[nb_tx], n);
242 nb_tx = (uint16_t)(nb_tx + ret);
243 nb_pkts = (uint16_t)(nb_pkts - ret);
251 /*********************************************************************
255 **********************************************************************/
257 ngbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
260 struct ngbe_rx_queue *rxq;
261 volatile struct ngbe_rx_desc *rx_ring;
262 volatile struct ngbe_rx_desc *rxdp;
263 struct ngbe_rx_entry *sw_ring;
264 struct ngbe_rx_entry *rxe;
265 struct rte_mbuf *rxm;
266 struct rte_mbuf *nmb;
267 struct ngbe_rx_desc rxd;
278 rx_id = rxq->rx_tail;
279 rx_ring = rxq->rx_ring;
280 sw_ring = rxq->sw_ring;
281 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id];
282 while (nb_rx < nb_pkts) {
284 * The order of operations here is important as the DD status
285 * bit must not be read after any other descriptor fields.
286 * rx_ring and rxdp are pointing to volatile data so the order
287 * of accesses cannot be reordered by the compiler. If they were
288 * not volatile, they could be reordered which could lead to
289 * using invalid descriptor fields when read from rxd.
291 rxdp = &rx_ring[rx_id];
292 staterr = rxdp->qw1.lo.status;
293 if (!(staterr & rte_cpu_to_le_32(NGBE_RXD_STAT_DD)))
300 * If the NGBE_RXD_STAT_EOP flag is not set, the Rx packet
301 * is likely to be invalid and to be dropped by the various
302 * validation checks performed by the network stack.
304 * Allocate a new mbuf to replenish the RX ring descriptor.
305 * If the allocation fails:
306 * - arrange for that Rx descriptor to be the first one
307 * being parsed the next time the receive function is
308 * invoked [on the same queue].
310 * - Stop parsing the Rx ring and return immediately.
312 * This policy do not drop the packet received in the Rx
313 * descriptor for which the allocation of a new mbuf failed.
314 * Thus, it allows that packet to be later retrieved if
315 * mbuf have been freed in the mean time.
316 * As a side effect, holding Rx descriptors instead of
317 * systematically giving them back to the NIC may lead to
318 * Rx ring exhaustion situations.
319 * However, the NIC can gracefully prevent such situations
320 * to happen by sending specific "back-pressure" flow control
321 * frames to its peer(s).
324 "port_id=%u queue_id=%u rx_id=%u ext_err_stat=0x%08x pkt_len=%u",
325 (uint16_t)rxq->port_id, (uint16_t)rxq->queue_id,
326 (uint16_t)rx_id, (uint32_t)staterr,
327 (uint16_t)rte_le_to_cpu_16(rxd.qw1.hi.len));
329 nmb = rte_mbuf_raw_alloc(rxq->mb_pool);
332 "Rx mbuf alloc failed port_id=%u queue_id=%u",
333 (uint16_t)rxq->port_id,
334 (uint16_t)rxq->queue_id);
335 dev->data->rx_mbuf_alloc_failed++;
340 rxe = &sw_ring[rx_id];
342 if (rx_id == rxq->nb_rx_desc)
345 /* Prefetch next mbuf while processing current one. */
346 rte_ngbe_prefetch(sw_ring[rx_id].mbuf);
349 * When next Rx descriptor is on a cache-line boundary,
350 * prefetch the next 4 Rx descriptors and the next 8 pointers
353 if ((rx_id & 0x3) == 0) {
354 rte_ngbe_prefetch(&rx_ring[rx_id]);
355 rte_ngbe_prefetch(&sw_ring[rx_id]);
360 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
361 NGBE_RXD_HDRADDR(rxdp, 0);
362 NGBE_RXD_PKTADDR(rxdp, dma_addr);
365 * Initialize the returned mbuf.
366 * setup generic mbuf fields:
367 * - number of segments,
370 * - Rx port identifier.
372 pkt_len = (uint16_t)(rte_le_to_cpu_16(rxd.qw1.hi.len));
373 rxm->data_off = RTE_PKTMBUF_HEADROOM;
374 rte_packet_prefetch((char *)rxm->buf_addr + rxm->data_off);
377 rxm->pkt_len = pkt_len;
378 rxm->data_len = pkt_len;
379 rxm->port = rxq->port_id;
382 * Store the mbuf address into the next entry of the array
383 * of returned packets.
385 rx_pkts[nb_rx++] = rxm;
387 rxq->rx_tail = rx_id;
390 * If the number of free Rx descriptors is greater than the Rx free
391 * threshold of the queue, advance the Receive Descriptor Tail (RDT)
393 * Update the RDT with the value of the last processed Rx descriptor
394 * minus 1, to guarantee that the RDT register is never equal to the
395 * RDH register, which creates a "full" ring situation from the
396 * hardware point of view...
398 nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold);
399 if (nb_hold > rxq->rx_free_thresh) {
401 "port_id=%u queue_id=%u rx_tail=%u nb_hold=%u nb_rx=%u",
402 (uint16_t)rxq->port_id, (uint16_t)rxq->queue_id,
403 (uint16_t)rx_id, (uint16_t)nb_hold,
405 rx_id = (uint16_t)((rx_id == 0) ?
406 (rxq->nb_rx_desc - 1) : (rx_id - 1));
407 ngbe_set32(rxq->rdt_reg_addr, rx_id);
410 rxq->nb_rx_hold = nb_hold;
415 /*********************************************************************
417 * Queue management functions
419 **********************************************************************/
422 ngbe_tx_queue_release_mbufs(struct ngbe_tx_queue *txq)
426 if (txq->sw_ring != NULL) {
427 for (i = 0; i < txq->nb_tx_desc; i++) {
428 if (txq->sw_ring[i].mbuf != NULL) {
429 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
430 txq->sw_ring[i].mbuf = NULL;
437 ngbe_tx_free_swring(struct ngbe_tx_queue *txq)
440 rte_free(txq->sw_ring);
444 ngbe_tx_queue_release(struct ngbe_tx_queue *txq)
447 if (txq->ops != NULL) {
448 txq->ops->release_mbufs(txq);
449 txq->ops->free_swring(txq);
456 ngbe_dev_tx_queue_release(void *txq)
458 ngbe_tx_queue_release(txq);
461 /* (Re)set dynamic ngbe_tx_queue fields to defaults */
463 ngbe_reset_tx_queue(struct ngbe_tx_queue *txq)
465 static const struct ngbe_tx_desc zeroed_desc = {0};
466 struct ngbe_tx_entry *txe = txq->sw_ring;
469 /* Zero out HW ring memory */
470 for (i = 0; i < txq->nb_tx_desc; i++)
471 txq->tx_ring[i] = zeroed_desc;
473 /* Initialize SW ring entries */
474 prev = (uint16_t)(txq->nb_tx_desc - 1);
475 for (i = 0; i < txq->nb_tx_desc; i++) {
476 /* the ring can also be modified by hardware */
477 volatile struct ngbe_tx_desc *txd = &txq->tx_ring[i];
479 txd->dw3 = rte_cpu_to_le_32(NGBE_TXD_DD);
482 txe[prev].next_id = i;
486 txq->tx_next_dd = (uint16_t)(txq->tx_free_thresh - 1);
490 * Always allow 1 descriptor to be un-allocated to avoid
491 * a H/W race condition
493 txq->last_desc_cleaned = (uint16_t)(txq->nb_tx_desc - 1);
494 txq->nb_tx_free = (uint16_t)(txq->nb_tx_desc - 1);
496 memset((void *)&txq->ctx_cache, 0,
497 NGBE_CTX_NUM * sizeof(struct ngbe_ctx_info));
500 static const struct ngbe_txq_ops def_txq_ops = {
501 .release_mbufs = ngbe_tx_queue_release_mbufs,
502 .free_swring = ngbe_tx_free_swring,
503 .reset = ngbe_reset_tx_queue,
507 ngbe_dev_tx_queue_setup(struct rte_eth_dev *dev,
510 unsigned int socket_id,
511 const struct rte_eth_txconf *tx_conf)
513 const struct rte_memzone *tz;
514 struct ngbe_tx_queue *txq;
516 uint16_t tx_free_thresh;
518 PMD_INIT_FUNC_TRACE();
519 hw = ngbe_dev_hw(dev);
522 * The Tx descriptor ring will be cleaned after txq->tx_free_thresh
523 * descriptors are used or if the number of descriptors required
524 * to transmit a packet is greater than the number of free Tx
526 * One descriptor in the Tx ring is used as a sentinel to avoid a
527 * H/W race condition, hence the maximum threshold constraints.
528 * When set to zero use default values.
530 tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ?
531 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH);
532 if (tx_free_thresh >= (nb_desc - 3)) {
534 "tx_free_thresh must be less than the number of TX descriptors minus 3. (tx_free_thresh=%u port=%d queue=%d)",
535 (unsigned int)tx_free_thresh,
536 (int)dev->data->port_id, (int)queue_idx);
540 if (nb_desc % tx_free_thresh != 0) {
542 "tx_free_thresh must be a divisor of the number of Tx descriptors. (tx_free_thresh=%u port=%d queue=%d)",
543 (unsigned int)tx_free_thresh,
544 (int)dev->data->port_id, (int)queue_idx);
548 /* Free memory prior to re-allocation if needed... */
549 if (dev->data->tx_queues[queue_idx] != NULL) {
550 ngbe_tx_queue_release(dev->data->tx_queues[queue_idx]);
551 dev->data->tx_queues[queue_idx] = NULL;
554 /* First allocate the Tx queue data structure */
555 txq = rte_zmalloc_socket("ethdev Tx queue",
556 sizeof(struct ngbe_tx_queue),
557 RTE_CACHE_LINE_SIZE, socket_id);
562 * Allocate Tx ring hardware descriptors. A memzone large enough to
563 * handle the maximum ring size is allocated in order to allow for
564 * resizing in later calls to the queue setup function.
566 tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx,
567 sizeof(struct ngbe_tx_desc) * NGBE_RING_DESC_MAX,
568 NGBE_ALIGN, socket_id);
570 ngbe_tx_queue_release(txq);
574 txq->nb_tx_desc = nb_desc;
575 txq->tx_free_thresh = tx_free_thresh;
576 txq->pthresh = tx_conf->tx_thresh.pthresh;
577 txq->hthresh = tx_conf->tx_thresh.hthresh;
578 txq->wthresh = tx_conf->tx_thresh.wthresh;
579 txq->queue_id = queue_idx;
580 txq->reg_idx = queue_idx;
581 txq->port_id = dev->data->port_id;
582 txq->ops = &def_txq_ops;
583 txq->tx_deferred_start = tx_conf->tx_deferred_start;
585 txq->tdt_reg_addr = NGBE_REG_ADDR(hw, NGBE_TXWP(txq->reg_idx));
586 txq->tdc_reg_addr = NGBE_REG_ADDR(hw, NGBE_TXCFG(txq->reg_idx));
588 txq->tx_ring_phys_addr = TMZ_PADDR(tz);
589 txq->tx_ring = (struct ngbe_tx_desc *)TMZ_VADDR(tz);
591 /* Allocate software ring */
592 txq->sw_ring = rte_zmalloc_socket("txq->sw_ring",
593 sizeof(struct ngbe_tx_entry) * nb_desc,
594 RTE_CACHE_LINE_SIZE, socket_id);
595 if (txq->sw_ring == NULL) {
596 ngbe_tx_queue_release(txq);
600 "sw_ring=%p hw_ring=%p dma_addr=0x%" PRIx64,
601 txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr);
603 txq->ops->reset(txq);
605 dev->data->tx_queues[queue_idx] = txq;
611 * ngbe_free_sc_cluster - free the not-yet-completed scattered cluster
613 * The "next" pointer of the last segment of (not-yet-completed) RSC clusters
614 * in the sw_sc_ring is not set to NULL but rather points to the next
615 * mbuf of this RSC aggregation (that has not been completed yet and still
616 * resides on the HW ring). So, instead of calling for rte_pktmbuf_free() we
617 * will just free first "nb_segs" segments of the cluster explicitly by calling
618 * an rte_pktmbuf_free_seg().
620 * @m scattered cluster head
623 ngbe_free_sc_cluster(struct rte_mbuf *m)
625 uint16_t i, nb_segs = m->nb_segs;
626 struct rte_mbuf *next_seg;
628 for (i = 0; i < nb_segs; i++) {
630 rte_pktmbuf_free_seg(m);
636 ngbe_rx_queue_release_mbufs(struct ngbe_rx_queue *rxq)
640 if (rxq->sw_ring != NULL) {
641 for (i = 0; i < rxq->nb_rx_desc; i++) {
642 if (rxq->sw_ring[i].mbuf != NULL) {
643 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
644 rxq->sw_ring[i].mbuf = NULL;
647 for (i = 0; i < rxq->rx_nb_avail; ++i) {
650 mb = rxq->rx_stage[rxq->rx_next_avail + i];
651 rte_pktmbuf_free_seg(mb);
653 rxq->rx_nb_avail = 0;
656 if (rxq->sw_sc_ring != NULL)
657 for (i = 0; i < rxq->nb_rx_desc; i++)
658 if (rxq->sw_sc_ring[i].fbuf != NULL) {
659 ngbe_free_sc_cluster(rxq->sw_sc_ring[i].fbuf);
660 rxq->sw_sc_ring[i].fbuf = NULL;
665 ngbe_rx_queue_release(struct ngbe_rx_queue *rxq)
668 ngbe_rx_queue_release_mbufs(rxq);
669 rte_free(rxq->sw_ring);
670 rte_free(rxq->sw_sc_ring);
676 ngbe_dev_rx_queue_release(void *rxq)
678 ngbe_rx_queue_release(rxq);
682 * Check if Rx Burst Bulk Alloc function can be used.
684 * 0: the preconditions are satisfied and the bulk allocation function
686 * -EINVAL: the preconditions are NOT satisfied and the default Rx burst
687 * function must be used.
690 check_rx_burst_bulk_alloc_preconditions(struct ngbe_rx_queue *rxq)
695 * Make sure the following pre-conditions are satisfied:
696 * rxq->rx_free_thresh >= RTE_PMD_NGBE_RX_MAX_BURST
697 * rxq->rx_free_thresh < rxq->nb_rx_desc
698 * (rxq->nb_rx_desc % rxq->rx_free_thresh) == 0
699 * Scattered packets are not supported. This should be checked
700 * outside of this function.
702 if (rxq->rx_free_thresh < RTE_PMD_NGBE_RX_MAX_BURST) {
704 "Rx Burst Bulk Alloc Preconditions: rxq->rx_free_thresh=%d, RTE_PMD_NGBE_RX_MAX_BURST=%d",
705 rxq->rx_free_thresh, RTE_PMD_NGBE_RX_MAX_BURST);
707 } else if (rxq->rx_free_thresh >= rxq->nb_rx_desc) {
709 "Rx Burst Bulk Alloc Preconditions: rxq->rx_free_thresh=%d, rxq->nb_rx_desc=%d",
710 rxq->rx_free_thresh, rxq->nb_rx_desc);
712 } else if ((rxq->nb_rx_desc % rxq->rx_free_thresh) != 0) {
714 "Rx Burst Bulk Alloc Preconditions: rxq->nb_rx_desc=%d, rxq->rx_free_thresh=%d",
715 rxq->nb_rx_desc, rxq->rx_free_thresh);
722 /* Reset dynamic ngbe_rx_queue fields back to defaults */
724 ngbe_reset_rx_queue(struct ngbe_adapter *adapter, struct ngbe_rx_queue *rxq)
726 static const struct ngbe_rx_desc zeroed_desc = {
727 {{0}, {0} }, {{0}, {0} } };
729 uint16_t len = rxq->nb_rx_desc;
732 * By default, the Rx queue setup function allocates enough memory for
733 * NGBE_RING_DESC_MAX. The Rx Burst bulk allocation function requires
734 * extra memory at the end of the descriptor ring to be zero'd out.
736 if (adapter->rx_bulk_alloc_allowed)
737 /* zero out extra memory */
738 len += RTE_PMD_NGBE_RX_MAX_BURST;
741 * Zero out HW ring memory. Zero out extra memory at the end of
742 * the H/W ring so look-ahead logic in Rx Burst bulk alloc function
743 * reads extra memory as zeros.
745 for (i = 0; i < len; i++)
746 rxq->rx_ring[i] = zeroed_desc;
749 * initialize extra software ring entries. Space for these extra
750 * entries is always allocated
752 memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf));
753 for (i = rxq->nb_rx_desc; i < len; ++i)
754 rxq->sw_ring[i].mbuf = &rxq->fake_mbuf;
756 rxq->rx_nb_avail = 0;
757 rxq->rx_next_avail = 0;
758 rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1);
761 rxq->pkt_first_seg = NULL;
762 rxq->pkt_last_seg = NULL;
766 ngbe_dev_rx_queue_setup(struct rte_eth_dev *dev,
769 unsigned int socket_id,
770 const struct rte_eth_rxconf *rx_conf,
771 struct rte_mempool *mp)
773 const struct rte_memzone *rz;
774 struct ngbe_rx_queue *rxq;
777 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
779 PMD_INIT_FUNC_TRACE();
780 hw = ngbe_dev_hw(dev);
782 /* Free memory prior to re-allocation if needed... */
783 if (dev->data->rx_queues[queue_idx] != NULL) {
784 ngbe_rx_queue_release(dev->data->rx_queues[queue_idx]);
785 dev->data->rx_queues[queue_idx] = NULL;
788 /* First allocate the Rx queue data structure */
789 rxq = rte_zmalloc_socket("ethdev RX queue",
790 sizeof(struct ngbe_rx_queue),
791 RTE_CACHE_LINE_SIZE, socket_id);
795 rxq->nb_rx_desc = nb_desc;
796 rxq->rx_free_thresh = rx_conf->rx_free_thresh;
797 rxq->queue_id = queue_idx;
798 rxq->reg_idx = queue_idx;
799 rxq->port_id = dev->data->port_id;
800 rxq->drop_en = rx_conf->rx_drop_en;
801 rxq->rx_deferred_start = rx_conf->rx_deferred_start;
804 * Allocate Rx ring hardware descriptors. A memzone large enough to
805 * handle the maximum ring size is allocated in order to allow for
806 * resizing in later calls to the queue setup function.
808 rz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx,
809 RX_RING_SZ, NGBE_ALIGN, socket_id);
811 ngbe_rx_queue_release(rxq);
816 * Zero init all the descriptors in the ring.
818 memset(rz->addr, 0, RX_RING_SZ);
820 rxq->rdt_reg_addr = NGBE_REG_ADDR(hw, NGBE_RXWP(rxq->reg_idx));
821 rxq->rdh_reg_addr = NGBE_REG_ADDR(hw, NGBE_RXRP(rxq->reg_idx));
823 rxq->rx_ring_phys_addr = TMZ_PADDR(rz);
824 rxq->rx_ring = (struct ngbe_rx_desc *)TMZ_VADDR(rz);
827 * Certain constraints must be met in order to use the bulk buffer
828 * allocation Rx burst function. If any of Rx queues doesn't meet them
829 * the feature should be disabled for the whole port.
831 if (check_rx_burst_bulk_alloc_preconditions(rxq)) {
833 "queue[%d] doesn't meet Rx Bulk Alloc preconditions - canceling the feature for the whole port[%d]",
834 rxq->queue_id, rxq->port_id);
835 adapter->rx_bulk_alloc_allowed = false;
839 * Allocate software ring. Allow for space at the end of the
840 * S/W ring to make sure look-ahead logic in bulk alloc Rx burst
841 * function does not access an invalid memory region.
844 if (adapter->rx_bulk_alloc_allowed)
845 len += RTE_PMD_NGBE_RX_MAX_BURST;
847 rxq->sw_ring = rte_zmalloc_socket("rxq->sw_ring",
848 sizeof(struct ngbe_rx_entry) * len,
849 RTE_CACHE_LINE_SIZE, socket_id);
850 if (rxq->sw_ring == NULL) {
851 ngbe_rx_queue_release(rxq);
856 * Always allocate even if it's not going to be needed in order to
859 * This ring is used in Scattered Rx cases and Scattered Rx may
860 * be requested in ngbe_dev_rx_init(), which is called later from
864 rte_zmalloc_socket("rxq->sw_sc_ring",
865 sizeof(struct ngbe_scattered_rx_entry) * len,
866 RTE_CACHE_LINE_SIZE, socket_id);
867 if (rxq->sw_sc_ring == NULL) {
868 ngbe_rx_queue_release(rxq);
873 "sw_ring=%p sw_sc_ring=%p hw_ring=%p dma_addr=0x%" PRIx64,
874 rxq->sw_ring, rxq->sw_sc_ring, rxq->rx_ring,
875 rxq->rx_ring_phys_addr);
877 dev->data->rx_queues[queue_idx] = rxq;
879 ngbe_reset_rx_queue(adapter, rxq);
885 ngbe_dev_clear_queues(struct rte_eth_dev *dev)
888 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
890 PMD_INIT_FUNC_TRACE();
892 for (i = 0; i < dev->data->nb_tx_queues; i++) {
893 struct ngbe_tx_queue *txq = dev->data->tx_queues[i];
896 txq->ops->release_mbufs(txq);
897 txq->ops->reset(txq);
901 for (i = 0; i < dev->data->nb_rx_queues; i++) {
902 struct ngbe_rx_queue *rxq = dev->data->rx_queues[i];
905 ngbe_rx_queue_release_mbufs(rxq);
906 ngbe_reset_rx_queue(adapter, rxq);
912 ngbe_dev_free_queues(struct rte_eth_dev *dev)
916 PMD_INIT_FUNC_TRACE();
918 for (i = 0; i < dev->data->nb_rx_queues; i++) {
919 ngbe_dev_rx_queue_release(dev->data->rx_queues[i]);
920 dev->data->rx_queues[i] = NULL;
922 dev->data->nb_rx_queues = 0;
924 for (i = 0; i < dev->data->nb_tx_queues; i++) {
925 ngbe_dev_tx_queue_release(dev->data->tx_queues[i]);
926 dev->data->tx_queues[i] = NULL;
928 dev->data->nb_tx_queues = 0;
932 ngbe_alloc_rx_queue_mbufs(struct ngbe_rx_queue *rxq)
934 struct ngbe_rx_entry *rxe = rxq->sw_ring;
938 /* Initialize software ring entries */
939 for (i = 0; i < rxq->nb_rx_desc; i++) {
940 /* the ring can also be modified by hardware */
941 volatile struct ngbe_rx_desc *rxd;
942 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mb_pool);
945 PMD_INIT_LOG(ERR, "Rx mbuf alloc failed queue_id=%u port_id=%u",
946 (unsigned int)rxq->queue_id,
947 (unsigned int)rxq->port_id);
951 mbuf->data_off = RTE_PKTMBUF_HEADROOM;
952 mbuf->port = rxq->port_id;
955 rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf));
956 rxd = &rxq->rx_ring[i];
957 NGBE_RXD_HDRADDR(rxd, 0);
958 NGBE_RXD_PKTADDR(rxd, dma_addr);
966 * Initializes Receive Unit.
969 ngbe_dev_rx_init(struct rte_eth_dev *dev)
972 struct ngbe_rx_queue *rxq;
980 PMD_INIT_FUNC_TRACE();
981 hw = ngbe_dev_hw(dev);
984 * Make sure receives are disabled while setting
985 * up the Rx context (registers, descriptor rings, etc.).
987 wr32m(hw, NGBE_MACRXCFG, NGBE_MACRXCFG_ENA, 0);
988 wr32m(hw, NGBE_PBRXCTL, NGBE_PBRXCTL_ENA, 0);
990 /* Enable receipt of broadcasted frames */
991 fctrl = rd32(hw, NGBE_PSRCTL);
992 fctrl |= NGBE_PSRCTL_BCA;
993 wr32(hw, NGBE_PSRCTL, fctrl);
995 hlreg0 = rd32(hw, NGBE_SECRXCTL);
996 hlreg0 &= ~NGBE_SECRXCTL_XDSA;
997 wr32(hw, NGBE_SECRXCTL, hlreg0);
999 wr32m(hw, NGBE_FRMSZ, NGBE_FRMSZ_MAX_MASK,
1000 NGBE_FRMSZ_MAX(NGBE_FRAME_SIZE_DFT));
1002 /* Setup Rx queues */
1003 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1004 rxq = dev->data->rx_queues[i];
1006 /* Setup the Base and Length of the Rx Descriptor Rings */
1007 bus_addr = rxq->rx_ring_phys_addr;
1008 wr32(hw, NGBE_RXBAL(rxq->reg_idx),
1009 (uint32_t)(bus_addr & BIT_MASK32));
1010 wr32(hw, NGBE_RXBAH(rxq->reg_idx),
1011 (uint32_t)(bus_addr >> 32));
1012 wr32(hw, NGBE_RXRP(rxq->reg_idx), 0);
1013 wr32(hw, NGBE_RXWP(rxq->reg_idx), 0);
1015 srrctl = NGBE_RXCFG_RNGLEN(rxq->nb_rx_desc);
1017 /* Set if packets are dropped when no descriptors available */
1019 srrctl |= NGBE_RXCFG_DROP;
1022 * Configure the Rx buffer size in the PKTLEN field of
1023 * the RXCFG register of the queue.
1024 * The value is in 1 KB resolution. Valid values can be from
1027 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) -
1028 RTE_PKTMBUF_HEADROOM);
1029 buf_size = ROUND_DOWN(buf_size, 0x1 << 10);
1030 srrctl |= NGBE_RXCFG_PKTLEN(buf_size);
1032 wr32(hw, NGBE_RXCFG(rxq->reg_idx), srrctl);
1039 * Initializes Transmit Unit.
1042 ngbe_dev_tx_init(struct rte_eth_dev *dev)
1045 struct ngbe_tx_queue *txq;
1049 PMD_INIT_FUNC_TRACE();
1050 hw = ngbe_dev_hw(dev);
1052 wr32m(hw, NGBE_SECTXCTL, NGBE_SECTXCTL_ODSA, NGBE_SECTXCTL_ODSA);
1053 wr32m(hw, NGBE_SECTXCTL, NGBE_SECTXCTL_XDSA, 0);
1055 /* Setup the Base and Length of the Tx Descriptor Rings */
1056 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1057 txq = dev->data->tx_queues[i];
1059 bus_addr = txq->tx_ring_phys_addr;
1060 wr32(hw, NGBE_TXBAL(txq->reg_idx),
1061 (uint32_t)(bus_addr & BIT_MASK32));
1062 wr32(hw, NGBE_TXBAH(txq->reg_idx),
1063 (uint32_t)(bus_addr >> 32));
1064 wr32m(hw, NGBE_TXCFG(txq->reg_idx), NGBE_TXCFG_BUFLEN_MASK,
1065 NGBE_TXCFG_BUFLEN(txq->nb_tx_desc));
1066 /* Setup the HW Tx Head and TX Tail descriptor pointers */
1067 wr32(hw, NGBE_TXRP(txq->reg_idx), 0);
1068 wr32(hw, NGBE_TXWP(txq->reg_idx), 0);
1073 * Start Transmit and Receive Units.
1076 ngbe_dev_rxtx_start(struct rte_eth_dev *dev)
1079 struct ngbe_tx_queue *txq;
1080 struct ngbe_rx_queue *rxq;
1086 PMD_INIT_FUNC_TRACE();
1087 hw = ngbe_dev_hw(dev);
1089 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1090 txq = dev->data->tx_queues[i];
1091 /* Setup Transmit Threshold Registers */
1092 wr32m(hw, NGBE_TXCFG(txq->reg_idx),
1093 NGBE_TXCFG_HTHRESH_MASK |
1094 NGBE_TXCFG_WTHRESH_MASK,
1095 NGBE_TXCFG_HTHRESH(txq->hthresh) |
1096 NGBE_TXCFG_WTHRESH(txq->wthresh));
1099 dmatxctl = rd32(hw, NGBE_DMATXCTRL);
1100 dmatxctl |= NGBE_DMATXCTRL_ENA;
1101 wr32(hw, NGBE_DMATXCTRL, dmatxctl);
1103 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1104 txq = dev->data->tx_queues[i];
1105 if (txq->tx_deferred_start == 0) {
1106 ret = ngbe_dev_tx_queue_start(dev, i);
1112 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1113 rxq = dev->data->rx_queues[i];
1114 if (rxq->rx_deferred_start == 0) {
1115 ret = ngbe_dev_rx_queue_start(dev, i);
1121 /* Enable Receive engine */
1122 rxctrl = rd32(hw, NGBE_PBRXCTL);
1123 rxctrl |= NGBE_PBRXCTL_ENA;
1124 hw->mac.enable_rx_dma(hw, rxctrl);
1130 ngbe_dev_save_rx_queue(struct ngbe_hw *hw, uint16_t rx_queue_id)
1132 u32 *reg = &hw->q_rx_regs[rx_queue_id * 8];
1133 *(reg++) = rd32(hw, NGBE_RXBAL(rx_queue_id));
1134 *(reg++) = rd32(hw, NGBE_RXBAH(rx_queue_id));
1135 *(reg++) = rd32(hw, NGBE_RXCFG(rx_queue_id));
1139 ngbe_dev_store_rx_queue(struct ngbe_hw *hw, uint16_t rx_queue_id)
1141 u32 *reg = &hw->q_rx_regs[rx_queue_id * 8];
1142 wr32(hw, NGBE_RXBAL(rx_queue_id), *(reg++));
1143 wr32(hw, NGBE_RXBAH(rx_queue_id), *(reg++));
1144 wr32(hw, NGBE_RXCFG(rx_queue_id), *(reg++) & ~NGBE_RXCFG_ENA);
1148 ngbe_dev_save_tx_queue(struct ngbe_hw *hw, uint16_t tx_queue_id)
1150 u32 *reg = &hw->q_tx_regs[tx_queue_id * 8];
1151 *(reg++) = rd32(hw, NGBE_TXBAL(tx_queue_id));
1152 *(reg++) = rd32(hw, NGBE_TXBAH(tx_queue_id));
1153 *(reg++) = rd32(hw, NGBE_TXCFG(tx_queue_id));
1157 ngbe_dev_store_tx_queue(struct ngbe_hw *hw, uint16_t tx_queue_id)
1159 u32 *reg = &hw->q_tx_regs[tx_queue_id * 8];
1160 wr32(hw, NGBE_TXBAL(tx_queue_id), *(reg++));
1161 wr32(hw, NGBE_TXBAH(tx_queue_id), *(reg++));
1162 wr32(hw, NGBE_TXCFG(tx_queue_id), *(reg++) & ~NGBE_TXCFG_ENA);
1166 * Start Receive Units for specified queue.
1169 ngbe_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1171 struct ngbe_hw *hw = ngbe_dev_hw(dev);
1172 struct ngbe_rx_queue *rxq;
1176 PMD_INIT_FUNC_TRACE();
1178 rxq = dev->data->rx_queues[rx_queue_id];
1180 /* Allocate buffers for descriptor rings */
1181 if (ngbe_alloc_rx_queue_mbufs(rxq) != 0) {
1182 PMD_INIT_LOG(ERR, "Could not alloc mbuf for queue:%d",
1186 rxdctl = rd32(hw, NGBE_RXCFG(rxq->reg_idx));
1187 rxdctl |= NGBE_RXCFG_ENA;
1188 wr32(hw, NGBE_RXCFG(rxq->reg_idx), rxdctl);
1190 /* Wait until Rx Enable ready */
1191 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS;
1194 rxdctl = rd32(hw, NGBE_RXCFG(rxq->reg_idx));
1195 } while (--poll_ms && !(rxdctl & NGBE_RXCFG_ENA));
1197 PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", rx_queue_id);
1199 wr32(hw, NGBE_RXRP(rxq->reg_idx), 0);
1200 wr32(hw, NGBE_RXWP(rxq->reg_idx), rxq->nb_rx_desc - 1);
1201 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
1207 * Stop Receive Units for specified queue.
1210 ngbe_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1212 struct ngbe_hw *hw = ngbe_dev_hw(dev);
1213 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
1214 struct ngbe_rx_queue *rxq;
1218 PMD_INIT_FUNC_TRACE();
1220 rxq = dev->data->rx_queues[rx_queue_id];
1222 ngbe_dev_save_rx_queue(hw, rxq->reg_idx);
1223 wr32m(hw, NGBE_RXCFG(rxq->reg_idx), NGBE_RXCFG_ENA, 0);
1225 /* Wait until Rx Enable bit clear */
1226 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS;
1229 rxdctl = rd32(hw, NGBE_RXCFG(rxq->reg_idx));
1230 } while (--poll_ms && (rxdctl & NGBE_RXCFG_ENA));
1232 PMD_INIT_LOG(ERR, "Could not disable Rx Queue %d", rx_queue_id);
1234 rte_delay_us(RTE_NGBE_WAIT_100_US);
1235 ngbe_dev_store_rx_queue(hw, rxq->reg_idx);
1237 ngbe_rx_queue_release_mbufs(rxq);
1238 ngbe_reset_rx_queue(adapter, rxq);
1239 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
1245 * Start Transmit Units for specified queue.
1248 ngbe_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
1250 struct ngbe_hw *hw = ngbe_dev_hw(dev);
1251 struct ngbe_tx_queue *txq;
1255 PMD_INIT_FUNC_TRACE();
1257 txq = dev->data->tx_queues[tx_queue_id];
1258 wr32m(hw, NGBE_TXCFG(txq->reg_idx), NGBE_TXCFG_ENA, NGBE_TXCFG_ENA);
1260 /* Wait until Tx Enable ready */
1261 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS;
1264 txdctl = rd32(hw, NGBE_TXCFG(txq->reg_idx));
1265 } while (--poll_ms && !(txdctl & NGBE_TXCFG_ENA));
1267 PMD_INIT_LOG(ERR, "Could not enable Tx Queue %d",
1271 wr32(hw, NGBE_TXWP(txq->reg_idx), txq->tx_tail);
1272 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
1278 * Stop Transmit Units for specified queue.
1281 ngbe_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
1283 struct ngbe_hw *hw = ngbe_dev_hw(dev);
1284 struct ngbe_tx_queue *txq;
1286 uint32_t txtdh, txtdt;
1289 PMD_INIT_FUNC_TRACE();
1291 txq = dev->data->tx_queues[tx_queue_id];
1293 /* Wait until Tx queue is empty */
1294 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS;
1296 rte_delay_us(RTE_NGBE_WAIT_100_US);
1297 txtdh = rd32(hw, NGBE_TXRP(txq->reg_idx));
1298 txtdt = rd32(hw, NGBE_TXWP(txq->reg_idx));
1299 } while (--poll_ms && (txtdh != txtdt));
1301 PMD_INIT_LOG(ERR, "Tx Queue %d is not empty when stopping.",
1304 ngbe_dev_save_tx_queue(hw, txq->reg_idx);
1305 wr32m(hw, NGBE_TXCFG(txq->reg_idx), NGBE_TXCFG_ENA, 0);
1307 /* Wait until Tx Enable bit clear */
1308 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS;
1311 txdctl = rd32(hw, NGBE_TXCFG(txq->reg_idx));
1312 } while (--poll_ms && (txdctl & NGBE_TXCFG_ENA));
1314 PMD_INIT_LOG(ERR, "Could not disable Tx Queue %d",
1317 rte_delay_us(RTE_NGBE_WAIT_100_US);
1318 ngbe_dev_store_tx_queue(hw, txq->reg_idx);
1320 if (txq->ops != NULL) {
1321 txq->ops->release_mbufs(txq);
1322 txq->ops->reset(txq);
1324 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;