1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018-2021 Beijing WangXun Technology Co., Ltd.
3 * Copyright(c) 2010-2017 Intel Corporation
9 #include <rte_ethdev.h>
10 #include <ethdev_driver.h>
11 #include <rte_malloc.h>
14 #include "ngbe_logs.h"
15 #include "base/ngbe.h"
16 #include "ngbe_ethdev.h"
17 #include "ngbe_rxtx.h"
19 #ifdef RTE_LIBRTE_IEEE1588
20 #define NGBE_TX_IEEE1588_TMST PKT_TX_IEEE1588_TMST
22 #define NGBE_TX_IEEE1588_TMST 0
25 /* Bit Mask to indicate what bits required for building Tx context */
26 static const u64 NGBE_TX_OFFLOAD_MASK = (RTE_MBUF_F_TX_IP_CKSUM |
27 RTE_MBUF_F_TX_OUTER_IPV6 |
28 RTE_MBUF_F_TX_OUTER_IPV4 |
32 RTE_MBUF_F_TX_L4_MASK |
33 RTE_MBUF_F_TX_TCP_SEG |
34 RTE_MBUF_F_TX_TUNNEL_MASK |
35 RTE_MBUF_F_TX_OUTER_IP_CKSUM |
36 NGBE_TX_IEEE1588_TMST);
38 #define NGBE_TX_OFFLOAD_NOTSUP_MASK \
39 (RTE_MBUF_F_TX_OFFLOAD_MASK ^ NGBE_TX_OFFLOAD_MASK)
42 * Prefetch a cache line into all cache levels.
44 #define rte_ngbe_prefetch(p) rte_prefetch0(p)
46 /*********************************************************************
50 **********************************************************************/
53 * Check for descriptors with their DD bit set and free mbufs.
54 * Return the total number of buffers freed.
56 static __rte_always_inline int
57 ngbe_tx_free_bufs(struct ngbe_tx_queue *txq)
59 struct ngbe_tx_entry *txep;
62 struct rte_mbuf *m, *free[RTE_NGBE_TX_MAX_FREE_BUF_SZ];
64 /* check DD bit on threshold descriptor */
65 status = txq->tx_ring[txq->tx_next_dd].dw3;
66 if (!(status & rte_cpu_to_le_32(NGBE_TXD_DD))) {
67 if (txq->nb_tx_free >> 1 < txq->tx_free_thresh)
68 ngbe_set32_masked(txq->tdc_reg_addr,
69 NGBE_TXCFG_FLUSH, NGBE_TXCFG_FLUSH);
74 * first buffer to free from S/W ring is at index
75 * tx_next_dd - (tx_free_thresh-1)
77 txep = &txq->sw_ring[txq->tx_next_dd - (txq->tx_free_thresh - 1)];
78 for (i = 0; i < txq->tx_free_thresh; ++i, ++txep) {
79 /* free buffers one at a time */
80 m = rte_pktmbuf_prefree_seg(txep->mbuf);
83 if (unlikely(m == NULL))
86 if (nb_free >= RTE_NGBE_TX_MAX_FREE_BUF_SZ ||
87 (nb_free > 0 && m->pool != free[0]->pool)) {
88 rte_mempool_put_bulk(free[0]->pool,
89 (void **)free, nb_free);
97 rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free);
99 /* buffers were freed, update counters */
100 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_free_thresh);
101 txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_free_thresh);
102 if (txq->tx_next_dd >= txq->nb_tx_desc)
103 txq->tx_next_dd = (uint16_t)(txq->tx_free_thresh - 1);
105 return txq->tx_free_thresh;
108 /* Populate 4 descriptors with data from 4 mbufs */
110 tx4(volatile struct ngbe_tx_desc *txdp, struct rte_mbuf **pkts)
112 uint64_t buf_dma_addr;
116 for (i = 0; i < 4; ++i, ++txdp, ++pkts) {
117 buf_dma_addr = rte_mbuf_data_iova(*pkts);
118 pkt_len = (*pkts)->data_len;
120 /* write data to descriptor */
121 txdp->qw0 = rte_cpu_to_le_64(buf_dma_addr);
122 txdp->dw2 = cpu_to_le32(NGBE_TXD_FLAGS |
123 NGBE_TXD_DATLEN(pkt_len));
124 txdp->dw3 = cpu_to_le32(NGBE_TXD_PAYLEN(pkt_len));
126 rte_prefetch0(&(*pkts)->pool);
130 /* Populate 1 descriptor with data from 1 mbuf */
132 tx1(volatile struct ngbe_tx_desc *txdp, struct rte_mbuf **pkts)
134 uint64_t buf_dma_addr;
137 buf_dma_addr = rte_mbuf_data_iova(*pkts);
138 pkt_len = (*pkts)->data_len;
140 /* write data to descriptor */
141 txdp->qw0 = cpu_to_le64(buf_dma_addr);
142 txdp->dw2 = cpu_to_le32(NGBE_TXD_FLAGS |
143 NGBE_TXD_DATLEN(pkt_len));
144 txdp->dw3 = cpu_to_le32(NGBE_TXD_PAYLEN(pkt_len));
146 rte_prefetch0(&(*pkts)->pool);
150 * Fill H/W descriptor ring with mbuf data.
151 * Copy mbuf pointers to the S/W ring.
154 ngbe_tx_fill_hw_ring(struct ngbe_tx_queue *txq, struct rte_mbuf **pkts,
157 volatile struct ngbe_tx_desc *txdp = &txq->tx_ring[txq->tx_tail];
158 struct ngbe_tx_entry *txep = &txq->sw_ring[txq->tx_tail];
159 const int N_PER_LOOP = 4;
160 const int N_PER_LOOP_MASK = N_PER_LOOP - 1;
161 int mainpart, leftover;
165 * Process most of the packets in chunks of N pkts. Any
166 * leftover packets will get processed one at a time.
168 mainpart = (nb_pkts & ((uint32_t)~N_PER_LOOP_MASK));
169 leftover = (nb_pkts & ((uint32_t)N_PER_LOOP_MASK));
170 for (i = 0; i < mainpart; i += N_PER_LOOP) {
171 /* Copy N mbuf pointers to the S/W ring */
172 for (j = 0; j < N_PER_LOOP; ++j)
173 (txep + i + j)->mbuf = *(pkts + i + j);
174 tx4(txdp + i, pkts + i);
177 if (unlikely(leftover > 0)) {
178 for (i = 0; i < leftover; ++i) {
179 (txep + mainpart + i)->mbuf = *(pkts + mainpart + i);
180 tx1(txdp + mainpart + i, pkts + mainpart + i);
185 static inline uint16_t
186 tx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
189 struct ngbe_tx_queue *txq = (struct ngbe_tx_queue *)tx_queue;
193 * Begin scanning the H/W ring for done descriptors when the
194 * number of available descriptors drops below tx_free_thresh.
195 * For each done descriptor, free the associated buffer.
197 if (txq->nb_tx_free < txq->tx_free_thresh)
198 ngbe_tx_free_bufs(txq);
200 /* Only use descriptors that are available */
201 nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts);
202 if (unlikely(nb_pkts == 0))
205 /* Use exactly nb_pkts descriptors */
206 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts);
209 * At this point, we know there are enough descriptors in the
210 * ring to transmit all the packets. This assumes that each
211 * mbuf contains a single segment, and that no new offloads
212 * are expected, which would require a new context descriptor.
216 * See if we're going to wrap-around. If so, handle the top
217 * of the descriptor ring first, then do the bottom. If not,
218 * the processing looks just like the "bottom" part anyway...
220 if ((txq->tx_tail + nb_pkts) > txq->nb_tx_desc) {
221 n = (uint16_t)(txq->nb_tx_desc - txq->tx_tail);
222 ngbe_tx_fill_hw_ring(txq, tx_pkts, n);
226 /* Fill H/W descriptor ring with mbuf data */
227 ngbe_tx_fill_hw_ring(txq, tx_pkts + n, (uint16_t)(nb_pkts - n));
228 txq->tx_tail = (uint16_t)(txq->tx_tail + (nb_pkts - n));
231 * Check for wrap-around. This would only happen if we used
232 * up to the last descriptor in the ring, no more, no less.
234 if (txq->tx_tail >= txq->nb_tx_desc)
237 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u",
238 (uint16_t)txq->port_id, (uint16_t)txq->queue_id,
239 (uint16_t)txq->tx_tail, (uint16_t)nb_pkts);
241 /* update tail pointer */
243 ngbe_set32_relaxed(txq->tdt_reg_addr, txq->tx_tail);
249 ngbe_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts,
254 /* Try to transmit at least chunks of TX_MAX_BURST pkts */
255 if (likely(nb_pkts <= RTE_PMD_NGBE_TX_MAX_BURST))
256 return tx_xmit_pkts(tx_queue, tx_pkts, nb_pkts);
258 /* transmit more than the max burst, in chunks of TX_MAX_BURST */
260 while (nb_pkts != 0) {
263 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_NGBE_TX_MAX_BURST);
264 ret = tx_xmit_pkts(tx_queue, &tx_pkts[nb_tx], n);
265 nb_tx = (uint16_t)(nb_tx + ret);
266 nb_pkts = (uint16_t)(nb_pkts - ret);
275 ngbe_set_xmit_ctx(struct ngbe_tx_queue *txq,
276 volatile struct ngbe_tx_ctx_desc *ctx_txd,
277 uint64_t ol_flags, union ngbe_tx_offload tx_offload)
279 union ngbe_tx_offload tx_offload_mask;
280 uint32_t type_tucmd_mlhl;
281 uint32_t mss_l4len_idx;
283 uint32_t vlan_macip_lens;
284 uint32_t tunnel_seed;
286 ctx_idx = txq->ctx_curr;
287 tx_offload_mask.data[0] = 0;
288 tx_offload_mask.data[1] = 0;
290 /* Specify which HW CTX to upload. */
291 mss_l4len_idx = NGBE_TXD_IDX(ctx_idx);
292 type_tucmd_mlhl = NGBE_TXD_CTXT;
294 tx_offload_mask.ptid |= ~0;
295 type_tucmd_mlhl |= NGBE_TXD_PTID(tx_offload.ptid);
297 /* check if TCP segmentation required for this packet */
298 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {
299 tx_offload_mask.l2_len |= ~0;
300 tx_offload_mask.l3_len |= ~0;
301 tx_offload_mask.l4_len |= ~0;
302 tx_offload_mask.tso_segsz |= ~0;
303 mss_l4len_idx |= NGBE_TXD_MSS(tx_offload.tso_segsz);
304 mss_l4len_idx |= NGBE_TXD_L4LEN(tx_offload.l4_len);
305 } else { /* no TSO, check if hardware checksum is needed */
306 if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM) {
307 tx_offload_mask.l2_len |= ~0;
308 tx_offload_mask.l3_len |= ~0;
311 switch (ol_flags & RTE_MBUF_F_TX_L4_MASK) {
312 case RTE_MBUF_F_TX_UDP_CKSUM:
314 NGBE_TXD_L4LEN(sizeof(struct rte_udp_hdr));
315 tx_offload_mask.l2_len |= ~0;
316 tx_offload_mask.l3_len |= ~0;
318 case RTE_MBUF_F_TX_TCP_CKSUM:
320 NGBE_TXD_L4LEN(sizeof(struct rte_tcp_hdr));
321 tx_offload_mask.l2_len |= ~0;
322 tx_offload_mask.l3_len |= ~0;
324 case RTE_MBUF_F_TX_SCTP_CKSUM:
326 NGBE_TXD_L4LEN(sizeof(struct rte_sctp_hdr));
327 tx_offload_mask.l2_len |= ~0;
328 tx_offload_mask.l3_len |= ~0;
335 vlan_macip_lens = NGBE_TXD_IPLEN(tx_offload.l3_len >> 1);
337 if (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) {
338 tx_offload_mask.outer_tun_len |= ~0;
339 tx_offload_mask.outer_l2_len |= ~0;
340 tx_offload_mask.outer_l3_len |= ~0;
341 tx_offload_mask.l2_len |= ~0;
342 tunnel_seed = NGBE_TXD_ETUNLEN(tx_offload.outer_tun_len >> 1);
343 tunnel_seed |= NGBE_TXD_EIPLEN(tx_offload.outer_l3_len >> 2);
345 switch (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) {
346 case RTE_MBUF_F_TX_TUNNEL_IPIP:
347 /* for non UDP / GRE tunneling, set to 0b */
350 PMD_TX_LOG(ERR, "Tunnel type not supported");
353 vlan_macip_lens |= NGBE_TXD_MACLEN(tx_offload.outer_l2_len);
356 vlan_macip_lens |= NGBE_TXD_MACLEN(tx_offload.l2_len);
359 if (ol_flags & RTE_MBUF_F_TX_VLAN) {
360 tx_offload_mask.vlan_tci |= ~0;
361 vlan_macip_lens |= NGBE_TXD_VLAN(tx_offload.vlan_tci);
364 txq->ctx_cache[ctx_idx].flags = ol_flags;
365 txq->ctx_cache[ctx_idx].tx_offload.data[0] =
366 tx_offload_mask.data[0] & tx_offload.data[0];
367 txq->ctx_cache[ctx_idx].tx_offload.data[1] =
368 tx_offload_mask.data[1] & tx_offload.data[1];
369 txq->ctx_cache[ctx_idx].tx_offload_mask = tx_offload_mask;
371 ctx_txd->dw0 = rte_cpu_to_le_32(vlan_macip_lens);
372 ctx_txd->dw1 = rte_cpu_to_le_32(tunnel_seed);
373 ctx_txd->dw2 = rte_cpu_to_le_32(type_tucmd_mlhl);
374 ctx_txd->dw3 = rte_cpu_to_le_32(mss_l4len_idx);
378 * Check which hardware context can be used. Use the existing match
379 * or create a new context descriptor.
381 static inline uint32_t
382 what_ctx_update(struct ngbe_tx_queue *txq, uint64_t flags,
383 union ngbe_tx_offload tx_offload)
385 /* If match with the current used context */
386 if (likely(txq->ctx_cache[txq->ctx_curr].flags == flags &&
387 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] ==
388 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0]
389 & tx_offload.data[0])) &&
390 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] ==
391 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1]
392 & tx_offload.data[1]))))
393 return txq->ctx_curr;
395 /* What if match with the next context */
397 if (likely(txq->ctx_cache[txq->ctx_curr].flags == flags &&
398 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] ==
399 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0]
400 & tx_offload.data[0])) &&
401 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] ==
402 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1]
403 & tx_offload.data[1]))))
404 return txq->ctx_curr;
406 /* Mismatch, use the previous context */
410 static inline uint32_t
411 tx_desc_cksum_flags_to_olinfo(uint64_t ol_flags)
415 if ((ol_flags & RTE_MBUF_F_TX_L4_MASK) != RTE_MBUF_F_TX_L4_NO_CKSUM) {
417 tmp |= NGBE_TXD_L4CS;
419 if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM) {
421 tmp |= NGBE_TXD_IPCS;
423 if (ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) {
425 tmp |= NGBE_TXD_EIPCS;
427 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {
429 /* implies IPv4 cksum */
430 if (ol_flags & RTE_MBUF_F_TX_IPV4)
431 tmp |= NGBE_TXD_IPCS;
432 tmp |= NGBE_TXD_L4CS;
434 if (ol_flags & RTE_MBUF_F_TX_VLAN)
440 static inline uint32_t
441 tx_desc_ol_flags_to_cmdtype(uint64_t ol_flags)
443 uint32_t cmdtype = 0;
445 if (ol_flags & RTE_MBUF_F_TX_VLAN)
446 cmdtype |= NGBE_TXD_VLE;
447 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG)
448 cmdtype |= NGBE_TXD_TSE;
452 static inline uint8_t
453 tx_desc_ol_flags_to_ptid(uint64_t oflags, uint32_t ptype)
458 return ngbe_encode_ptype(ptype);
460 /* Only support flags in NGBE_TX_OFFLOAD_MASK */
461 tun = !!(oflags & RTE_MBUF_F_TX_TUNNEL_MASK);
464 ptype = RTE_PTYPE_L2_ETHER;
465 if (oflags & RTE_MBUF_F_TX_VLAN)
466 ptype |= RTE_PTYPE_L2_ETHER_VLAN;
469 if (oflags & (RTE_MBUF_F_TX_OUTER_IPV4 | RTE_MBUF_F_TX_OUTER_IP_CKSUM))
470 ptype |= RTE_PTYPE_L3_IPV4;
471 else if (oflags & (RTE_MBUF_F_TX_OUTER_IPV6))
472 ptype |= RTE_PTYPE_L3_IPV6;
474 if (oflags & (RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_IP_CKSUM))
475 ptype |= (tun ? RTE_PTYPE_INNER_L3_IPV4 : RTE_PTYPE_L3_IPV4);
476 else if (oflags & (RTE_MBUF_F_TX_IPV6))
477 ptype |= (tun ? RTE_PTYPE_INNER_L3_IPV6 : RTE_PTYPE_L3_IPV6);
480 switch (oflags & (RTE_MBUF_F_TX_L4_MASK)) {
481 case RTE_MBUF_F_TX_TCP_CKSUM:
482 ptype |= (tun ? RTE_PTYPE_INNER_L4_TCP : RTE_PTYPE_L4_TCP);
484 case RTE_MBUF_F_TX_UDP_CKSUM:
485 ptype |= (tun ? RTE_PTYPE_INNER_L4_UDP : RTE_PTYPE_L4_UDP);
487 case RTE_MBUF_F_TX_SCTP_CKSUM:
488 ptype |= (tun ? RTE_PTYPE_INNER_L4_SCTP : RTE_PTYPE_L4_SCTP);
492 if (oflags & RTE_MBUF_F_TX_TCP_SEG)
493 ptype |= (tun ? RTE_PTYPE_INNER_L4_TCP : RTE_PTYPE_L4_TCP);
496 switch (oflags & RTE_MBUF_F_TX_TUNNEL_MASK) {
497 case RTE_MBUF_F_TX_TUNNEL_IPIP:
498 case RTE_MBUF_F_TX_TUNNEL_IP:
499 ptype |= RTE_PTYPE_L2_ETHER |
505 return ngbe_encode_ptype(ptype);
508 /* Reset transmit descriptors after they have been used */
510 ngbe_xmit_cleanup(struct ngbe_tx_queue *txq)
512 struct ngbe_tx_entry *sw_ring = txq->sw_ring;
513 volatile struct ngbe_tx_desc *txr = txq->tx_ring;
514 uint16_t last_desc_cleaned = txq->last_desc_cleaned;
515 uint16_t nb_tx_desc = txq->nb_tx_desc;
516 uint16_t desc_to_clean_to;
517 uint16_t nb_tx_to_clean;
520 /* Determine the last descriptor needing to be cleaned */
521 desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->tx_free_thresh);
522 if (desc_to_clean_to >= nb_tx_desc)
523 desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc);
525 /* Check to make sure the last descriptor to clean is done */
526 desc_to_clean_to = sw_ring[desc_to_clean_to].last_id;
527 status = txr[desc_to_clean_to].dw3;
528 if (!(status & rte_cpu_to_le_32(NGBE_TXD_DD))) {
530 "Tx descriptor %4u is not done"
531 "(port=%d queue=%d)",
533 txq->port_id, txq->queue_id);
534 if (txq->nb_tx_free >> 1 < txq->tx_free_thresh)
535 ngbe_set32_masked(txq->tdc_reg_addr,
536 NGBE_TXCFG_FLUSH, NGBE_TXCFG_FLUSH);
537 /* Failed to clean any descriptors, better luck next time */
541 /* Figure out how many descriptors will be cleaned */
542 if (last_desc_cleaned > desc_to_clean_to)
543 nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) +
546 nb_tx_to_clean = (uint16_t)(desc_to_clean_to -
550 "Cleaning %4u Tx descriptors: %4u to %4u (port=%d queue=%d)",
551 nb_tx_to_clean, last_desc_cleaned, desc_to_clean_to,
552 txq->port_id, txq->queue_id);
555 * The last descriptor to clean is done, so that means all the
556 * descriptors from the last descriptor that was cleaned
557 * up to the last descriptor with the RS bit set
558 * are done. Only reset the threshold descriptor.
560 txr[desc_to_clean_to].dw3 = 0;
562 /* Update the txq to reflect the last descriptor that was cleaned */
563 txq->last_desc_cleaned = desc_to_clean_to;
564 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + nb_tx_to_clean);
571 ngbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
574 struct ngbe_tx_queue *txq;
575 struct ngbe_tx_entry *sw_ring;
576 struct ngbe_tx_entry *txe, *txn;
577 volatile struct ngbe_tx_desc *txr;
578 volatile struct ngbe_tx_desc *txd;
579 struct rte_mbuf *tx_pkt;
580 struct rte_mbuf *m_seg;
581 uint64_t buf_dma_addr;
582 uint32_t olinfo_status;
583 uint32_t cmd_type_len;
594 union ngbe_tx_offload tx_offload;
596 tx_offload.data[0] = 0;
597 tx_offload.data[1] = 0;
599 sw_ring = txq->sw_ring;
601 tx_id = txq->tx_tail;
602 txe = &sw_ring[tx_id];
604 /* Determine if the descriptor ring needs to be cleaned. */
605 if (txq->nb_tx_free < txq->tx_free_thresh)
606 ngbe_xmit_cleanup(txq);
608 rte_prefetch0(&txe->mbuf->pool);
611 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
614 pkt_len = tx_pkt->pkt_len;
617 * Determine how many (if any) context descriptors
618 * are needed for offload functionality.
620 ol_flags = tx_pkt->ol_flags;
622 /* If hardware offload required */
623 tx_ol_req = ol_flags & NGBE_TX_OFFLOAD_MASK;
625 tx_offload.ptid = tx_desc_ol_flags_to_ptid(tx_ol_req,
626 tx_pkt->packet_type);
627 tx_offload.l2_len = tx_pkt->l2_len;
628 tx_offload.l3_len = tx_pkt->l3_len;
629 tx_offload.l4_len = tx_pkt->l4_len;
630 tx_offload.vlan_tci = tx_pkt->vlan_tci;
631 tx_offload.tso_segsz = tx_pkt->tso_segsz;
632 tx_offload.outer_l2_len = tx_pkt->outer_l2_len;
633 tx_offload.outer_l3_len = tx_pkt->outer_l3_len;
634 tx_offload.outer_tun_len = 0;
636 /* If new context need be built or reuse the exist ctx*/
637 ctx = what_ctx_update(txq, tx_ol_req, tx_offload);
638 /* Only allocate context descriptor if required */
639 new_ctx = (ctx == NGBE_CTX_NUM);
644 * Keep track of how many descriptors are used this loop
645 * This will always be the number of segments + the number of
646 * Context descriptors required to transmit the packet
648 nb_used = (uint16_t)(tx_pkt->nb_segs + new_ctx);
651 * The number of descriptors that must be allocated for a
652 * packet is the number of segments of that packet, plus 1
653 * Context Descriptor for the hardware offload, if any.
654 * Determine the last Tx descriptor to allocate in the Tx ring
655 * for the packet, starting from the current position (tx_id)
658 tx_last = (uint16_t)(tx_id + nb_used - 1);
661 if (tx_last >= txq->nb_tx_desc)
662 tx_last = (uint16_t)(tx_last - txq->nb_tx_desc);
664 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u pktlen=%u"
665 " tx_first=%u tx_last=%u",
666 (uint16_t)txq->port_id,
667 (uint16_t)txq->queue_id,
673 * Make sure there are enough Tx descriptors available to
674 * transmit the entire packet.
675 * nb_used better be less than or equal to txq->tx_free_thresh
677 if (nb_used > txq->nb_tx_free) {
679 "Not enough free Tx descriptors "
680 "nb_used=%4u nb_free=%4u "
681 "(port=%d queue=%d)",
682 nb_used, txq->nb_tx_free,
683 txq->port_id, txq->queue_id);
685 if (ngbe_xmit_cleanup(txq) != 0) {
686 /* Could not clean any descriptors */
692 /* nb_used better be <= txq->tx_free_thresh */
693 if (unlikely(nb_used > txq->tx_free_thresh)) {
695 "The number of descriptors needed to "
696 "transmit the packet exceeds the "
697 "RS bit threshold. This will impact "
699 "nb_used=%4u nb_free=%4u "
700 "tx_free_thresh=%4u. "
701 "(port=%d queue=%d)",
702 nb_used, txq->nb_tx_free,
704 txq->port_id, txq->queue_id);
706 * Loop here until there are enough Tx
707 * descriptors or until the ring cannot be
710 while (nb_used > txq->nb_tx_free) {
711 if (ngbe_xmit_cleanup(txq) != 0) {
713 * Could not clean any
725 * By now there are enough free Tx descriptors to transmit
730 * Set common flags of all Tx Data Descriptors.
732 * The following bits must be set in the first Data Descriptor
733 * and are ignored in the other ones:
736 * The following bits must only be set in the last Data
740 cmd_type_len = NGBE_TXD_FCS;
742 #ifdef RTE_LIBRTE_IEEE1588
743 if (ol_flags & PKT_TX_IEEE1588_TMST)
744 cmd_type_len |= NGBE_TXD_1588;
749 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {
750 /* when TSO is on, paylen in descriptor is the
751 * not the packet len but the tcp payload len
753 pkt_len -= (tx_offload.l2_len +
754 tx_offload.l3_len + tx_offload.l4_len);
756 (tx_pkt->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK)
757 ? tx_offload.outer_l2_len +
758 tx_offload.outer_l3_len : 0;
762 * Setup the Tx Context Descriptor if required
765 volatile struct ngbe_tx_ctx_desc *ctx_txd;
767 ctx_txd = (volatile struct ngbe_tx_ctx_desc *)
770 txn = &sw_ring[txe->next_id];
771 rte_prefetch0(&txn->mbuf->pool);
773 if (txe->mbuf != NULL) {
774 rte_pktmbuf_free_seg(txe->mbuf);
778 ngbe_set_xmit_ctx(txq, ctx_txd, tx_ol_req,
781 txe->last_id = tx_last;
782 tx_id = txe->next_id;
787 * Setup the Tx Data Descriptor,
788 * This path will go through
789 * whatever new/reuse the context descriptor
791 cmd_type_len |= tx_desc_ol_flags_to_cmdtype(ol_flags);
793 tx_desc_cksum_flags_to_olinfo(ol_flags);
794 olinfo_status |= NGBE_TXD_IDX(ctx);
797 olinfo_status |= NGBE_TXD_PAYLEN(pkt_len);
802 txn = &sw_ring[txe->next_id];
803 rte_prefetch0(&txn->mbuf->pool);
805 if (txe->mbuf != NULL)
806 rte_pktmbuf_free_seg(txe->mbuf);
810 * Set up Transmit Data Descriptor.
812 slen = m_seg->data_len;
813 buf_dma_addr = rte_mbuf_data_iova(m_seg);
814 txd->qw0 = rte_cpu_to_le_64(buf_dma_addr);
815 txd->dw2 = rte_cpu_to_le_32(cmd_type_len | slen);
816 txd->dw3 = rte_cpu_to_le_32(olinfo_status);
817 txe->last_id = tx_last;
818 tx_id = txe->next_id;
821 } while (m_seg != NULL);
824 * The last packet data descriptor needs End Of Packet (EOP)
826 cmd_type_len |= NGBE_TXD_EOP;
827 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_used);
829 txd->dw2 |= rte_cpu_to_le_32(cmd_type_len);
837 * Set the Transmit Descriptor Tail (TDT)
839 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u",
840 (uint16_t)txq->port_id, (uint16_t)txq->queue_id,
841 (uint16_t)tx_id, (uint16_t)nb_tx);
842 ngbe_set32_relaxed(txq->tdt_reg_addr, tx_id);
843 txq->tx_tail = tx_id;
848 /*********************************************************************
852 **********************************************************************/
854 ngbe_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
859 struct ngbe_tx_queue *txq = (struct ngbe_tx_queue *)tx_queue;
861 for (i = 0; i < nb_pkts; i++) {
863 ol_flags = m->ol_flags;
866 * Check if packet meets requirements for number of segments
868 * NOTE: for ngbe it's always (40 - WTHRESH) for both TSO and
872 if (m->nb_segs > NGBE_TX_MAX_SEG - txq->wthresh) {
877 if (ol_flags & NGBE_TX_OFFLOAD_NOTSUP_MASK) {
878 rte_errno = -ENOTSUP;
882 #ifdef RTE_ETHDEV_DEBUG_TX
883 ret = rte_validate_tx_offload(m);
889 ret = rte_net_intel_cksum_prepare(m);
899 /*********************************************************************
903 **********************************************************************/
904 static inline uint32_t
905 ngbe_rxd_pkt_info_to_pkt_type(uint32_t pkt_info, uint16_t ptid_mask)
907 uint16_t ptid = NGBE_RXD_PTID(pkt_info);
911 return ngbe_decode_ptype(ptid);
914 static inline uint64_t
915 ngbe_rxd_pkt_info_to_pkt_flags(uint32_t pkt_info)
917 static uint64_t ip_rss_types_map[16] __rte_cache_aligned = {
918 0, RTE_MBUF_F_RX_RSS_HASH, RTE_MBUF_F_RX_RSS_HASH, RTE_MBUF_F_RX_RSS_HASH,
919 0, RTE_MBUF_F_RX_RSS_HASH, 0, RTE_MBUF_F_RX_RSS_HASH,
920 RTE_MBUF_F_RX_RSS_HASH, 0, 0, 0,
921 0, 0, 0, RTE_MBUF_F_RX_FDIR,
923 #ifdef RTE_LIBRTE_IEEE1588
924 static uint64_t ip_pkt_etqf_map[8] = {
925 0, 0, 0, PKT_RX_IEEE1588_PTP,
928 int etfid = ngbe_etflt_id(NGBE_RXD_PTID(pkt_info));
929 if (likely(-1 != etfid))
930 return ip_pkt_etqf_map[etfid] |
931 ip_rss_types_map[NGBE_RXD_RSSTYPE(pkt_info)];
933 return ip_rss_types_map[NGBE_RXD_RSSTYPE(pkt_info)];
935 return ip_rss_types_map[NGBE_RXD_RSSTYPE(pkt_info)];
939 static inline uint64_t
940 rx_desc_status_to_pkt_flags(uint32_t rx_status, uint64_t vlan_flags)
945 * Check if VLAN present only.
946 * Do not check whether L3/L4 rx checksum done by NIC or not,
947 * That can be found from rte_eth_rxmode.offloads flag
949 pkt_flags = (rx_status & NGBE_RXD_STAT_VLAN &&
950 vlan_flags & RTE_MBUF_F_RX_VLAN_STRIPPED)
953 #ifdef RTE_LIBRTE_IEEE1588
954 if (rx_status & NGBE_RXD_STAT_1588)
955 pkt_flags = pkt_flags | PKT_RX_IEEE1588_TMST;
960 static inline uint64_t
961 rx_desc_error_to_pkt_flags(uint32_t rx_status)
963 uint64_t pkt_flags = 0;
965 /* checksum offload can't be disabled */
966 if (rx_status & NGBE_RXD_STAT_IPCS)
967 pkt_flags |= (rx_status & NGBE_RXD_ERR_IPCS
968 ? RTE_MBUF_F_RX_IP_CKSUM_BAD : RTE_MBUF_F_RX_IP_CKSUM_GOOD);
970 if (rx_status & NGBE_RXD_STAT_L4CS)
971 pkt_flags |= (rx_status & NGBE_RXD_ERR_L4CS
972 ? RTE_MBUF_F_RX_L4_CKSUM_BAD : RTE_MBUF_F_RX_L4_CKSUM_GOOD);
974 if (rx_status & NGBE_RXD_STAT_EIPCS &&
975 rx_status & NGBE_RXD_ERR_EIPCS)
976 pkt_flags |= RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD;
982 * LOOK_AHEAD defines how many desc statuses to check beyond the
983 * current descriptor.
984 * It must be a pound define for optimal performance.
985 * Do not change the value of LOOK_AHEAD, as the ngbe_rx_scan_hw_ring
986 * function only works with LOOK_AHEAD=8.
989 #if (LOOK_AHEAD != 8)
990 #error "PMD NGBE: LOOK_AHEAD must be 8\n"
993 ngbe_rx_scan_hw_ring(struct ngbe_rx_queue *rxq)
995 volatile struct ngbe_rx_desc *rxdp;
996 struct ngbe_rx_entry *rxep;
1001 uint32_t s[LOOK_AHEAD];
1002 uint32_t pkt_info[LOOK_AHEAD];
1003 int i, j, nb_rx = 0;
1006 /* get references to current descriptor and S/W ring entry */
1007 rxdp = &rxq->rx_ring[rxq->rx_tail];
1008 rxep = &rxq->sw_ring[rxq->rx_tail];
1010 status = rxdp->qw1.lo.status;
1011 /* check to make sure there is at least 1 packet to receive */
1012 if (!(status & rte_cpu_to_le_32(NGBE_RXD_STAT_DD)))
1016 * Scan LOOK_AHEAD descriptors at a time to determine which descriptors
1017 * reference packets that are ready to be received.
1019 for (i = 0; i < RTE_PMD_NGBE_RX_MAX_BURST;
1020 i += LOOK_AHEAD, rxdp += LOOK_AHEAD, rxep += LOOK_AHEAD) {
1021 /* Read desc statuses backwards to avoid race condition */
1022 for (j = 0; j < LOOK_AHEAD; j++)
1023 s[j] = rte_le_to_cpu_32(rxdp[j].qw1.lo.status);
1025 rte_atomic_thread_fence(__ATOMIC_ACQUIRE);
1027 /* Compute how many status bits were set */
1028 for (nb_dd = 0; nb_dd < LOOK_AHEAD &&
1029 (s[nb_dd] & NGBE_RXD_STAT_DD); nb_dd++)
1032 for (j = 0; j < nb_dd; j++)
1033 pkt_info[j] = rte_le_to_cpu_32(rxdp[j].qw0.dw0);
1037 /* Translate descriptor info to mbuf format */
1038 for (j = 0; j < nb_dd; ++j) {
1040 pkt_len = rte_le_to_cpu_16(rxdp[j].qw1.hi.len) -
1042 mb->data_len = pkt_len;
1043 mb->pkt_len = pkt_len;
1044 mb->vlan_tci = rte_le_to_cpu_16(rxdp[j].qw1.hi.tag);
1046 /* convert descriptor fields to rte mbuf flags */
1047 pkt_flags = rx_desc_status_to_pkt_flags(s[j],
1049 pkt_flags |= rx_desc_error_to_pkt_flags(s[j]);
1051 ngbe_rxd_pkt_info_to_pkt_flags(pkt_info[j]);
1052 mb->ol_flags = pkt_flags;
1054 ngbe_rxd_pkt_info_to_pkt_type(pkt_info[j],
1057 if (likely(pkt_flags & RTE_MBUF_F_RX_RSS_HASH))
1059 rte_le_to_cpu_32(rxdp[j].qw0.dw1);
1062 /* Move mbuf pointers from the S/W ring to the stage */
1063 for (j = 0; j < LOOK_AHEAD; ++j)
1064 rxq->rx_stage[i + j] = rxep[j].mbuf;
1066 /* stop if all requested packets could not be received */
1067 if (nb_dd != LOOK_AHEAD)
1071 /* clear software ring entries so we can cleanup correctly */
1072 for (i = 0; i < nb_rx; ++i)
1073 rxq->sw_ring[rxq->rx_tail + i].mbuf = NULL;
1079 ngbe_rx_alloc_bufs(struct ngbe_rx_queue *rxq, bool reset_mbuf)
1081 volatile struct ngbe_rx_desc *rxdp;
1082 struct ngbe_rx_entry *rxep;
1083 struct rte_mbuf *mb;
1088 /* allocate buffers in bulk directly into the S/W ring */
1089 alloc_idx = rxq->rx_free_trigger - (rxq->rx_free_thresh - 1);
1090 rxep = &rxq->sw_ring[alloc_idx];
1091 diag = rte_mempool_get_bulk(rxq->mb_pool, (void *)rxep,
1092 rxq->rx_free_thresh);
1093 if (unlikely(diag != 0))
1096 rxdp = &rxq->rx_ring[alloc_idx];
1097 for (i = 0; i < rxq->rx_free_thresh; ++i) {
1098 /* populate the static rte mbuf fields */
1101 mb->port = rxq->port_id;
1103 rte_mbuf_refcnt_set(mb, 1);
1104 mb->data_off = RTE_PKTMBUF_HEADROOM;
1106 /* populate the descriptors */
1107 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mb));
1108 NGBE_RXD_HDRADDR(&rxdp[i], 0);
1109 NGBE_RXD_PKTADDR(&rxdp[i], dma_addr);
1112 /* update state of internal queue structure */
1113 rxq->rx_free_trigger = rxq->rx_free_trigger + rxq->rx_free_thresh;
1114 if (rxq->rx_free_trigger >= rxq->nb_rx_desc)
1115 rxq->rx_free_trigger = rxq->rx_free_thresh - 1;
1121 static inline uint16_t
1122 ngbe_rx_fill_from_stage(struct ngbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
1125 struct rte_mbuf **stage = &rxq->rx_stage[rxq->rx_next_avail];
1128 /* how many packets are ready to return? */
1129 nb_pkts = (uint16_t)RTE_MIN(nb_pkts, rxq->rx_nb_avail);
1131 /* copy mbuf pointers to the application's packet list */
1132 for (i = 0; i < nb_pkts; ++i)
1133 rx_pkts[i] = stage[i];
1135 /* update internal queue state */
1136 rxq->rx_nb_avail = (uint16_t)(rxq->rx_nb_avail - nb_pkts);
1137 rxq->rx_next_avail = (uint16_t)(rxq->rx_next_avail + nb_pkts);
1142 static inline uint16_t
1143 ngbe_rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
1146 struct ngbe_rx_queue *rxq = (struct ngbe_rx_queue *)rx_queue;
1147 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id];
1150 /* Any previously recv'd pkts will be returned from the Rx stage */
1151 if (rxq->rx_nb_avail)
1152 return ngbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
1154 /* Scan the H/W ring for packets to receive */
1155 nb_rx = (uint16_t)ngbe_rx_scan_hw_ring(rxq);
1157 /* update internal queue state */
1158 rxq->rx_next_avail = 0;
1159 rxq->rx_nb_avail = nb_rx;
1160 rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_rx);
1162 /* if required, allocate new buffers to replenish descriptors */
1163 if (rxq->rx_tail > rxq->rx_free_trigger) {
1164 uint16_t cur_free_trigger = rxq->rx_free_trigger;
1166 if (ngbe_rx_alloc_bufs(rxq, true) != 0) {
1169 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u "
1170 "queue_id=%u", (uint16_t)rxq->port_id,
1171 (uint16_t)rxq->queue_id);
1173 dev->data->rx_mbuf_alloc_failed +=
1174 rxq->rx_free_thresh;
1177 * Need to rewind any previous receives if we cannot
1178 * allocate new buffers to replenish the old ones.
1180 rxq->rx_nb_avail = 0;
1181 rxq->rx_tail = (uint16_t)(rxq->rx_tail - nb_rx);
1182 for (i = 0, j = rxq->rx_tail; i < nb_rx; ++i, ++j)
1183 rxq->sw_ring[j].mbuf = rxq->rx_stage[i];
1188 /* update tail pointer */
1190 ngbe_set32_relaxed(rxq->rdt_reg_addr, cur_free_trigger);
1193 if (rxq->rx_tail >= rxq->nb_rx_desc)
1196 /* received any packets this loop? */
1197 if (rxq->rx_nb_avail)
1198 return ngbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
1203 /* split requests into chunks of size RTE_PMD_NGBE_RX_MAX_BURST */
1205 ngbe_recv_pkts_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
1210 if (unlikely(nb_pkts == 0))
1213 if (likely(nb_pkts <= RTE_PMD_NGBE_RX_MAX_BURST))
1214 return ngbe_rx_recv_pkts(rx_queue, rx_pkts, nb_pkts);
1216 /* request is relatively large, chunk it up */
1221 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_NGBE_RX_MAX_BURST);
1222 ret = ngbe_rx_recv_pkts(rx_queue, &rx_pkts[nb_rx], n);
1223 nb_rx = (uint16_t)(nb_rx + ret);
1224 nb_pkts = (uint16_t)(nb_pkts - ret);
1233 ngbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
1236 struct ngbe_rx_queue *rxq;
1237 volatile struct ngbe_rx_desc *rx_ring;
1238 volatile struct ngbe_rx_desc *rxdp;
1239 struct ngbe_rx_entry *sw_ring;
1240 struct ngbe_rx_entry *rxe;
1241 struct rte_mbuf *rxm;
1242 struct rte_mbuf *nmb;
1243 struct ngbe_rx_desc rxd;
1256 rx_id = rxq->rx_tail;
1257 rx_ring = rxq->rx_ring;
1258 sw_ring = rxq->sw_ring;
1259 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id];
1260 while (nb_rx < nb_pkts) {
1262 * The order of operations here is important as the DD status
1263 * bit must not be read after any other descriptor fields.
1264 * rx_ring and rxdp are pointing to volatile data so the order
1265 * of accesses cannot be reordered by the compiler. If they were
1266 * not volatile, they could be reordered which could lead to
1267 * using invalid descriptor fields when read from rxd.
1269 rxdp = &rx_ring[rx_id];
1270 staterr = rxdp->qw1.lo.status;
1271 if (!(staterr & rte_cpu_to_le_32(NGBE_RXD_STAT_DD)))
1278 * If the NGBE_RXD_STAT_EOP flag is not set, the Rx packet
1279 * is likely to be invalid and to be dropped by the various
1280 * validation checks performed by the network stack.
1282 * Allocate a new mbuf to replenish the RX ring descriptor.
1283 * If the allocation fails:
1284 * - arrange for that Rx descriptor to be the first one
1285 * being parsed the next time the receive function is
1286 * invoked [on the same queue].
1288 * - Stop parsing the Rx ring and return immediately.
1290 * This policy do not drop the packet received in the Rx
1291 * descriptor for which the allocation of a new mbuf failed.
1292 * Thus, it allows that packet to be later retrieved if
1293 * mbuf have been freed in the mean time.
1294 * As a side effect, holding Rx descriptors instead of
1295 * systematically giving them back to the NIC may lead to
1296 * Rx ring exhaustion situations.
1297 * However, the NIC can gracefully prevent such situations
1298 * to happen by sending specific "back-pressure" flow control
1299 * frames to its peer(s).
1302 "port_id=%u queue_id=%u rx_id=%u ext_err_stat=0x%08x pkt_len=%u",
1303 (uint16_t)rxq->port_id, (uint16_t)rxq->queue_id,
1304 (uint16_t)rx_id, (uint32_t)staterr,
1305 (uint16_t)rte_le_to_cpu_16(rxd.qw1.hi.len));
1307 nmb = rte_mbuf_raw_alloc(rxq->mb_pool);
1310 "Rx mbuf alloc failed port_id=%u queue_id=%u",
1311 (uint16_t)rxq->port_id,
1312 (uint16_t)rxq->queue_id);
1313 dev->data->rx_mbuf_alloc_failed++;
1318 rxe = &sw_ring[rx_id];
1320 if (rx_id == rxq->nb_rx_desc)
1323 /* Prefetch next mbuf while processing current one. */
1324 rte_ngbe_prefetch(sw_ring[rx_id].mbuf);
1327 * When next Rx descriptor is on a cache-line boundary,
1328 * prefetch the next 4 Rx descriptors and the next 8 pointers
1331 if ((rx_id & 0x3) == 0) {
1332 rte_ngbe_prefetch(&rx_ring[rx_id]);
1333 rte_ngbe_prefetch(&sw_ring[rx_id]);
1338 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
1339 NGBE_RXD_HDRADDR(rxdp, 0);
1340 NGBE_RXD_PKTADDR(rxdp, dma_addr);
1343 * Initialize the returned mbuf.
1344 * 1) setup generic mbuf fields:
1345 * - number of segments,
1348 * - Rx port identifier.
1349 * 2) integrate hardware offload data, if any:
1350 * - RSS flag & hash,
1351 * - IP checksum flag,
1352 * - VLAN TCI, if any,
1355 pkt_len = (uint16_t)(rte_le_to_cpu_16(rxd.qw1.hi.len) -
1357 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1358 rte_packet_prefetch((char *)rxm->buf_addr + rxm->data_off);
1361 rxm->pkt_len = pkt_len;
1362 rxm->data_len = pkt_len;
1363 rxm->port = rxq->port_id;
1365 pkt_info = rte_le_to_cpu_32(rxd.qw0.dw0);
1366 /* Only valid if RTE_MBUF_F_RX_VLAN set in pkt_flags */
1367 rxm->vlan_tci = rte_le_to_cpu_16(rxd.qw1.hi.tag);
1369 pkt_flags = rx_desc_status_to_pkt_flags(staterr,
1371 pkt_flags |= rx_desc_error_to_pkt_flags(staterr);
1372 pkt_flags |= ngbe_rxd_pkt_info_to_pkt_flags(pkt_info);
1373 rxm->ol_flags = pkt_flags;
1374 rxm->packet_type = ngbe_rxd_pkt_info_to_pkt_type(pkt_info,
1377 if (likely(pkt_flags & RTE_MBUF_F_RX_RSS_HASH))
1378 rxm->hash.rss = rte_le_to_cpu_32(rxd.qw0.dw1);
1381 * Store the mbuf address into the next entry of the array
1382 * of returned packets.
1384 rx_pkts[nb_rx++] = rxm;
1386 rxq->rx_tail = rx_id;
1389 * If the number of free Rx descriptors is greater than the Rx free
1390 * threshold of the queue, advance the Receive Descriptor Tail (RDT)
1392 * Update the RDT with the value of the last processed Rx descriptor
1393 * minus 1, to guarantee that the RDT register is never equal to the
1394 * RDH register, which creates a "full" ring situation from the
1395 * hardware point of view...
1397 nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold);
1398 if (nb_hold > rxq->rx_free_thresh) {
1400 "port_id=%u queue_id=%u rx_tail=%u nb_hold=%u nb_rx=%u",
1401 (uint16_t)rxq->port_id, (uint16_t)rxq->queue_id,
1402 (uint16_t)rx_id, (uint16_t)nb_hold,
1404 rx_id = (uint16_t)((rx_id == 0) ?
1405 (rxq->nb_rx_desc - 1) : (rx_id - 1));
1406 ngbe_set32(rxq->rdt_reg_addr, rx_id);
1409 rxq->nb_rx_hold = nb_hold;
1414 * ngbe_fill_cluster_head_buf - fill the first mbuf of the returned packet
1416 * Fill the following info in the HEAD buffer of the Rx cluster:
1417 * - RX port identifier
1418 * - hardware offload data, if any:
1420 * - IP checksum flag
1421 * - VLAN TCI, if any
1423 * @head HEAD of the packet cluster
1424 * @desc HW descriptor to get data from
1425 * @rxq Pointer to the Rx queue
1428 ngbe_fill_cluster_head_buf(struct rte_mbuf *head, struct ngbe_rx_desc *desc,
1429 struct ngbe_rx_queue *rxq, uint32_t staterr)
1434 head->port = rxq->port_id;
1436 /* The vlan_tci field is only valid when RTE_MBUF_F_RX_VLAN is
1437 * set in the pkt_flags field.
1439 head->vlan_tci = rte_le_to_cpu_16(desc->qw1.hi.tag);
1440 pkt_info = rte_le_to_cpu_32(desc->qw0.dw0);
1441 pkt_flags = rx_desc_status_to_pkt_flags(staterr, rxq->vlan_flags);
1442 pkt_flags |= rx_desc_error_to_pkt_flags(staterr);
1443 pkt_flags |= ngbe_rxd_pkt_info_to_pkt_flags(pkt_info);
1444 head->ol_flags = pkt_flags;
1445 head->packet_type = ngbe_rxd_pkt_info_to_pkt_type(pkt_info,
1448 if (likely(pkt_flags & RTE_MBUF_F_RX_RSS_HASH))
1449 head->hash.rss = rte_le_to_cpu_32(desc->qw0.dw1);
1453 * ngbe_recv_pkts_sc - receive handler for scatter case.
1455 * @rx_queue Rx queue handle
1456 * @rx_pkts table of received packets
1457 * @nb_pkts size of rx_pkts table
1458 * @bulk_alloc if TRUE bulk allocation is used for a HW ring refilling
1460 * Returns the number of received packets/clusters (according to the "bulk
1461 * receive" interface).
1463 static inline uint16_t
1464 ngbe_recv_pkts_sc(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts,
1467 struct ngbe_rx_queue *rxq = rx_queue;
1468 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id];
1469 volatile struct ngbe_rx_desc *rx_ring = rxq->rx_ring;
1470 struct ngbe_rx_entry *sw_ring = rxq->sw_ring;
1471 struct ngbe_scattered_rx_entry *sw_sc_ring = rxq->sw_sc_ring;
1472 uint16_t rx_id = rxq->rx_tail;
1474 uint16_t nb_hold = rxq->nb_rx_hold;
1475 uint16_t prev_id = rxq->rx_tail;
1477 while (nb_rx < nb_pkts) {
1479 struct ngbe_rx_entry *rxe;
1480 struct ngbe_scattered_rx_entry *sc_entry;
1481 struct ngbe_scattered_rx_entry *next_sc_entry = NULL;
1482 struct ngbe_rx_entry *next_rxe = NULL;
1483 struct rte_mbuf *first_seg;
1484 struct rte_mbuf *rxm;
1485 struct rte_mbuf *nmb = NULL;
1486 struct ngbe_rx_desc rxd;
1489 volatile struct ngbe_rx_desc *rxdp;
1493 rxdp = &rx_ring[rx_id];
1494 staterr = rte_le_to_cpu_32(rxdp->qw1.lo.status);
1496 if (!(staterr & NGBE_RXD_STAT_DD))
1501 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u "
1502 "staterr=0x%x data_len=%u",
1503 rxq->port_id, rxq->queue_id, rx_id, staterr,
1504 rte_le_to_cpu_16(rxd.qw1.hi.len));
1507 nmb = rte_mbuf_raw_alloc(rxq->mb_pool);
1509 PMD_RX_LOG(DEBUG, "Rx mbuf alloc failed "
1510 "port_id=%u queue_id=%u",
1511 rxq->port_id, rxq->queue_id);
1513 dev->data->rx_mbuf_alloc_failed++;
1516 } else if (nb_hold > rxq->rx_free_thresh) {
1517 uint16_t next_rdt = rxq->rx_free_trigger;
1519 if (!ngbe_rx_alloc_bufs(rxq, false)) {
1521 ngbe_set32_relaxed(rxq->rdt_reg_addr,
1523 nb_hold -= rxq->rx_free_thresh;
1525 PMD_RX_LOG(DEBUG, "Rx bulk alloc failed "
1526 "port_id=%u queue_id=%u",
1527 rxq->port_id, rxq->queue_id);
1529 dev->data->rx_mbuf_alloc_failed++;
1535 rxe = &sw_ring[rx_id];
1536 eop = staterr & NGBE_RXD_STAT_EOP;
1538 next_id = rx_id + 1;
1539 if (next_id == rxq->nb_rx_desc)
1542 /* Prefetch next mbuf while processing current one. */
1543 rte_ngbe_prefetch(sw_ring[next_id].mbuf);
1546 * When next Rx descriptor is on a cache-line boundary,
1547 * prefetch the next 4 RX descriptors and the next 4 pointers
1550 if ((next_id & 0x3) == 0) {
1551 rte_ngbe_prefetch(&rx_ring[next_id]);
1552 rte_ngbe_prefetch(&sw_ring[next_id]);
1559 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
1561 * Update Rx descriptor with the physical address of the
1562 * new data buffer of the new allocated mbuf.
1566 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1567 NGBE_RXD_HDRADDR(rxdp, 0);
1568 NGBE_RXD_PKTADDR(rxdp, dma);
1574 * Set data length & data buffer address of mbuf.
1576 data_len = rte_le_to_cpu_16(rxd.qw1.hi.len);
1577 rxm->data_len = data_len;
1583 next_sc_entry = &sw_sc_ring[nextp_id];
1584 next_rxe = &sw_ring[nextp_id];
1585 rte_ngbe_prefetch(next_rxe);
1588 sc_entry = &sw_sc_ring[rx_id];
1589 first_seg = sc_entry->fbuf;
1590 sc_entry->fbuf = NULL;
1593 * If this is the first buffer of the received packet,
1594 * set the pointer to the first mbuf of the packet and
1595 * initialize its context.
1596 * Otherwise, update the total length and the number of segments
1597 * of the current scattered packet, and update the pointer to
1598 * the last mbuf of the current packet.
1600 if (first_seg == NULL) {
1602 first_seg->pkt_len = data_len;
1603 first_seg->nb_segs = 1;
1605 first_seg->pkt_len += data_len;
1606 first_seg->nb_segs++;
1613 * If this is not the last buffer of the received packet, update
1614 * the pointer to the first mbuf at the NEXTP entry in the
1615 * sw_sc_ring and continue to parse the Rx ring.
1617 if (!eop && next_rxe) {
1618 rxm->next = next_rxe->mbuf;
1619 next_sc_entry->fbuf = first_seg;
1623 /* Initialize the first mbuf of the returned packet */
1624 ngbe_fill_cluster_head_buf(first_seg, &rxd, rxq, staterr);
1626 /* Deal with the case, when HW CRC srip is disabled. */
1627 first_seg->pkt_len -= rxq->crc_len;
1628 if (unlikely(rxm->data_len <= rxq->crc_len)) {
1629 struct rte_mbuf *lp;
1631 for (lp = first_seg; lp->next != rxm; lp = lp->next)
1634 first_seg->nb_segs--;
1635 lp->data_len -= rxq->crc_len - rxm->data_len;
1637 rte_pktmbuf_free_seg(rxm);
1639 rxm->data_len -= rxq->crc_len;
1642 /* Prefetch data of first segment, if configured to do so. */
1643 rte_packet_prefetch((char *)first_seg->buf_addr +
1644 first_seg->data_off);
1647 * Store the mbuf address into the next entry of the array
1648 * of returned packets.
1650 rx_pkts[nb_rx++] = first_seg;
1654 * Record index of the next Rx descriptor to probe.
1656 rxq->rx_tail = rx_id;
1659 * If the number of free Rx descriptors is greater than the Rx free
1660 * threshold of the queue, advance the Receive Descriptor Tail (RDT)
1662 * Update the RDT with the value of the last processed Rx descriptor
1663 * minus 1, to guarantee that the RDT register is never equal to the
1664 * RDH register, which creates a "full" ring situation from the
1665 * hardware point of view...
1667 if (!bulk_alloc && nb_hold > rxq->rx_free_thresh) {
1668 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u "
1669 "nb_hold=%u nb_rx=%u",
1670 rxq->port_id, rxq->queue_id, rx_id, nb_hold, nb_rx);
1673 ngbe_set32_relaxed(rxq->rdt_reg_addr, prev_id);
1677 rxq->nb_rx_hold = nb_hold;
1682 ngbe_recv_pkts_sc_single_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
1685 return ngbe_recv_pkts_sc(rx_queue, rx_pkts, nb_pkts, false);
1689 ngbe_recv_pkts_sc_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
1692 return ngbe_recv_pkts_sc(rx_queue, rx_pkts, nb_pkts, true);
1695 /*********************************************************************
1697 * Queue management functions
1699 **********************************************************************/
1702 ngbe_tx_queue_release_mbufs(struct ngbe_tx_queue *txq)
1706 if (txq->sw_ring != NULL) {
1707 for (i = 0; i < txq->nb_tx_desc; i++) {
1708 if (txq->sw_ring[i].mbuf != NULL) {
1709 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
1710 txq->sw_ring[i].mbuf = NULL;
1717 ngbe_tx_done_cleanup_full(struct ngbe_tx_queue *txq, uint32_t free_cnt)
1719 struct ngbe_tx_entry *swr_ring = txq->sw_ring;
1720 uint16_t i, tx_last, tx_id;
1721 uint16_t nb_tx_free_last;
1722 uint16_t nb_tx_to_clean;
1725 /* Start free mbuf from the next of tx_tail */
1726 tx_last = txq->tx_tail;
1727 tx_id = swr_ring[tx_last].next_id;
1729 if (txq->nb_tx_free == 0 && ngbe_xmit_cleanup(txq))
1732 nb_tx_to_clean = txq->nb_tx_free;
1733 nb_tx_free_last = txq->nb_tx_free;
1735 free_cnt = txq->nb_tx_desc;
1737 /* Loop through swr_ring to count the amount of
1738 * freeable mubfs and packets.
1740 for (pkt_cnt = 0; pkt_cnt < free_cnt; ) {
1741 for (i = 0; i < nb_tx_to_clean &&
1742 pkt_cnt < free_cnt &&
1743 tx_id != tx_last; i++) {
1744 if (swr_ring[tx_id].mbuf != NULL) {
1745 rte_pktmbuf_free_seg(swr_ring[tx_id].mbuf);
1746 swr_ring[tx_id].mbuf = NULL;
1749 * last segment in the packet,
1750 * increment packet count
1752 pkt_cnt += (swr_ring[tx_id].last_id == tx_id);
1755 tx_id = swr_ring[tx_id].next_id;
1758 if (pkt_cnt < free_cnt) {
1759 if (ngbe_xmit_cleanup(txq))
1762 nb_tx_to_clean = txq->nb_tx_free - nb_tx_free_last;
1763 nb_tx_free_last = txq->nb_tx_free;
1767 return (int)pkt_cnt;
1771 ngbe_tx_done_cleanup_simple(struct ngbe_tx_queue *txq,
1776 if (free_cnt == 0 || free_cnt > txq->nb_tx_desc)
1777 free_cnt = txq->nb_tx_desc;
1779 cnt = free_cnt - free_cnt % txq->tx_free_thresh;
1781 for (i = 0; i < cnt; i += n) {
1782 if (txq->nb_tx_desc - txq->nb_tx_free < txq->tx_free_thresh)
1785 n = ngbe_tx_free_bufs(txq);
1795 ngbe_dev_tx_done_cleanup(void *tx_queue, uint32_t free_cnt)
1797 struct ngbe_tx_queue *txq = (struct ngbe_tx_queue *)tx_queue;
1798 if (txq->offloads == 0 &&
1799 txq->tx_free_thresh >= RTE_PMD_NGBE_TX_MAX_BURST)
1800 return ngbe_tx_done_cleanup_simple(txq, free_cnt);
1802 return ngbe_tx_done_cleanup_full(txq, free_cnt);
1806 ngbe_tx_free_swring(struct ngbe_tx_queue *txq)
1809 rte_free(txq->sw_ring);
1813 ngbe_tx_queue_release(struct ngbe_tx_queue *txq)
1816 if (txq->ops != NULL) {
1817 txq->ops->release_mbufs(txq);
1818 txq->ops->free_swring(txq);
1825 ngbe_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
1827 ngbe_tx_queue_release(dev->data->tx_queues[qid]);
1830 /* (Re)set dynamic ngbe_tx_queue fields to defaults */
1832 ngbe_reset_tx_queue(struct ngbe_tx_queue *txq)
1834 static const struct ngbe_tx_desc zeroed_desc = {0};
1835 struct ngbe_tx_entry *txe = txq->sw_ring;
1838 /* Zero out HW ring memory */
1839 for (i = 0; i < txq->nb_tx_desc; i++)
1840 txq->tx_ring[i] = zeroed_desc;
1842 /* Initialize SW ring entries */
1843 prev = (uint16_t)(txq->nb_tx_desc - 1);
1844 for (i = 0; i < txq->nb_tx_desc; i++) {
1845 /* the ring can also be modified by hardware */
1846 volatile struct ngbe_tx_desc *txd = &txq->tx_ring[i];
1848 txd->dw3 = rte_cpu_to_le_32(NGBE_TXD_DD);
1851 txe[prev].next_id = i;
1855 txq->tx_next_dd = (uint16_t)(txq->tx_free_thresh - 1);
1859 * Always allow 1 descriptor to be un-allocated to avoid
1860 * a H/W race condition
1862 txq->last_desc_cleaned = (uint16_t)(txq->nb_tx_desc - 1);
1863 txq->nb_tx_free = (uint16_t)(txq->nb_tx_desc - 1);
1865 memset((void *)&txq->ctx_cache, 0,
1866 NGBE_CTX_NUM * sizeof(struct ngbe_ctx_info));
1869 static const struct ngbe_txq_ops def_txq_ops = {
1870 .release_mbufs = ngbe_tx_queue_release_mbufs,
1871 .free_swring = ngbe_tx_free_swring,
1872 .reset = ngbe_reset_tx_queue,
1875 /* Takes an ethdev and a queue and sets up the tx function to be used based on
1876 * the queue parameters. Used in tx_queue_setup by primary process and then
1877 * in dev_init by secondary process when attaching to an existing ethdev.
1880 ngbe_set_tx_function(struct rte_eth_dev *dev, struct ngbe_tx_queue *txq)
1882 /* Use a simple Tx queue (no offloads, no multi segs) if possible */
1883 if (txq->offloads == 0 &&
1884 txq->tx_free_thresh >= RTE_PMD_NGBE_TX_MAX_BURST) {
1885 PMD_INIT_LOG(DEBUG, "Using simple tx code path");
1886 dev->tx_pkt_burst = ngbe_xmit_pkts_simple;
1887 dev->tx_pkt_prepare = NULL;
1889 PMD_INIT_LOG(DEBUG, "Using full-featured tx code path");
1891 " - offloads = 0x%" PRIx64,
1894 " - tx_free_thresh = %lu [RTE_PMD_NGBE_TX_MAX_BURST=%lu]",
1895 (unsigned long)txq->tx_free_thresh,
1896 (unsigned long)RTE_PMD_NGBE_TX_MAX_BURST);
1897 dev->tx_pkt_burst = ngbe_xmit_pkts;
1898 dev->tx_pkt_prepare = ngbe_prep_pkts;
1902 static const struct {
1903 eth_tx_burst_t pkt_burst;
1905 } ngbe_tx_burst_infos[] = {
1906 { ngbe_xmit_pkts_simple, "Scalar Simple"},
1907 { ngbe_xmit_pkts, "Scalar"},
1911 ngbe_tx_burst_mode_get(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id,
1912 struct rte_eth_burst_mode *mode)
1914 eth_tx_burst_t pkt_burst = dev->tx_pkt_burst;
1918 for (i = 0; i < RTE_DIM(ngbe_tx_burst_infos); ++i) {
1919 if (pkt_burst == ngbe_tx_burst_infos[i].pkt_burst) {
1920 snprintf(mode->info, sizeof(mode->info), "%s",
1921 ngbe_tx_burst_infos[i].info);
1931 ngbe_get_tx_port_offloads(struct rte_eth_dev *dev)
1933 uint64_t tx_offload_capa;
1934 struct ngbe_hw *hw = ngbe_dev_hw(dev);
1937 RTE_ETH_TX_OFFLOAD_VLAN_INSERT |
1938 RTE_ETH_TX_OFFLOAD_IPV4_CKSUM |
1939 RTE_ETH_TX_OFFLOAD_UDP_CKSUM |
1940 RTE_ETH_TX_OFFLOAD_TCP_CKSUM |
1941 RTE_ETH_TX_OFFLOAD_SCTP_CKSUM |
1942 RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM |
1943 RTE_ETH_TX_OFFLOAD_TCP_TSO |
1944 RTE_ETH_TX_OFFLOAD_UDP_TSO |
1945 RTE_ETH_TX_OFFLOAD_UDP_TNL_TSO |
1946 RTE_ETH_TX_OFFLOAD_IP_TNL_TSO |
1947 RTE_ETH_TX_OFFLOAD_IPIP_TNL_TSO |
1948 RTE_ETH_TX_OFFLOAD_MULTI_SEGS;
1951 tx_offload_capa |= RTE_ETH_TX_OFFLOAD_QINQ_INSERT;
1953 return tx_offload_capa;
1957 ngbe_dev_tx_queue_setup(struct rte_eth_dev *dev,
1960 unsigned int socket_id,
1961 const struct rte_eth_txconf *tx_conf)
1963 const struct rte_memzone *tz;
1964 struct ngbe_tx_queue *txq;
1966 uint16_t tx_free_thresh;
1969 PMD_INIT_FUNC_TRACE();
1970 hw = ngbe_dev_hw(dev);
1972 offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads;
1975 * The Tx descriptor ring will be cleaned after txq->tx_free_thresh
1976 * descriptors are used or if the number of descriptors required
1977 * to transmit a packet is greater than the number of free Tx
1979 * One descriptor in the Tx ring is used as a sentinel to avoid a
1980 * H/W race condition, hence the maximum threshold constraints.
1981 * When set to zero use default values.
1983 tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ?
1984 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH);
1985 if (tx_free_thresh >= (nb_desc - 3)) {
1987 "tx_free_thresh must be less than the number of TX descriptors minus 3. (tx_free_thresh=%u port=%d queue=%d)",
1988 (unsigned int)tx_free_thresh,
1989 (int)dev->data->port_id, (int)queue_idx);
1993 if (nb_desc % tx_free_thresh != 0) {
1995 "tx_free_thresh must be a divisor of the number of Tx descriptors. (tx_free_thresh=%u port=%d queue=%d)",
1996 (unsigned int)tx_free_thresh,
1997 (int)dev->data->port_id, (int)queue_idx);
2001 /* Free memory prior to re-allocation if needed... */
2002 if (dev->data->tx_queues[queue_idx] != NULL) {
2003 ngbe_tx_queue_release(dev->data->tx_queues[queue_idx]);
2004 dev->data->tx_queues[queue_idx] = NULL;
2007 /* First allocate the Tx queue data structure */
2008 txq = rte_zmalloc_socket("ethdev Tx queue",
2009 sizeof(struct ngbe_tx_queue),
2010 RTE_CACHE_LINE_SIZE, socket_id);
2015 * Allocate Tx ring hardware descriptors. A memzone large enough to
2016 * handle the maximum ring size is allocated in order to allow for
2017 * resizing in later calls to the queue setup function.
2019 tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx,
2020 sizeof(struct ngbe_tx_desc) * NGBE_RING_DESC_MAX,
2021 NGBE_ALIGN, socket_id);
2023 ngbe_tx_queue_release(txq);
2027 txq->nb_tx_desc = nb_desc;
2028 txq->tx_free_thresh = tx_free_thresh;
2029 txq->pthresh = tx_conf->tx_thresh.pthresh;
2030 txq->hthresh = tx_conf->tx_thresh.hthresh;
2031 txq->wthresh = tx_conf->tx_thresh.wthresh;
2032 txq->queue_id = queue_idx;
2033 txq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ?
2034 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx);
2035 txq->port_id = dev->data->port_id;
2036 txq->offloads = offloads;
2037 txq->ops = &def_txq_ops;
2038 txq->tx_deferred_start = tx_conf->tx_deferred_start;
2040 txq->tdt_reg_addr = NGBE_REG_ADDR(hw, NGBE_TXWP(txq->reg_idx));
2041 txq->tdc_reg_addr = NGBE_REG_ADDR(hw, NGBE_TXCFG(txq->reg_idx));
2043 txq->tx_ring_phys_addr = TMZ_PADDR(tz);
2044 txq->tx_ring = (struct ngbe_tx_desc *)TMZ_VADDR(tz);
2046 /* Allocate software ring */
2047 txq->sw_ring = rte_zmalloc_socket("txq->sw_ring",
2048 sizeof(struct ngbe_tx_entry) * nb_desc,
2049 RTE_CACHE_LINE_SIZE, socket_id);
2050 if (txq->sw_ring == NULL) {
2051 ngbe_tx_queue_release(txq);
2055 "sw_ring=%p hw_ring=%p dma_addr=0x%" PRIx64,
2056 txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr);
2058 /* set up scalar Tx function as appropriate */
2059 ngbe_set_tx_function(dev, txq);
2061 txq->ops->reset(txq);
2063 dev->data->tx_queues[queue_idx] = txq;
2069 * ngbe_free_sc_cluster - free the not-yet-completed scattered cluster
2071 * The "next" pointer of the last segment of (not-yet-completed) RSC clusters
2072 * in the sw_sc_ring is not set to NULL but rather points to the next
2073 * mbuf of this RSC aggregation (that has not been completed yet and still
2074 * resides on the HW ring). So, instead of calling for rte_pktmbuf_free() we
2075 * will just free first "nb_segs" segments of the cluster explicitly by calling
2076 * an rte_pktmbuf_free_seg().
2078 * @m scattered cluster head
2081 ngbe_free_sc_cluster(struct rte_mbuf *m)
2083 uint16_t i, nb_segs = m->nb_segs;
2084 struct rte_mbuf *next_seg;
2086 for (i = 0; i < nb_segs; i++) {
2088 rte_pktmbuf_free_seg(m);
2094 ngbe_rx_queue_release_mbufs(struct ngbe_rx_queue *rxq)
2098 if (rxq->sw_ring != NULL) {
2099 for (i = 0; i < rxq->nb_rx_desc; i++) {
2100 if (rxq->sw_ring[i].mbuf != NULL) {
2101 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
2102 rxq->sw_ring[i].mbuf = NULL;
2105 for (i = 0; i < rxq->rx_nb_avail; ++i) {
2106 struct rte_mbuf *mb;
2108 mb = rxq->rx_stage[rxq->rx_next_avail + i];
2109 rte_pktmbuf_free_seg(mb);
2111 rxq->rx_nb_avail = 0;
2114 if (rxq->sw_sc_ring != NULL)
2115 for (i = 0; i < rxq->nb_rx_desc; i++)
2116 if (rxq->sw_sc_ring[i].fbuf != NULL) {
2117 ngbe_free_sc_cluster(rxq->sw_sc_ring[i].fbuf);
2118 rxq->sw_sc_ring[i].fbuf = NULL;
2123 ngbe_rx_queue_release(struct ngbe_rx_queue *rxq)
2126 ngbe_rx_queue_release_mbufs(rxq);
2127 rte_free(rxq->sw_ring);
2128 rte_free(rxq->sw_sc_ring);
2134 ngbe_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
2136 ngbe_rx_queue_release(dev->data->rx_queues[qid]);
2140 * Check if Rx Burst Bulk Alloc function can be used.
2142 * 0: the preconditions are satisfied and the bulk allocation function
2144 * -EINVAL: the preconditions are NOT satisfied and the default Rx burst
2145 * function must be used.
2148 check_rx_burst_bulk_alloc_preconditions(struct ngbe_rx_queue *rxq)
2153 * Make sure the following pre-conditions are satisfied:
2154 * rxq->rx_free_thresh >= RTE_PMD_NGBE_RX_MAX_BURST
2155 * rxq->rx_free_thresh < rxq->nb_rx_desc
2156 * (rxq->nb_rx_desc % rxq->rx_free_thresh) == 0
2157 * Scattered packets are not supported. This should be checked
2158 * outside of this function.
2160 if (rxq->rx_free_thresh < RTE_PMD_NGBE_RX_MAX_BURST) {
2162 "Rx Burst Bulk Alloc Preconditions: rxq->rx_free_thresh=%d, RTE_PMD_NGBE_RX_MAX_BURST=%d",
2163 rxq->rx_free_thresh, RTE_PMD_NGBE_RX_MAX_BURST);
2165 } else if (rxq->rx_free_thresh >= rxq->nb_rx_desc) {
2167 "Rx Burst Bulk Alloc Preconditions: rxq->rx_free_thresh=%d, rxq->nb_rx_desc=%d",
2168 rxq->rx_free_thresh, rxq->nb_rx_desc);
2170 } else if ((rxq->nb_rx_desc % rxq->rx_free_thresh) != 0) {
2172 "Rx Burst Bulk Alloc Preconditions: rxq->nb_rx_desc=%d, rxq->rx_free_thresh=%d",
2173 rxq->nb_rx_desc, rxq->rx_free_thresh);
2180 /* Reset dynamic ngbe_rx_queue fields back to defaults */
2182 ngbe_reset_rx_queue(struct ngbe_adapter *adapter, struct ngbe_rx_queue *rxq)
2184 static const struct ngbe_rx_desc zeroed_desc = {
2185 {{0}, {0} }, {{0}, {0} } };
2187 uint16_t len = rxq->nb_rx_desc;
2190 * By default, the Rx queue setup function allocates enough memory for
2191 * NGBE_RING_DESC_MAX. The Rx Burst bulk allocation function requires
2192 * extra memory at the end of the descriptor ring to be zero'd out.
2194 if (adapter->rx_bulk_alloc_allowed)
2195 /* zero out extra memory */
2196 len += RTE_PMD_NGBE_RX_MAX_BURST;
2199 * Zero out HW ring memory. Zero out extra memory at the end of
2200 * the H/W ring so look-ahead logic in Rx Burst bulk alloc function
2201 * reads extra memory as zeros.
2203 for (i = 0; i < len; i++)
2204 rxq->rx_ring[i] = zeroed_desc;
2207 * initialize extra software ring entries. Space for these extra
2208 * entries is always allocated
2210 memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf));
2211 for (i = rxq->nb_rx_desc; i < len; ++i)
2212 rxq->sw_ring[i].mbuf = &rxq->fake_mbuf;
2214 rxq->rx_nb_avail = 0;
2215 rxq->rx_next_avail = 0;
2216 rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1);
2218 rxq->nb_rx_hold = 0;
2219 rxq->pkt_first_seg = NULL;
2220 rxq->pkt_last_seg = NULL;
2224 ngbe_get_rx_queue_offloads(struct rte_eth_dev *dev __rte_unused)
2226 return RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
2230 ngbe_get_rx_port_offloads(struct rte_eth_dev *dev)
2233 struct ngbe_hw *hw = ngbe_dev_hw(dev);
2235 offloads = RTE_ETH_RX_OFFLOAD_IPV4_CKSUM |
2236 RTE_ETH_RX_OFFLOAD_UDP_CKSUM |
2237 RTE_ETH_RX_OFFLOAD_TCP_CKSUM |
2238 RTE_ETH_RX_OFFLOAD_KEEP_CRC |
2239 RTE_ETH_RX_OFFLOAD_VLAN_FILTER |
2240 RTE_ETH_RX_OFFLOAD_SCATTER;
2243 offloads |= (RTE_ETH_RX_OFFLOAD_QINQ_STRIP |
2244 RTE_ETH_RX_OFFLOAD_VLAN_EXTEND);
2250 ngbe_dev_rx_queue_setup(struct rte_eth_dev *dev,
2253 unsigned int socket_id,
2254 const struct rte_eth_rxconf *rx_conf,
2255 struct rte_mempool *mp)
2257 const struct rte_memzone *rz;
2258 struct ngbe_rx_queue *rxq;
2261 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
2264 PMD_INIT_FUNC_TRACE();
2265 hw = ngbe_dev_hw(dev);
2267 offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads;
2269 /* Free memory prior to re-allocation if needed... */
2270 if (dev->data->rx_queues[queue_idx] != NULL) {
2271 ngbe_rx_queue_release(dev->data->rx_queues[queue_idx]);
2272 dev->data->rx_queues[queue_idx] = NULL;
2275 /* First allocate the Rx queue data structure */
2276 rxq = rte_zmalloc_socket("ethdev RX queue",
2277 sizeof(struct ngbe_rx_queue),
2278 RTE_CACHE_LINE_SIZE, socket_id);
2282 rxq->nb_rx_desc = nb_desc;
2283 rxq->rx_free_thresh = rx_conf->rx_free_thresh;
2284 rxq->queue_id = queue_idx;
2285 rxq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ?
2286 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx);
2287 rxq->port_id = dev->data->port_id;
2288 if (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC)
2289 rxq->crc_len = RTE_ETHER_CRC_LEN;
2292 rxq->drop_en = rx_conf->rx_drop_en;
2293 rxq->rx_deferred_start = rx_conf->rx_deferred_start;
2294 rxq->offloads = offloads;
2297 * Allocate Rx ring hardware descriptors. A memzone large enough to
2298 * handle the maximum ring size is allocated in order to allow for
2299 * resizing in later calls to the queue setup function.
2301 rz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx,
2302 RX_RING_SZ, NGBE_ALIGN, socket_id);
2304 ngbe_rx_queue_release(rxq);
2309 * Zero init all the descriptors in the ring.
2311 memset(rz->addr, 0, RX_RING_SZ);
2313 rxq->rdt_reg_addr = NGBE_REG_ADDR(hw, NGBE_RXWP(rxq->reg_idx));
2314 rxq->rdh_reg_addr = NGBE_REG_ADDR(hw, NGBE_RXRP(rxq->reg_idx));
2316 rxq->rx_ring_phys_addr = TMZ_PADDR(rz);
2317 rxq->rx_ring = (struct ngbe_rx_desc *)TMZ_VADDR(rz);
2320 * Certain constraints must be met in order to use the bulk buffer
2321 * allocation Rx burst function. If any of Rx queues doesn't meet them
2322 * the feature should be disabled for the whole port.
2324 if (check_rx_burst_bulk_alloc_preconditions(rxq)) {
2326 "queue[%d] doesn't meet Rx Bulk Alloc preconditions - canceling the feature for the whole port[%d]",
2327 rxq->queue_id, rxq->port_id);
2328 adapter->rx_bulk_alloc_allowed = false;
2332 * Allocate software ring. Allow for space at the end of the
2333 * S/W ring to make sure look-ahead logic in bulk alloc Rx burst
2334 * function does not access an invalid memory region.
2337 if (adapter->rx_bulk_alloc_allowed)
2338 len += RTE_PMD_NGBE_RX_MAX_BURST;
2340 rxq->sw_ring = rte_zmalloc_socket("rxq->sw_ring",
2341 sizeof(struct ngbe_rx_entry) * len,
2342 RTE_CACHE_LINE_SIZE, socket_id);
2343 if (rxq->sw_ring == NULL) {
2344 ngbe_rx_queue_release(rxq);
2349 * Always allocate even if it's not going to be needed in order to
2350 * simplify the code.
2352 * This ring is used in Scattered Rx cases and Scattered Rx may
2353 * be requested in ngbe_dev_rx_init(), which is called later from
2357 rte_zmalloc_socket("rxq->sw_sc_ring",
2358 sizeof(struct ngbe_scattered_rx_entry) * len,
2359 RTE_CACHE_LINE_SIZE, socket_id);
2360 if (rxq->sw_sc_ring == NULL) {
2361 ngbe_rx_queue_release(rxq);
2366 "sw_ring=%p sw_sc_ring=%p hw_ring=%p dma_addr=0x%" PRIx64,
2367 rxq->sw_ring, rxq->sw_sc_ring, rxq->rx_ring,
2368 rxq->rx_ring_phys_addr);
2370 dev->data->rx_queues[queue_idx] = rxq;
2372 ngbe_reset_rx_queue(adapter, rxq);
2378 ngbe_dev_rx_queue_count(void *rx_queue)
2380 #define NGBE_RXQ_SCAN_INTERVAL 4
2381 volatile struct ngbe_rx_desc *rxdp;
2382 struct ngbe_rx_queue *rxq = rx_queue;
2385 rxdp = &rxq->rx_ring[rxq->rx_tail];
2387 while ((desc < rxq->nb_rx_desc) &&
2388 (rxdp->qw1.lo.status &
2389 rte_cpu_to_le_32(NGBE_RXD_STAT_DD))) {
2390 desc += NGBE_RXQ_SCAN_INTERVAL;
2391 rxdp += NGBE_RXQ_SCAN_INTERVAL;
2392 if (rxq->rx_tail + desc >= rxq->nb_rx_desc)
2393 rxdp = &(rxq->rx_ring[rxq->rx_tail +
2394 desc - rxq->nb_rx_desc]);
2401 ngbe_dev_rx_descriptor_status(void *rx_queue, uint16_t offset)
2403 struct ngbe_rx_queue *rxq = rx_queue;
2404 volatile uint32_t *status;
2405 uint32_t nb_hold, desc;
2407 if (unlikely(offset >= rxq->nb_rx_desc))
2410 nb_hold = rxq->nb_rx_hold;
2411 if (offset >= rxq->nb_rx_desc - nb_hold)
2412 return RTE_ETH_RX_DESC_UNAVAIL;
2414 desc = rxq->rx_tail + offset;
2415 if (desc >= rxq->nb_rx_desc)
2416 desc -= rxq->nb_rx_desc;
2418 status = &rxq->rx_ring[desc].qw1.lo.status;
2419 if (*status & rte_cpu_to_le_32(NGBE_RXD_STAT_DD))
2420 return RTE_ETH_RX_DESC_DONE;
2422 return RTE_ETH_RX_DESC_AVAIL;
2426 ngbe_dev_tx_descriptor_status(void *tx_queue, uint16_t offset)
2428 struct ngbe_tx_queue *txq = tx_queue;
2429 volatile uint32_t *status;
2432 if (unlikely(offset >= txq->nb_tx_desc))
2435 desc = txq->tx_tail + offset;
2436 if (desc >= txq->nb_tx_desc) {
2437 desc -= txq->nb_tx_desc;
2438 if (desc >= txq->nb_tx_desc)
2439 desc -= txq->nb_tx_desc;
2442 status = &txq->tx_ring[desc].dw3;
2443 if (*status & rte_cpu_to_le_32(NGBE_TXD_DD))
2444 return RTE_ETH_TX_DESC_DONE;
2446 return RTE_ETH_TX_DESC_FULL;
2450 ngbe_dev_clear_queues(struct rte_eth_dev *dev)
2453 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
2455 PMD_INIT_FUNC_TRACE();
2457 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2458 struct ngbe_tx_queue *txq = dev->data->tx_queues[i];
2461 txq->ops->release_mbufs(txq);
2462 txq->ops->reset(txq);
2466 for (i = 0; i < dev->data->nb_rx_queues; i++) {
2467 struct ngbe_rx_queue *rxq = dev->data->rx_queues[i];
2470 ngbe_rx_queue_release_mbufs(rxq);
2471 ngbe_reset_rx_queue(adapter, rxq);
2477 ngbe_dev_free_queues(struct rte_eth_dev *dev)
2481 PMD_INIT_FUNC_TRACE();
2483 for (i = 0; i < dev->data->nb_rx_queues; i++) {
2484 ngbe_dev_rx_queue_release(dev, i);
2485 dev->data->rx_queues[i] = NULL;
2487 dev->data->nb_rx_queues = 0;
2489 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2490 ngbe_dev_tx_queue_release(dev, i);
2491 dev->data->tx_queues[i] = NULL;
2493 dev->data->nb_tx_queues = 0;
2497 * Receive Side Scaling (RSS)
2500 * The source and destination IP addresses of the IP header and the source
2501 * and destination ports of TCP/UDP headers, if any, of received packets are
2502 * hashed against a configurable random key to compute a 32-bit RSS hash result.
2503 * The seven (7) LSBs of the 32-bit hash result are used as an index into a
2504 * 128-entry redirection table (RETA). Each entry of the RETA provides a 3-bit
2505 * RSS output index which is used as the Rx queue index where to store the
2507 * The following output is supplied in the Rx write-back descriptor:
2508 * - 32-bit result of the Microsoft RSS hash function,
2509 * - 4-bit RSS type field.
2513 * Used as the default key.
2515 static uint8_t rss_intel_key[40] = {
2516 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
2517 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
2518 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
2519 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
2520 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
2524 ngbe_rss_disable(struct rte_eth_dev *dev)
2526 struct ngbe_hw *hw = ngbe_dev_hw(dev);
2528 wr32m(hw, NGBE_RACTL, NGBE_RACTL_RSSENA, 0);
2532 ngbe_dev_rss_hash_update(struct rte_eth_dev *dev,
2533 struct rte_eth_rss_conf *rss_conf)
2535 struct ngbe_hw *hw = ngbe_dev_hw(dev);
2543 PMD_DRV_LOG(ERR, "RSS hash update is not supported on this "
2548 hash_key = rss_conf->rss_key;
2550 /* Fill in RSS hash key */
2551 for (i = 0; i < 10; i++) {
2552 rss_key = LS32(hash_key[(i * 4) + 0], 0, 0xFF);
2553 rss_key |= LS32(hash_key[(i * 4) + 1], 8, 0xFF);
2554 rss_key |= LS32(hash_key[(i * 4) + 2], 16, 0xFF);
2555 rss_key |= LS32(hash_key[(i * 4) + 3], 24, 0xFF);
2556 wr32a(hw, NGBE_REG_RSSKEY, i, rss_key);
2560 /* Set configured hashing protocols */
2561 rss_hf = rss_conf->rss_hf & NGBE_RSS_OFFLOAD_ALL;
2563 mrqc = rd32(hw, NGBE_RACTL);
2564 mrqc &= ~NGBE_RACTL_RSSMASK;
2565 if (rss_hf & RTE_ETH_RSS_IPV4)
2566 mrqc |= NGBE_RACTL_RSSIPV4;
2567 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_TCP)
2568 mrqc |= NGBE_RACTL_RSSIPV4TCP;
2569 if (rss_hf & RTE_ETH_RSS_IPV6 ||
2570 rss_hf & RTE_ETH_RSS_IPV6_EX)
2571 mrqc |= NGBE_RACTL_RSSIPV6;
2572 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV6_TCP ||
2573 rss_hf & RTE_ETH_RSS_IPV6_TCP_EX)
2574 mrqc |= NGBE_RACTL_RSSIPV6TCP;
2575 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_UDP)
2576 mrqc |= NGBE_RACTL_RSSIPV4UDP;
2577 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV6_UDP ||
2578 rss_hf & RTE_ETH_RSS_IPV6_UDP_EX)
2579 mrqc |= NGBE_RACTL_RSSIPV6UDP;
2582 mrqc |= NGBE_RACTL_RSSENA;
2584 mrqc &= ~NGBE_RACTL_RSSENA;
2586 wr32(hw, NGBE_RACTL, mrqc);
2592 ngbe_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
2593 struct rte_eth_rss_conf *rss_conf)
2595 struct ngbe_hw *hw = ngbe_dev_hw(dev);
2602 hash_key = rss_conf->rss_key;
2604 /* Return RSS hash key */
2605 for (i = 0; i < 10; i++) {
2606 rss_key = rd32a(hw, NGBE_REG_RSSKEY, i);
2607 hash_key[(i * 4) + 0] = RS32(rss_key, 0, 0xFF);
2608 hash_key[(i * 4) + 1] = RS32(rss_key, 8, 0xFF);
2609 hash_key[(i * 4) + 2] = RS32(rss_key, 16, 0xFF);
2610 hash_key[(i * 4) + 3] = RS32(rss_key, 24, 0xFF);
2616 mrqc = rd32(hw, NGBE_RACTL);
2617 if (mrqc & NGBE_RACTL_RSSIPV4)
2618 rss_hf |= RTE_ETH_RSS_IPV4;
2619 if (mrqc & NGBE_RACTL_RSSIPV4TCP)
2620 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_TCP;
2621 if (mrqc & NGBE_RACTL_RSSIPV6)
2622 rss_hf |= RTE_ETH_RSS_IPV6 |
2623 RTE_ETH_RSS_IPV6_EX;
2624 if (mrqc & NGBE_RACTL_RSSIPV6TCP)
2625 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_TCP |
2626 RTE_ETH_RSS_IPV6_TCP_EX;
2627 if (mrqc & NGBE_RACTL_RSSIPV4UDP)
2628 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_UDP;
2629 if (mrqc & NGBE_RACTL_RSSIPV6UDP)
2630 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_UDP |
2631 RTE_ETH_RSS_IPV6_UDP_EX;
2632 if (!(mrqc & NGBE_RACTL_RSSENA))
2635 rss_hf &= NGBE_RSS_OFFLOAD_ALL;
2637 rss_conf->rss_hf = rss_hf;
2642 ngbe_rss_configure(struct rte_eth_dev *dev)
2644 struct rte_eth_rss_conf rss_conf;
2645 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
2646 struct ngbe_hw *hw = ngbe_dev_hw(dev);
2651 PMD_INIT_FUNC_TRACE();
2654 * Fill in redirection table
2655 * The byte-swap is needed because NIC registers are in
2656 * little-endian order.
2658 if (adapter->rss_reta_updated == 0) {
2660 for (i = 0, j = 0; i < RTE_ETH_RSS_RETA_SIZE_128; i++, j++) {
2661 if (j == dev->data->nb_rx_queues)
2663 reta = (reta >> 8) | LS32(j, 24, 0xFF);
2665 wr32a(hw, NGBE_REG_RSSTBL, i >> 2, reta);
2669 * Configure the RSS key and the RSS protocols used to compute
2670 * the RSS hash of input packets.
2672 rss_conf = dev->data->dev_conf.rx_adv_conf.rss_conf;
2673 if (rss_conf.rss_key == NULL)
2674 rss_conf.rss_key = rss_intel_key; /* Default hash key */
2675 ngbe_dev_rss_hash_update(dev, &rss_conf);
2678 void ngbe_configure_port(struct rte_eth_dev *dev)
2680 struct ngbe_hw *hw = ngbe_dev_hw(dev);
2682 uint16_t tpids[8] = {RTE_ETHER_TYPE_VLAN, RTE_ETHER_TYPE_QINQ,
2687 PMD_INIT_FUNC_TRACE();
2689 /* default outer vlan tpid */
2690 wr32(hw, NGBE_EXTAG,
2691 NGBE_EXTAG_ETAG(RTE_ETHER_TYPE_ETAG) |
2692 NGBE_EXTAG_VLAN(RTE_ETHER_TYPE_QINQ));
2694 /* default inner vlan tpid */
2695 wr32m(hw, NGBE_VLANCTL,
2696 NGBE_VLANCTL_TPID_MASK,
2697 NGBE_VLANCTL_TPID(RTE_ETHER_TYPE_VLAN));
2698 wr32m(hw, NGBE_DMATXCTRL,
2699 NGBE_DMATXCTRL_TPID_MASK,
2700 NGBE_DMATXCTRL_TPID(RTE_ETHER_TYPE_VLAN));
2702 /* default vlan tpid filters */
2703 for (i = 0; i < 8; i++) {
2704 wr32m(hw, NGBE_TAGTPID(i / 2),
2705 (i % 2 ? NGBE_TAGTPID_MSB_MASK
2706 : NGBE_TAGTPID_LSB_MASK),
2707 (i % 2 ? NGBE_TAGTPID_MSB(tpids[i])
2708 : NGBE_TAGTPID_LSB(tpids[i])));
2713 ngbe_alloc_rx_queue_mbufs(struct ngbe_rx_queue *rxq)
2715 struct ngbe_rx_entry *rxe = rxq->sw_ring;
2719 /* Initialize software ring entries */
2720 for (i = 0; i < rxq->nb_rx_desc; i++) {
2721 /* the ring can also be modified by hardware */
2722 volatile struct ngbe_rx_desc *rxd;
2723 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mb_pool);
2726 PMD_INIT_LOG(ERR, "Rx mbuf alloc failed queue_id=%u port_id=%u",
2727 (unsigned int)rxq->queue_id,
2728 (unsigned int)rxq->port_id);
2732 mbuf->data_off = RTE_PKTMBUF_HEADROOM;
2733 mbuf->port = rxq->port_id;
2736 rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf));
2737 rxd = &rxq->rx_ring[i];
2738 NGBE_RXD_HDRADDR(rxd, 0);
2739 NGBE_RXD_PKTADDR(rxd, dma_addr);
2747 ngbe_dev_mq_rx_configure(struct rte_eth_dev *dev)
2749 if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
2750 switch (dev->data->dev_conf.rxmode.mq_mode) {
2751 case RTE_ETH_MQ_RX_RSS:
2752 ngbe_rss_configure(dev);
2755 case RTE_ETH_MQ_RX_NONE:
2757 /* if mq_mode is none, disable rss mode.*/
2758 ngbe_rss_disable(dev);
2767 ngbe_set_rx_function(struct rte_eth_dev *dev)
2769 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
2771 if (dev->data->scattered_rx) {
2773 * Set the scattered callback: there are bulk and
2774 * single allocation versions.
2776 if (adapter->rx_bulk_alloc_allowed) {
2777 PMD_INIT_LOG(DEBUG, "Using a Scattered with bulk "
2778 "allocation callback (port=%d).",
2779 dev->data->port_id);
2780 dev->rx_pkt_burst = ngbe_recv_pkts_sc_bulk_alloc;
2782 PMD_INIT_LOG(DEBUG, "Using Regular (non-vector, "
2783 "single allocation) "
2784 "Scattered Rx callback "
2786 dev->data->port_id);
2788 dev->rx_pkt_burst = ngbe_recv_pkts_sc_single_alloc;
2791 * Below we set "simple" callbacks according to port/queues parameters.
2792 * If parameters allow we are going to choose between the following
2795 * - Single buffer allocation (the simplest one)
2797 } else if (adapter->rx_bulk_alloc_allowed) {
2798 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are "
2799 "satisfied. Rx Burst Bulk Alloc function "
2800 "will be used on port=%d.",
2801 dev->data->port_id);
2803 dev->rx_pkt_burst = ngbe_recv_pkts_bulk_alloc;
2805 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are not "
2806 "satisfied, or Scattered Rx is requested "
2808 dev->data->port_id);
2810 dev->rx_pkt_burst = ngbe_recv_pkts;
2814 static const struct {
2815 eth_rx_burst_t pkt_burst;
2817 } ngbe_rx_burst_infos[] = {
2818 { ngbe_recv_pkts_sc_single_alloc, "Scalar Scattered"},
2819 { ngbe_recv_pkts_sc_bulk_alloc, "Scalar Scattered Bulk Alloc"},
2820 { ngbe_recv_pkts_bulk_alloc, "Scalar Bulk Alloc"},
2821 { ngbe_recv_pkts, "Scalar"},
2825 ngbe_rx_burst_mode_get(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id,
2826 struct rte_eth_burst_mode *mode)
2828 eth_rx_burst_t pkt_burst = dev->rx_pkt_burst;
2832 for (i = 0; i < RTE_DIM(ngbe_rx_burst_infos); ++i) {
2833 if (pkt_burst == ngbe_rx_burst_infos[i].pkt_burst) {
2834 snprintf(mode->info, sizeof(mode->info), "%s",
2835 ngbe_rx_burst_infos[i].info);
2845 * Initializes Receive Unit.
2848 ngbe_dev_rx_init(struct rte_eth_dev *dev)
2851 struct ngbe_rx_queue *rxq;
2860 struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode;
2862 PMD_INIT_FUNC_TRACE();
2863 hw = ngbe_dev_hw(dev);
2866 * Make sure receives are disabled while setting
2867 * up the Rx context (registers, descriptor rings, etc.).
2869 wr32m(hw, NGBE_MACRXCFG, NGBE_MACRXCFG_ENA, 0);
2870 wr32m(hw, NGBE_PBRXCTL, NGBE_PBRXCTL_ENA, 0);
2872 /* Enable receipt of broadcasted frames */
2873 fctrl = rd32(hw, NGBE_PSRCTL);
2874 fctrl |= NGBE_PSRCTL_BCA;
2875 wr32(hw, NGBE_PSRCTL, fctrl);
2878 * Configure CRC stripping, if any.
2880 hlreg0 = rd32(hw, NGBE_SECRXCTL);
2881 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC)
2882 hlreg0 &= ~NGBE_SECRXCTL_CRCSTRIP;
2884 hlreg0 |= NGBE_SECRXCTL_CRCSTRIP;
2885 hlreg0 &= ~NGBE_SECRXCTL_XDSA;
2886 wr32(hw, NGBE_SECRXCTL, hlreg0);
2889 * Configure jumbo frame support, if any.
2891 wr32m(hw, NGBE_FRMSZ, NGBE_FRMSZ_MAX_MASK,
2892 NGBE_FRMSZ_MAX(dev->data->mtu + NGBE_ETH_OVERHEAD));
2895 * If loopback mode is configured, set LPBK bit.
2897 hlreg0 = rd32(hw, NGBE_PSRCTL);
2898 if (hw->is_pf && dev->data->dev_conf.lpbk_mode)
2899 hlreg0 |= NGBE_PSRCTL_LBENA;
2901 hlreg0 &= ~NGBE_PSRCTL_LBENA;
2903 wr32(hw, NGBE_PSRCTL, hlreg0);
2906 * Assume no header split and no VLAN strip support
2907 * on any Rx queue first .
2909 rx_conf->offloads &= ~RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
2911 /* Setup Rx queues */
2912 for (i = 0; i < dev->data->nb_rx_queues; i++) {
2913 rxq = dev->data->rx_queues[i];
2916 * Reset crc_len in case it was changed after queue setup by a
2917 * call to configure.
2919 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC)
2920 rxq->crc_len = RTE_ETHER_CRC_LEN;
2924 /* Setup the Base and Length of the Rx Descriptor Rings */
2925 bus_addr = rxq->rx_ring_phys_addr;
2926 wr32(hw, NGBE_RXBAL(rxq->reg_idx),
2927 (uint32_t)(bus_addr & BIT_MASK32));
2928 wr32(hw, NGBE_RXBAH(rxq->reg_idx),
2929 (uint32_t)(bus_addr >> 32));
2930 wr32(hw, NGBE_RXRP(rxq->reg_idx), 0);
2931 wr32(hw, NGBE_RXWP(rxq->reg_idx), 0);
2933 srrctl = NGBE_RXCFG_RNGLEN(rxq->nb_rx_desc);
2935 /* Set if packets are dropped when no descriptors available */
2937 srrctl |= NGBE_RXCFG_DROP;
2940 * Configure the Rx buffer size in the PKTLEN field of
2941 * the RXCFG register of the queue.
2942 * The value is in 1 KB resolution. Valid values can be from
2945 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) -
2946 RTE_PKTMBUF_HEADROOM);
2947 buf_size = ROUND_DOWN(buf_size, 0x1 << 10);
2948 srrctl |= NGBE_RXCFG_PKTLEN(buf_size);
2950 wr32(hw, NGBE_RXCFG(rxq->reg_idx), srrctl);
2952 /* It adds dual VLAN length for supporting dual VLAN */
2953 if (dev->data->mtu + NGBE_ETH_OVERHEAD +
2954 2 * NGBE_VLAN_TAG_SIZE > buf_size)
2955 dev->data->scattered_rx = 1;
2956 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP)
2957 rx_conf->offloads |= RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
2960 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_SCATTER)
2961 dev->data->scattered_rx = 1;
2964 * Device configured with multiple RX queues.
2966 ngbe_dev_mq_rx_configure(dev);
2969 * Setup the Checksum Register.
2970 * Disable Full-Packet Checksum which is mutually exclusive with RSS.
2971 * Enable IP/L4 checksum computation by hardware if requested to do so.
2973 rxcsum = rd32(hw, NGBE_PSRCTL);
2974 rxcsum |= NGBE_PSRCTL_PCSD;
2975 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_CHECKSUM)
2976 rxcsum |= NGBE_PSRCTL_L4CSUM;
2978 rxcsum &= ~NGBE_PSRCTL_L4CSUM;
2980 wr32(hw, NGBE_PSRCTL, rxcsum);
2983 rdrxctl = rd32(hw, NGBE_SECRXCTL);
2984 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC)
2985 rdrxctl &= ~NGBE_SECRXCTL_CRCSTRIP;
2987 rdrxctl |= NGBE_SECRXCTL_CRCSTRIP;
2988 wr32(hw, NGBE_SECRXCTL, rdrxctl);
2991 ngbe_set_rx_function(dev);
2997 * Initializes Transmit Unit.
3000 ngbe_dev_tx_init(struct rte_eth_dev *dev)
3003 struct ngbe_tx_queue *txq;
3007 PMD_INIT_FUNC_TRACE();
3008 hw = ngbe_dev_hw(dev);
3010 wr32m(hw, NGBE_SECTXCTL, NGBE_SECTXCTL_ODSA, NGBE_SECTXCTL_ODSA);
3011 wr32m(hw, NGBE_SECTXCTL, NGBE_SECTXCTL_XDSA, 0);
3013 /* Setup the Base and Length of the Tx Descriptor Rings */
3014 for (i = 0; i < dev->data->nb_tx_queues; i++) {
3015 txq = dev->data->tx_queues[i];
3017 bus_addr = txq->tx_ring_phys_addr;
3018 wr32(hw, NGBE_TXBAL(txq->reg_idx),
3019 (uint32_t)(bus_addr & BIT_MASK32));
3020 wr32(hw, NGBE_TXBAH(txq->reg_idx),
3021 (uint32_t)(bus_addr >> 32));
3022 wr32m(hw, NGBE_TXCFG(txq->reg_idx), NGBE_TXCFG_BUFLEN_MASK,
3023 NGBE_TXCFG_BUFLEN(txq->nb_tx_desc));
3024 /* Setup the HW Tx Head and TX Tail descriptor pointers */
3025 wr32(hw, NGBE_TXRP(txq->reg_idx), 0);
3026 wr32(hw, NGBE_TXWP(txq->reg_idx), 0);
3031 * Set up link loopback mode Tx->Rx.
3034 ngbe_setup_loopback_link(struct ngbe_hw *hw)
3036 PMD_INIT_FUNC_TRACE();
3038 wr32m(hw, NGBE_MACRXCFG, NGBE_MACRXCFG_LB, NGBE_MACRXCFG_LB);
3044 * Start Transmit and Receive Units.
3047 ngbe_dev_rxtx_start(struct rte_eth_dev *dev)
3050 struct ngbe_tx_queue *txq;
3051 struct ngbe_rx_queue *rxq;
3057 PMD_INIT_FUNC_TRACE();
3058 hw = ngbe_dev_hw(dev);
3060 for (i = 0; i < dev->data->nb_tx_queues; i++) {
3061 txq = dev->data->tx_queues[i];
3062 /* Setup Transmit Threshold Registers */
3063 wr32m(hw, NGBE_TXCFG(txq->reg_idx),
3064 NGBE_TXCFG_HTHRESH_MASK |
3065 NGBE_TXCFG_WTHRESH_MASK,
3066 NGBE_TXCFG_HTHRESH(txq->hthresh) |
3067 NGBE_TXCFG_WTHRESH(txq->wthresh));
3070 dmatxctl = rd32(hw, NGBE_DMATXCTRL);
3071 dmatxctl |= NGBE_DMATXCTRL_ENA;
3072 wr32(hw, NGBE_DMATXCTRL, dmatxctl);
3074 for (i = 0; i < dev->data->nb_tx_queues; i++) {
3075 txq = dev->data->tx_queues[i];
3076 if (txq->tx_deferred_start == 0) {
3077 ret = ngbe_dev_tx_queue_start(dev, i);
3083 for (i = 0; i < dev->data->nb_rx_queues; i++) {
3084 rxq = dev->data->rx_queues[i];
3085 if (rxq->rx_deferred_start == 0) {
3086 ret = ngbe_dev_rx_queue_start(dev, i);
3092 /* Enable Receive engine */
3093 rxctrl = rd32(hw, NGBE_PBRXCTL);
3094 rxctrl |= NGBE_PBRXCTL_ENA;
3095 hw->mac.enable_rx_dma(hw, rxctrl);
3097 /* If loopback mode is enabled, set up the link accordingly */
3098 if (hw->is_pf && dev->data->dev_conf.lpbk_mode)
3099 ngbe_setup_loopback_link(hw);
3105 ngbe_dev_save_rx_queue(struct ngbe_hw *hw, uint16_t rx_queue_id)
3107 u32 *reg = &hw->q_rx_regs[rx_queue_id * 8];
3108 *(reg++) = rd32(hw, NGBE_RXBAL(rx_queue_id));
3109 *(reg++) = rd32(hw, NGBE_RXBAH(rx_queue_id));
3110 *(reg++) = rd32(hw, NGBE_RXCFG(rx_queue_id));
3114 ngbe_dev_store_rx_queue(struct ngbe_hw *hw, uint16_t rx_queue_id)
3116 u32 *reg = &hw->q_rx_regs[rx_queue_id * 8];
3117 wr32(hw, NGBE_RXBAL(rx_queue_id), *(reg++));
3118 wr32(hw, NGBE_RXBAH(rx_queue_id), *(reg++));
3119 wr32(hw, NGBE_RXCFG(rx_queue_id), *(reg++) & ~NGBE_RXCFG_ENA);
3123 ngbe_dev_save_tx_queue(struct ngbe_hw *hw, uint16_t tx_queue_id)
3125 u32 *reg = &hw->q_tx_regs[tx_queue_id * 8];
3126 *(reg++) = rd32(hw, NGBE_TXBAL(tx_queue_id));
3127 *(reg++) = rd32(hw, NGBE_TXBAH(tx_queue_id));
3128 *(reg++) = rd32(hw, NGBE_TXCFG(tx_queue_id));
3132 ngbe_dev_store_tx_queue(struct ngbe_hw *hw, uint16_t tx_queue_id)
3134 u32 *reg = &hw->q_tx_regs[tx_queue_id * 8];
3135 wr32(hw, NGBE_TXBAL(tx_queue_id), *(reg++));
3136 wr32(hw, NGBE_TXBAH(tx_queue_id), *(reg++));
3137 wr32(hw, NGBE_TXCFG(tx_queue_id), *(reg++) & ~NGBE_TXCFG_ENA);
3141 * Start Receive Units for specified queue.
3144 ngbe_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
3146 struct ngbe_hw *hw = ngbe_dev_hw(dev);
3147 struct ngbe_rx_queue *rxq;
3151 PMD_INIT_FUNC_TRACE();
3153 rxq = dev->data->rx_queues[rx_queue_id];
3155 /* Allocate buffers for descriptor rings */
3156 if (ngbe_alloc_rx_queue_mbufs(rxq) != 0) {
3157 PMD_INIT_LOG(ERR, "Could not alloc mbuf for queue:%d",
3161 rxdctl = rd32(hw, NGBE_RXCFG(rxq->reg_idx));
3162 rxdctl |= NGBE_RXCFG_ENA;
3163 wr32(hw, NGBE_RXCFG(rxq->reg_idx), rxdctl);
3165 /* Wait until Rx Enable ready */
3166 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS;
3169 rxdctl = rd32(hw, NGBE_RXCFG(rxq->reg_idx));
3170 } while (--poll_ms && !(rxdctl & NGBE_RXCFG_ENA));
3172 PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", rx_queue_id);
3174 wr32(hw, NGBE_RXRP(rxq->reg_idx), 0);
3175 wr32(hw, NGBE_RXWP(rxq->reg_idx), rxq->nb_rx_desc - 1);
3176 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
3182 * Stop Receive Units for specified queue.
3185 ngbe_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
3187 struct ngbe_hw *hw = ngbe_dev_hw(dev);
3188 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
3189 struct ngbe_rx_queue *rxq;
3193 PMD_INIT_FUNC_TRACE();
3195 rxq = dev->data->rx_queues[rx_queue_id];
3197 ngbe_dev_save_rx_queue(hw, rxq->reg_idx);
3198 wr32m(hw, NGBE_RXCFG(rxq->reg_idx), NGBE_RXCFG_ENA, 0);
3200 /* Wait until Rx Enable bit clear */
3201 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS;
3204 rxdctl = rd32(hw, NGBE_RXCFG(rxq->reg_idx));
3205 } while (--poll_ms && (rxdctl & NGBE_RXCFG_ENA));
3207 PMD_INIT_LOG(ERR, "Could not disable Rx Queue %d", rx_queue_id);
3209 rte_delay_us(RTE_NGBE_WAIT_100_US);
3210 ngbe_dev_store_rx_queue(hw, rxq->reg_idx);
3212 ngbe_rx_queue_release_mbufs(rxq);
3213 ngbe_reset_rx_queue(adapter, rxq);
3214 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
3220 * Start Transmit Units for specified queue.
3223 ngbe_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
3225 struct ngbe_hw *hw = ngbe_dev_hw(dev);
3226 struct ngbe_tx_queue *txq;
3230 PMD_INIT_FUNC_TRACE();
3232 txq = dev->data->tx_queues[tx_queue_id];
3233 wr32m(hw, NGBE_TXCFG(txq->reg_idx), NGBE_TXCFG_ENA, NGBE_TXCFG_ENA);
3235 /* Wait until Tx Enable ready */
3236 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS;
3239 txdctl = rd32(hw, NGBE_TXCFG(txq->reg_idx));
3240 } while (--poll_ms && !(txdctl & NGBE_TXCFG_ENA));
3242 PMD_INIT_LOG(ERR, "Could not enable Tx Queue %d",
3246 wr32(hw, NGBE_TXWP(txq->reg_idx), txq->tx_tail);
3247 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
3253 * Stop Transmit Units for specified queue.
3256 ngbe_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
3258 struct ngbe_hw *hw = ngbe_dev_hw(dev);
3259 struct ngbe_tx_queue *txq;
3261 uint32_t txtdh, txtdt;
3264 PMD_INIT_FUNC_TRACE();
3266 txq = dev->data->tx_queues[tx_queue_id];
3268 /* Wait until Tx queue is empty */
3269 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS;
3271 rte_delay_us(RTE_NGBE_WAIT_100_US);
3272 txtdh = rd32(hw, NGBE_TXRP(txq->reg_idx));
3273 txtdt = rd32(hw, NGBE_TXWP(txq->reg_idx));
3274 } while (--poll_ms && (txtdh != txtdt));
3276 PMD_INIT_LOG(ERR, "Tx Queue %d is not empty when stopping.",
3279 ngbe_dev_save_tx_queue(hw, txq->reg_idx);
3280 wr32m(hw, NGBE_TXCFG(txq->reg_idx), NGBE_TXCFG_ENA, 0);
3282 /* Wait until Tx Enable bit clear */
3283 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS;
3286 txdctl = rd32(hw, NGBE_TXCFG(txq->reg_idx));
3287 } while (--poll_ms && (txdctl & NGBE_TXCFG_ENA));
3289 PMD_INIT_LOG(ERR, "Could not disable Tx Queue %d",
3292 rte_delay_us(RTE_NGBE_WAIT_100_US);
3293 ngbe_dev_store_tx_queue(hw, txq->reg_idx);
3295 if (txq->ops != NULL) {
3296 txq->ops->release_mbufs(txq);
3297 txq->ops->reset(txq);
3299 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
3305 ngbe_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
3306 struct rte_eth_rxq_info *qinfo)
3308 struct ngbe_rx_queue *rxq;
3310 rxq = dev->data->rx_queues[queue_id];
3312 qinfo->mp = rxq->mb_pool;
3313 qinfo->scattered_rx = dev->data->scattered_rx;
3314 qinfo->nb_desc = rxq->nb_rx_desc;
3316 qinfo->conf.rx_free_thresh = rxq->rx_free_thresh;
3317 qinfo->conf.rx_drop_en = rxq->drop_en;
3318 qinfo->conf.rx_deferred_start = rxq->rx_deferred_start;
3319 qinfo->conf.offloads = rxq->offloads;
3323 ngbe_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
3324 struct rte_eth_txq_info *qinfo)
3326 struct ngbe_tx_queue *txq;
3328 txq = dev->data->tx_queues[queue_id];
3330 qinfo->nb_desc = txq->nb_tx_desc;
3332 qinfo->conf.tx_thresh.pthresh = txq->pthresh;
3333 qinfo->conf.tx_thresh.hthresh = txq->hthresh;
3334 qinfo->conf.tx_thresh.wthresh = txq->wthresh;
3336 qinfo->conf.tx_free_thresh = txq->tx_free_thresh;
3337 qinfo->conf.offloads = txq->offloads;
3338 qinfo->conf.tx_deferred_start = txq->tx_deferred_start;