1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018-2021 Beijing WangXun Technology Co., Ltd.
3 * Copyright(c) 2010-2017 Intel Corporation
9 #include <rte_ethdev.h>
10 #include <ethdev_driver.h>
11 #include <rte_malloc.h>
14 #include "ngbe_logs.h"
15 #include "base/ngbe.h"
16 #include "ngbe_ethdev.h"
17 #include "ngbe_rxtx.h"
19 #ifdef RTE_LIBRTE_IEEE1588
20 #define NGBE_TX_IEEE1588_TMST PKT_TX_IEEE1588_TMST
22 #define NGBE_TX_IEEE1588_TMST 0
25 /* Bit Mask to indicate what bits required for building Tx context */
26 static const u64 NGBE_TX_OFFLOAD_MASK = (RTE_MBUF_F_TX_IP_CKSUM |
27 RTE_MBUF_F_TX_OUTER_IPV6 |
28 RTE_MBUF_F_TX_OUTER_IPV4 |
32 RTE_MBUF_F_TX_L4_MASK |
33 RTE_MBUF_F_TX_TCP_SEG |
34 RTE_MBUF_F_TX_TUNNEL_MASK |
35 RTE_MBUF_F_TX_OUTER_IP_CKSUM |
36 NGBE_TX_IEEE1588_TMST);
38 #define NGBE_TX_OFFLOAD_NOTSUP_MASK \
39 (RTE_MBUF_F_TX_OFFLOAD_MASK ^ NGBE_TX_OFFLOAD_MASK)
42 * Prefetch a cache line into all cache levels.
44 #define rte_ngbe_prefetch(p) rte_prefetch0(p)
46 /*********************************************************************
50 **********************************************************************/
53 * Check for descriptors with their DD bit set and free mbufs.
54 * Return the total number of buffers freed.
56 static __rte_always_inline int
57 ngbe_tx_free_bufs(struct ngbe_tx_queue *txq)
59 struct ngbe_tx_entry *txep;
62 struct rte_mbuf *m, *free[RTE_NGBE_TX_MAX_FREE_BUF_SZ];
64 /* check DD bit on threshold descriptor */
65 status = txq->tx_ring[txq->tx_next_dd].dw3;
66 if (!(status & rte_cpu_to_le_32(NGBE_TXD_DD))) {
67 if (txq->nb_tx_free >> 1 < txq->tx_free_thresh)
68 ngbe_set32_masked(txq->tdc_reg_addr,
69 NGBE_TXCFG_FLUSH, NGBE_TXCFG_FLUSH);
74 * first buffer to free from S/W ring is at index
75 * tx_next_dd - (tx_free_thresh-1)
77 txep = &txq->sw_ring[txq->tx_next_dd - (txq->tx_free_thresh - 1)];
78 for (i = 0; i < txq->tx_free_thresh; ++i, ++txep) {
79 /* free buffers one at a time */
80 m = rte_pktmbuf_prefree_seg(txep->mbuf);
83 if (unlikely(m == NULL))
86 if (nb_free >= RTE_NGBE_TX_MAX_FREE_BUF_SZ ||
87 (nb_free > 0 && m->pool != free[0]->pool)) {
88 rte_mempool_put_bulk(free[0]->pool,
89 (void **)free, nb_free);
97 rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free);
99 /* buffers were freed, update counters */
100 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_free_thresh);
101 txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_free_thresh);
102 if (txq->tx_next_dd >= txq->nb_tx_desc)
103 txq->tx_next_dd = (uint16_t)(txq->tx_free_thresh - 1);
105 return txq->tx_free_thresh;
108 /* Populate 4 descriptors with data from 4 mbufs */
110 tx4(volatile struct ngbe_tx_desc *txdp, struct rte_mbuf **pkts)
112 uint64_t buf_dma_addr;
116 for (i = 0; i < 4; ++i, ++txdp, ++pkts) {
117 buf_dma_addr = rte_mbuf_data_iova(*pkts);
118 pkt_len = (*pkts)->data_len;
120 /* write data to descriptor */
121 txdp->qw0 = rte_cpu_to_le_64(buf_dma_addr);
122 txdp->dw2 = cpu_to_le32(NGBE_TXD_FLAGS |
123 NGBE_TXD_DATLEN(pkt_len));
124 txdp->dw3 = cpu_to_le32(NGBE_TXD_PAYLEN(pkt_len));
126 rte_prefetch0(&(*pkts)->pool);
130 /* Populate 1 descriptor with data from 1 mbuf */
132 tx1(volatile struct ngbe_tx_desc *txdp, struct rte_mbuf **pkts)
134 uint64_t buf_dma_addr;
137 buf_dma_addr = rte_mbuf_data_iova(*pkts);
138 pkt_len = (*pkts)->data_len;
140 /* write data to descriptor */
141 txdp->qw0 = cpu_to_le64(buf_dma_addr);
142 txdp->dw2 = cpu_to_le32(NGBE_TXD_FLAGS |
143 NGBE_TXD_DATLEN(pkt_len));
144 txdp->dw3 = cpu_to_le32(NGBE_TXD_PAYLEN(pkt_len));
146 rte_prefetch0(&(*pkts)->pool);
150 * Fill H/W descriptor ring with mbuf data.
151 * Copy mbuf pointers to the S/W ring.
154 ngbe_tx_fill_hw_ring(struct ngbe_tx_queue *txq, struct rte_mbuf **pkts,
157 volatile struct ngbe_tx_desc *txdp = &txq->tx_ring[txq->tx_tail];
158 struct ngbe_tx_entry *txep = &txq->sw_ring[txq->tx_tail];
159 const int N_PER_LOOP = 4;
160 const int N_PER_LOOP_MASK = N_PER_LOOP - 1;
161 int mainpart, leftover;
165 * Process most of the packets in chunks of N pkts. Any
166 * leftover packets will get processed one at a time.
168 mainpart = (nb_pkts & ((uint32_t)~N_PER_LOOP_MASK));
169 leftover = (nb_pkts & ((uint32_t)N_PER_LOOP_MASK));
170 for (i = 0; i < mainpart; i += N_PER_LOOP) {
171 /* Copy N mbuf pointers to the S/W ring */
172 for (j = 0; j < N_PER_LOOP; ++j)
173 (txep + i + j)->mbuf = *(pkts + i + j);
174 tx4(txdp + i, pkts + i);
177 if (unlikely(leftover > 0)) {
178 for (i = 0; i < leftover; ++i) {
179 (txep + mainpart + i)->mbuf = *(pkts + mainpart + i);
180 tx1(txdp + mainpart + i, pkts + mainpart + i);
185 static inline uint16_t
186 tx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
189 struct ngbe_tx_queue *txq = (struct ngbe_tx_queue *)tx_queue;
193 * Begin scanning the H/W ring for done descriptors when the
194 * number of available descriptors drops below tx_free_thresh.
195 * For each done descriptor, free the associated buffer.
197 if (txq->nb_tx_free < txq->tx_free_thresh)
198 ngbe_tx_free_bufs(txq);
200 /* Only use descriptors that are available */
201 nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts);
202 if (unlikely(nb_pkts == 0))
205 /* Use exactly nb_pkts descriptors */
206 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts);
209 * At this point, we know there are enough descriptors in the
210 * ring to transmit all the packets. This assumes that each
211 * mbuf contains a single segment, and that no new offloads
212 * are expected, which would require a new context descriptor.
216 * See if we're going to wrap-around. If so, handle the top
217 * of the descriptor ring first, then do the bottom. If not,
218 * the processing looks just like the "bottom" part anyway...
220 if ((txq->tx_tail + nb_pkts) > txq->nb_tx_desc) {
221 n = (uint16_t)(txq->nb_tx_desc - txq->tx_tail);
222 ngbe_tx_fill_hw_ring(txq, tx_pkts, n);
226 /* Fill H/W descriptor ring with mbuf data */
227 ngbe_tx_fill_hw_ring(txq, tx_pkts + n, (uint16_t)(nb_pkts - n));
228 txq->tx_tail = (uint16_t)(txq->tx_tail + (nb_pkts - n));
231 * Check for wrap-around. This would only happen if we used
232 * up to the last descriptor in the ring, no more, no less.
234 if (txq->tx_tail >= txq->nb_tx_desc)
237 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u",
238 (uint16_t)txq->port_id, (uint16_t)txq->queue_id,
239 (uint16_t)txq->tx_tail, (uint16_t)nb_pkts);
241 /* update tail pointer */
243 ngbe_set32_relaxed(txq->tdt_reg_addr, txq->tx_tail);
249 ngbe_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts,
254 /* Try to transmit at least chunks of TX_MAX_BURST pkts */
255 if (likely(nb_pkts <= RTE_PMD_NGBE_TX_MAX_BURST))
256 return tx_xmit_pkts(tx_queue, tx_pkts, nb_pkts);
258 /* transmit more than the max burst, in chunks of TX_MAX_BURST */
260 while (nb_pkts != 0) {
263 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_NGBE_TX_MAX_BURST);
264 ret = tx_xmit_pkts(tx_queue, &tx_pkts[nb_tx], n);
265 nb_tx = (uint16_t)(nb_tx + ret);
266 nb_pkts = (uint16_t)(nb_pkts - ret);
275 ngbe_set_xmit_ctx(struct ngbe_tx_queue *txq,
276 volatile struct ngbe_tx_ctx_desc *ctx_txd,
277 uint64_t ol_flags, union ngbe_tx_offload tx_offload)
279 union ngbe_tx_offload tx_offload_mask;
280 uint32_t type_tucmd_mlhl;
281 uint32_t mss_l4len_idx;
283 uint32_t vlan_macip_lens;
284 uint32_t tunnel_seed;
286 ctx_idx = txq->ctx_curr;
287 tx_offload_mask.data[0] = 0;
288 tx_offload_mask.data[1] = 0;
290 /* Specify which HW CTX to upload. */
291 mss_l4len_idx = NGBE_TXD_IDX(ctx_idx);
292 type_tucmd_mlhl = NGBE_TXD_CTXT;
294 tx_offload_mask.ptid |= ~0;
295 type_tucmd_mlhl |= NGBE_TXD_PTID(tx_offload.ptid);
297 /* check if TCP segmentation required for this packet */
298 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {
299 tx_offload_mask.l2_len |= ~0;
300 tx_offload_mask.l3_len |= ~0;
301 tx_offload_mask.l4_len |= ~0;
302 tx_offload_mask.tso_segsz |= ~0;
303 mss_l4len_idx |= NGBE_TXD_MSS(tx_offload.tso_segsz);
304 mss_l4len_idx |= NGBE_TXD_L4LEN(tx_offload.l4_len);
305 } else { /* no TSO, check if hardware checksum is needed */
306 if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM) {
307 tx_offload_mask.l2_len |= ~0;
308 tx_offload_mask.l3_len |= ~0;
311 switch (ol_flags & RTE_MBUF_F_TX_L4_MASK) {
312 case RTE_MBUF_F_TX_UDP_CKSUM:
314 NGBE_TXD_L4LEN(sizeof(struct rte_udp_hdr));
315 tx_offload_mask.l2_len |= ~0;
316 tx_offload_mask.l3_len |= ~0;
318 case RTE_MBUF_F_TX_TCP_CKSUM:
320 NGBE_TXD_L4LEN(sizeof(struct rte_tcp_hdr));
321 tx_offload_mask.l2_len |= ~0;
322 tx_offload_mask.l3_len |= ~0;
324 case RTE_MBUF_F_TX_SCTP_CKSUM:
326 NGBE_TXD_L4LEN(sizeof(struct rte_sctp_hdr));
327 tx_offload_mask.l2_len |= ~0;
328 tx_offload_mask.l3_len |= ~0;
335 vlan_macip_lens = NGBE_TXD_IPLEN(tx_offload.l3_len >> 1);
337 if (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) {
338 tx_offload_mask.outer_tun_len |= ~0;
339 tx_offload_mask.outer_l2_len |= ~0;
340 tx_offload_mask.outer_l3_len |= ~0;
341 tx_offload_mask.l2_len |= ~0;
342 tunnel_seed = NGBE_TXD_ETUNLEN(tx_offload.outer_tun_len >> 1);
343 tunnel_seed |= NGBE_TXD_EIPLEN(tx_offload.outer_l3_len >> 2);
345 switch (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) {
346 case RTE_MBUF_F_TX_TUNNEL_IPIP:
347 /* for non UDP / GRE tunneling, set to 0b */
350 PMD_TX_LOG(ERR, "Tunnel type not supported");
353 vlan_macip_lens |= NGBE_TXD_MACLEN(tx_offload.outer_l2_len);
356 vlan_macip_lens |= NGBE_TXD_MACLEN(tx_offload.l2_len);
359 if (ol_flags & RTE_MBUF_F_TX_VLAN) {
360 tx_offload_mask.vlan_tci |= ~0;
361 vlan_macip_lens |= NGBE_TXD_VLAN(tx_offload.vlan_tci);
364 txq->ctx_cache[ctx_idx].flags = ol_flags;
365 txq->ctx_cache[ctx_idx].tx_offload.data[0] =
366 tx_offload_mask.data[0] & tx_offload.data[0];
367 txq->ctx_cache[ctx_idx].tx_offload.data[1] =
368 tx_offload_mask.data[1] & tx_offload.data[1];
369 txq->ctx_cache[ctx_idx].tx_offload_mask = tx_offload_mask;
371 ctx_txd->dw0 = rte_cpu_to_le_32(vlan_macip_lens);
372 ctx_txd->dw1 = rte_cpu_to_le_32(tunnel_seed);
373 ctx_txd->dw2 = rte_cpu_to_le_32(type_tucmd_mlhl);
374 ctx_txd->dw3 = rte_cpu_to_le_32(mss_l4len_idx);
378 * Check which hardware context can be used. Use the existing match
379 * or create a new context descriptor.
381 static inline uint32_t
382 what_ctx_update(struct ngbe_tx_queue *txq, uint64_t flags,
383 union ngbe_tx_offload tx_offload)
385 /* If match with the current used context */
386 if (likely(txq->ctx_cache[txq->ctx_curr].flags == flags &&
387 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] ==
388 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0]
389 & tx_offload.data[0])) &&
390 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] ==
391 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1]
392 & tx_offload.data[1]))))
393 return txq->ctx_curr;
395 /* What if match with the next context */
397 if (likely(txq->ctx_cache[txq->ctx_curr].flags == flags &&
398 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] ==
399 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0]
400 & tx_offload.data[0])) &&
401 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] ==
402 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1]
403 & tx_offload.data[1]))))
404 return txq->ctx_curr;
406 /* Mismatch, use the previous context */
410 static inline uint32_t
411 tx_desc_cksum_flags_to_olinfo(uint64_t ol_flags)
415 if ((ol_flags & RTE_MBUF_F_TX_L4_MASK) != RTE_MBUF_F_TX_L4_NO_CKSUM) {
417 tmp |= NGBE_TXD_L4CS;
419 if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM) {
421 tmp |= NGBE_TXD_IPCS;
423 if (ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) {
425 tmp |= NGBE_TXD_EIPCS;
427 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {
429 /* implies IPv4 cksum */
430 if (ol_flags & RTE_MBUF_F_TX_IPV4)
431 tmp |= NGBE_TXD_IPCS;
432 tmp |= NGBE_TXD_L4CS;
434 if (ol_flags & RTE_MBUF_F_TX_VLAN)
440 static inline uint32_t
441 tx_desc_ol_flags_to_cmdtype(uint64_t ol_flags)
443 uint32_t cmdtype = 0;
445 if (ol_flags & RTE_MBUF_F_TX_VLAN)
446 cmdtype |= NGBE_TXD_VLE;
447 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG)
448 cmdtype |= NGBE_TXD_TSE;
452 static inline uint8_t
453 tx_desc_ol_flags_to_ptid(uint64_t oflags, uint32_t ptype)
458 return ngbe_encode_ptype(ptype);
460 /* Only support flags in NGBE_TX_OFFLOAD_MASK */
461 tun = !!(oflags & RTE_MBUF_F_TX_TUNNEL_MASK);
464 ptype = RTE_PTYPE_L2_ETHER;
465 if (oflags & RTE_MBUF_F_TX_VLAN)
466 ptype |= RTE_PTYPE_L2_ETHER_VLAN;
469 if (oflags & (RTE_MBUF_F_TX_OUTER_IPV4 | RTE_MBUF_F_TX_OUTER_IP_CKSUM))
470 ptype |= RTE_PTYPE_L3_IPV4;
471 else if (oflags & (RTE_MBUF_F_TX_OUTER_IPV6))
472 ptype |= RTE_PTYPE_L3_IPV6;
474 if (oflags & (RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_IP_CKSUM))
475 ptype |= (tun ? RTE_PTYPE_INNER_L3_IPV4 : RTE_PTYPE_L3_IPV4);
476 else if (oflags & (RTE_MBUF_F_TX_IPV6))
477 ptype |= (tun ? RTE_PTYPE_INNER_L3_IPV6 : RTE_PTYPE_L3_IPV6);
480 switch (oflags & (RTE_MBUF_F_TX_L4_MASK)) {
481 case RTE_MBUF_F_TX_TCP_CKSUM:
482 ptype |= (tun ? RTE_PTYPE_INNER_L4_TCP : RTE_PTYPE_L4_TCP);
484 case RTE_MBUF_F_TX_UDP_CKSUM:
485 ptype |= (tun ? RTE_PTYPE_INNER_L4_UDP : RTE_PTYPE_L4_UDP);
487 case RTE_MBUF_F_TX_SCTP_CKSUM:
488 ptype |= (tun ? RTE_PTYPE_INNER_L4_SCTP : RTE_PTYPE_L4_SCTP);
492 if (oflags & RTE_MBUF_F_TX_TCP_SEG)
493 ptype |= (tun ? RTE_PTYPE_INNER_L4_TCP : RTE_PTYPE_L4_TCP);
496 switch (oflags & RTE_MBUF_F_TX_TUNNEL_MASK) {
497 case RTE_MBUF_F_TX_TUNNEL_IPIP:
498 case RTE_MBUF_F_TX_TUNNEL_IP:
499 ptype |= RTE_PTYPE_L2_ETHER |
505 return ngbe_encode_ptype(ptype);
508 /* Reset transmit descriptors after they have been used */
510 ngbe_xmit_cleanup(struct ngbe_tx_queue *txq)
512 struct ngbe_tx_entry *sw_ring = txq->sw_ring;
513 volatile struct ngbe_tx_desc *txr = txq->tx_ring;
514 uint16_t last_desc_cleaned = txq->last_desc_cleaned;
515 uint16_t nb_tx_desc = txq->nb_tx_desc;
516 uint16_t desc_to_clean_to;
517 uint16_t nb_tx_to_clean;
520 /* Determine the last descriptor needing to be cleaned */
521 desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->tx_free_thresh);
522 if (desc_to_clean_to >= nb_tx_desc)
523 desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc);
525 /* Check to make sure the last descriptor to clean is done */
526 desc_to_clean_to = sw_ring[desc_to_clean_to].last_id;
527 status = txr[desc_to_clean_to].dw3;
528 if (!(status & rte_cpu_to_le_32(NGBE_TXD_DD))) {
530 "Tx descriptor %4u is not done"
531 "(port=%d queue=%d)",
533 txq->port_id, txq->queue_id);
534 if (txq->nb_tx_free >> 1 < txq->tx_free_thresh)
535 ngbe_set32_masked(txq->tdc_reg_addr,
536 NGBE_TXCFG_FLUSH, NGBE_TXCFG_FLUSH);
537 /* Failed to clean any descriptors, better luck next time */
541 /* Figure out how many descriptors will be cleaned */
542 if (last_desc_cleaned > desc_to_clean_to)
543 nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) +
546 nb_tx_to_clean = (uint16_t)(desc_to_clean_to -
550 "Cleaning %4u Tx descriptors: %4u to %4u (port=%d queue=%d)",
551 nb_tx_to_clean, last_desc_cleaned, desc_to_clean_to,
552 txq->port_id, txq->queue_id);
555 * The last descriptor to clean is done, so that means all the
556 * descriptors from the last descriptor that was cleaned
557 * up to the last descriptor with the RS bit set
558 * are done. Only reset the threshold descriptor.
560 txr[desc_to_clean_to].dw3 = 0;
562 /* Update the txq to reflect the last descriptor that was cleaned */
563 txq->last_desc_cleaned = desc_to_clean_to;
564 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + nb_tx_to_clean);
571 ngbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
574 struct ngbe_tx_queue *txq;
575 struct ngbe_tx_entry *sw_ring;
576 struct ngbe_tx_entry *txe, *txn;
577 volatile struct ngbe_tx_desc *txr;
578 volatile struct ngbe_tx_desc *txd;
579 struct rte_mbuf *tx_pkt;
580 struct rte_mbuf *m_seg;
581 uint64_t buf_dma_addr;
582 uint32_t olinfo_status;
583 uint32_t cmd_type_len;
594 union ngbe_tx_offload tx_offload;
596 tx_offload.data[0] = 0;
597 tx_offload.data[1] = 0;
599 sw_ring = txq->sw_ring;
601 tx_id = txq->tx_tail;
602 txe = &sw_ring[tx_id];
604 /* Determine if the descriptor ring needs to be cleaned. */
605 if (txq->nb_tx_free < txq->tx_free_thresh)
606 ngbe_xmit_cleanup(txq);
608 rte_prefetch0(&txe->mbuf->pool);
611 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
614 pkt_len = tx_pkt->pkt_len;
617 * Determine how many (if any) context descriptors
618 * are needed for offload functionality.
620 ol_flags = tx_pkt->ol_flags;
622 /* If hardware offload required */
623 tx_ol_req = ol_flags & NGBE_TX_OFFLOAD_MASK;
625 tx_offload.ptid = tx_desc_ol_flags_to_ptid(tx_ol_req,
626 tx_pkt->packet_type);
627 tx_offload.l2_len = tx_pkt->l2_len;
628 tx_offload.l3_len = tx_pkt->l3_len;
629 tx_offload.l4_len = tx_pkt->l4_len;
630 tx_offload.vlan_tci = tx_pkt->vlan_tci;
631 tx_offload.tso_segsz = tx_pkt->tso_segsz;
632 tx_offload.outer_l2_len = tx_pkt->outer_l2_len;
633 tx_offload.outer_l3_len = tx_pkt->outer_l3_len;
634 tx_offload.outer_tun_len = 0;
636 /* If new context need be built or reuse the exist ctx*/
637 ctx = what_ctx_update(txq, tx_ol_req, tx_offload);
638 /* Only allocate context descriptor if required */
639 new_ctx = (ctx == NGBE_CTX_NUM);
644 * Keep track of how many descriptors are used this loop
645 * This will always be the number of segments + the number of
646 * Context descriptors required to transmit the packet
648 nb_used = (uint16_t)(tx_pkt->nb_segs + new_ctx);
651 * The number of descriptors that must be allocated for a
652 * packet is the number of segments of that packet, plus 1
653 * Context Descriptor for the hardware offload, if any.
654 * Determine the last Tx descriptor to allocate in the Tx ring
655 * for the packet, starting from the current position (tx_id)
658 tx_last = (uint16_t)(tx_id + nb_used - 1);
661 if (tx_last >= txq->nb_tx_desc)
662 tx_last = (uint16_t)(tx_last - txq->nb_tx_desc);
664 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u pktlen=%u"
665 " tx_first=%u tx_last=%u",
666 (uint16_t)txq->port_id,
667 (uint16_t)txq->queue_id,
673 * Make sure there are enough Tx descriptors available to
674 * transmit the entire packet.
675 * nb_used better be less than or equal to txq->tx_free_thresh
677 if (nb_used > txq->nb_tx_free) {
679 "Not enough free Tx descriptors "
680 "nb_used=%4u nb_free=%4u "
681 "(port=%d queue=%d)",
682 nb_used, txq->nb_tx_free,
683 txq->port_id, txq->queue_id);
685 if (ngbe_xmit_cleanup(txq) != 0) {
686 /* Could not clean any descriptors */
692 /* nb_used better be <= txq->tx_free_thresh */
693 if (unlikely(nb_used > txq->tx_free_thresh)) {
695 "The number of descriptors needed to "
696 "transmit the packet exceeds the "
697 "RS bit threshold. This will impact "
699 "nb_used=%4u nb_free=%4u "
700 "tx_free_thresh=%4u. "
701 "(port=%d queue=%d)",
702 nb_used, txq->nb_tx_free,
704 txq->port_id, txq->queue_id);
706 * Loop here until there are enough Tx
707 * descriptors or until the ring cannot be
710 while (nb_used > txq->nb_tx_free) {
711 if (ngbe_xmit_cleanup(txq) != 0) {
713 * Could not clean any
725 * By now there are enough free Tx descriptors to transmit
730 * Set common flags of all Tx Data Descriptors.
732 * The following bits must be set in the first Data Descriptor
733 * and are ignored in the other ones:
736 * The following bits must only be set in the last Data
740 cmd_type_len = NGBE_TXD_FCS;
742 #ifdef RTE_LIBRTE_IEEE1588
743 if (ol_flags & PKT_TX_IEEE1588_TMST)
744 cmd_type_len |= NGBE_TXD_1588;
749 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {
750 /* when TSO is on, paylen in descriptor is the
751 * not the packet len but the tcp payload len
753 pkt_len -= (tx_offload.l2_len +
754 tx_offload.l3_len + tx_offload.l4_len);
756 (tx_pkt->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK)
757 ? tx_offload.outer_l2_len +
758 tx_offload.outer_l3_len : 0;
762 * Setup the Tx Context Descriptor if required
765 volatile struct ngbe_tx_ctx_desc *ctx_txd;
767 ctx_txd = (volatile struct ngbe_tx_ctx_desc *)
770 txn = &sw_ring[txe->next_id];
771 rte_prefetch0(&txn->mbuf->pool);
773 if (txe->mbuf != NULL) {
774 rte_pktmbuf_free_seg(txe->mbuf);
778 ngbe_set_xmit_ctx(txq, ctx_txd, tx_ol_req,
781 txe->last_id = tx_last;
782 tx_id = txe->next_id;
787 * Setup the Tx Data Descriptor,
788 * This path will go through
789 * whatever new/reuse the context descriptor
791 cmd_type_len |= tx_desc_ol_flags_to_cmdtype(ol_flags);
793 tx_desc_cksum_flags_to_olinfo(ol_flags);
794 olinfo_status |= NGBE_TXD_IDX(ctx);
797 olinfo_status |= NGBE_TXD_PAYLEN(pkt_len);
802 txn = &sw_ring[txe->next_id];
803 rte_prefetch0(&txn->mbuf->pool);
805 if (txe->mbuf != NULL)
806 rte_pktmbuf_free_seg(txe->mbuf);
810 * Set up Transmit Data Descriptor.
812 slen = m_seg->data_len;
813 buf_dma_addr = rte_mbuf_data_iova(m_seg);
814 txd->qw0 = rte_cpu_to_le_64(buf_dma_addr);
815 txd->dw2 = rte_cpu_to_le_32(cmd_type_len | slen);
816 txd->dw3 = rte_cpu_to_le_32(olinfo_status);
817 txe->last_id = tx_last;
818 tx_id = txe->next_id;
821 } while (m_seg != NULL);
824 * The last packet data descriptor needs End Of Packet (EOP)
826 cmd_type_len |= NGBE_TXD_EOP;
827 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_used);
829 txd->dw2 |= rte_cpu_to_le_32(cmd_type_len);
837 * Set the Transmit Descriptor Tail (TDT)
839 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u",
840 (uint16_t)txq->port_id, (uint16_t)txq->queue_id,
841 (uint16_t)tx_id, (uint16_t)nb_tx);
842 ngbe_set32_relaxed(txq->tdt_reg_addr, tx_id);
843 txq->tx_tail = tx_id;
848 /*********************************************************************
852 **********************************************************************/
854 ngbe_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
859 struct ngbe_tx_queue *txq = (struct ngbe_tx_queue *)tx_queue;
861 for (i = 0; i < nb_pkts; i++) {
863 ol_flags = m->ol_flags;
866 * Check if packet meets requirements for number of segments
868 * NOTE: for ngbe it's always (40 - WTHRESH) for both TSO and
872 if (m->nb_segs > NGBE_TX_MAX_SEG - txq->wthresh) {
877 if (ol_flags & NGBE_TX_OFFLOAD_NOTSUP_MASK) {
878 rte_errno = -ENOTSUP;
882 #ifdef RTE_ETHDEV_DEBUG_TX
883 ret = rte_validate_tx_offload(m);
889 ret = rte_net_intel_cksum_prepare(m);
899 /*********************************************************************
903 **********************************************************************/
904 static inline uint32_t
905 ngbe_rxd_pkt_info_to_pkt_type(uint32_t pkt_info, uint16_t ptid_mask)
907 uint16_t ptid = NGBE_RXD_PTID(pkt_info);
911 return ngbe_decode_ptype(ptid);
914 static inline uint64_t
915 ngbe_rxd_pkt_info_to_pkt_flags(uint32_t pkt_info)
917 static uint64_t ip_rss_types_map[16] __rte_cache_aligned = {
918 0, RTE_MBUF_F_RX_RSS_HASH, RTE_MBUF_F_RX_RSS_HASH, RTE_MBUF_F_RX_RSS_HASH,
919 0, RTE_MBUF_F_RX_RSS_HASH, 0, RTE_MBUF_F_RX_RSS_HASH,
920 RTE_MBUF_F_RX_RSS_HASH, 0, 0, 0,
921 0, 0, 0, RTE_MBUF_F_RX_FDIR,
923 #ifdef RTE_LIBRTE_IEEE1588
924 static uint64_t ip_pkt_etqf_map[8] = {
925 0, 0, 0, PKT_RX_IEEE1588_PTP,
928 int etfid = ngbe_etflt_id(NGBE_RXD_PTID(pkt_info));
929 if (likely(-1 != etfid))
930 return ip_pkt_etqf_map[etfid] |
931 ip_rss_types_map[NGBE_RXD_RSSTYPE(pkt_info)];
933 return ip_rss_types_map[NGBE_RXD_RSSTYPE(pkt_info)];
935 return ip_rss_types_map[NGBE_RXD_RSSTYPE(pkt_info)];
939 static inline uint64_t
940 rx_desc_status_to_pkt_flags(uint32_t rx_status, uint64_t vlan_flags)
945 * Check if VLAN present only.
946 * Do not check whether L3/L4 rx checksum done by NIC or not,
947 * That can be found from rte_eth_rxmode.offloads flag
949 pkt_flags = (rx_status & NGBE_RXD_STAT_VLAN &&
950 vlan_flags & RTE_MBUF_F_RX_VLAN_STRIPPED)
953 #ifdef RTE_LIBRTE_IEEE1588
954 if (rx_status & NGBE_RXD_STAT_1588)
955 pkt_flags = pkt_flags | PKT_RX_IEEE1588_TMST;
960 static inline uint64_t
961 rx_desc_error_to_pkt_flags(uint32_t rx_status)
963 uint64_t pkt_flags = 0;
965 /* checksum offload can't be disabled */
966 if (rx_status & NGBE_RXD_STAT_IPCS)
967 pkt_flags |= (rx_status & NGBE_RXD_ERR_IPCS
968 ? RTE_MBUF_F_RX_IP_CKSUM_BAD : RTE_MBUF_F_RX_IP_CKSUM_GOOD);
970 if (rx_status & NGBE_RXD_STAT_L4CS)
971 pkt_flags |= (rx_status & NGBE_RXD_ERR_L4CS
972 ? RTE_MBUF_F_RX_L4_CKSUM_BAD : RTE_MBUF_F_RX_L4_CKSUM_GOOD);
974 if (rx_status & NGBE_RXD_STAT_EIPCS &&
975 rx_status & NGBE_RXD_ERR_EIPCS)
976 pkt_flags |= RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD;
982 * LOOK_AHEAD defines how many desc statuses to check beyond the
983 * current descriptor.
984 * It must be a pound define for optimal performance.
985 * Do not change the value of LOOK_AHEAD, as the ngbe_rx_scan_hw_ring
986 * function only works with LOOK_AHEAD=8.
989 #if (LOOK_AHEAD != 8)
990 #error "PMD NGBE: LOOK_AHEAD must be 8\n"
993 ngbe_rx_scan_hw_ring(struct ngbe_rx_queue *rxq)
995 volatile struct ngbe_rx_desc *rxdp;
996 struct ngbe_rx_entry *rxep;
1001 uint32_t s[LOOK_AHEAD];
1002 uint32_t pkt_info[LOOK_AHEAD];
1003 int i, j, nb_rx = 0;
1006 /* get references to current descriptor and S/W ring entry */
1007 rxdp = &rxq->rx_ring[rxq->rx_tail];
1008 rxep = &rxq->sw_ring[rxq->rx_tail];
1010 status = rxdp->qw1.lo.status;
1011 /* check to make sure there is at least 1 packet to receive */
1012 if (!(status & rte_cpu_to_le_32(NGBE_RXD_STAT_DD)))
1016 * Scan LOOK_AHEAD descriptors at a time to determine which descriptors
1017 * reference packets that are ready to be received.
1019 for (i = 0; i < RTE_PMD_NGBE_RX_MAX_BURST;
1020 i += LOOK_AHEAD, rxdp += LOOK_AHEAD, rxep += LOOK_AHEAD) {
1021 /* Read desc statuses backwards to avoid race condition */
1022 for (j = 0; j < LOOK_AHEAD; j++)
1023 s[j] = rte_le_to_cpu_32(rxdp[j].qw1.lo.status);
1025 rte_atomic_thread_fence(__ATOMIC_ACQUIRE);
1027 /* Compute how many status bits were set */
1028 for (nb_dd = 0; nb_dd < LOOK_AHEAD &&
1029 (s[nb_dd] & NGBE_RXD_STAT_DD); nb_dd++)
1032 for (j = 0; j < nb_dd; j++)
1033 pkt_info[j] = rte_le_to_cpu_32(rxdp[j].qw0.dw0);
1037 /* Translate descriptor info to mbuf format */
1038 for (j = 0; j < nb_dd; ++j) {
1040 pkt_len = rte_le_to_cpu_16(rxdp[j].qw1.hi.len) -
1042 mb->data_len = pkt_len;
1043 mb->pkt_len = pkt_len;
1044 mb->vlan_tci = rte_le_to_cpu_16(rxdp[j].qw1.hi.tag);
1046 /* convert descriptor fields to rte mbuf flags */
1047 pkt_flags = rx_desc_status_to_pkt_flags(s[j],
1049 pkt_flags |= rx_desc_error_to_pkt_flags(s[j]);
1051 ngbe_rxd_pkt_info_to_pkt_flags(pkt_info[j]);
1052 mb->ol_flags = pkt_flags;
1054 ngbe_rxd_pkt_info_to_pkt_type(pkt_info[j],
1057 if (likely(pkt_flags & RTE_MBUF_F_RX_RSS_HASH))
1059 rte_le_to_cpu_32(rxdp[j].qw0.dw1);
1062 /* Move mbuf pointers from the S/W ring to the stage */
1063 for (j = 0; j < LOOK_AHEAD; ++j)
1064 rxq->rx_stage[i + j] = rxep[j].mbuf;
1066 /* stop if all requested packets could not be received */
1067 if (nb_dd != LOOK_AHEAD)
1071 /* clear software ring entries so we can cleanup correctly */
1072 for (i = 0; i < nb_rx; ++i)
1073 rxq->sw_ring[rxq->rx_tail + i].mbuf = NULL;
1079 ngbe_rx_alloc_bufs(struct ngbe_rx_queue *rxq, bool reset_mbuf)
1081 volatile struct ngbe_rx_desc *rxdp;
1082 struct ngbe_rx_entry *rxep;
1083 struct rte_mbuf *mb;
1088 /* allocate buffers in bulk directly into the S/W ring */
1089 alloc_idx = rxq->rx_free_trigger - (rxq->rx_free_thresh - 1);
1090 rxep = &rxq->sw_ring[alloc_idx];
1091 diag = rte_mempool_get_bulk(rxq->mb_pool, (void *)rxep,
1092 rxq->rx_free_thresh);
1093 if (unlikely(diag != 0))
1096 rxdp = &rxq->rx_ring[alloc_idx];
1097 for (i = 0; i < rxq->rx_free_thresh; ++i) {
1098 /* populate the static rte mbuf fields */
1101 mb->port = rxq->port_id;
1103 rte_mbuf_refcnt_set(mb, 1);
1104 mb->data_off = RTE_PKTMBUF_HEADROOM;
1106 /* populate the descriptors */
1107 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mb));
1108 NGBE_RXD_HDRADDR(&rxdp[i], 0);
1109 NGBE_RXD_PKTADDR(&rxdp[i], dma_addr);
1112 /* update state of internal queue structure */
1113 rxq->rx_free_trigger = rxq->rx_free_trigger + rxq->rx_free_thresh;
1114 if (rxq->rx_free_trigger >= rxq->nb_rx_desc)
1115 rxq->rx_free_trigger = rxq->rx_free_thresh - 1;
1121 static inline uint16_t
1122 ngbe_rx_fill_from_stage(struct ngbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
1125 struct rte_mbuf **stage = &rxq->rx_stage[rxq->rx_next_avail];
1128 /* how many packets are ready to return? */
1129 nb_pkts = (uint16_t)RTE_MIN(nb_pkts, rxq->rx_nb_avail);
1131 /* copy mbuf pointers to the application's packet list */
1132 for (i = 0; i < nb_pkts; ++i)
1133 rx_pkts[i] = stage[i];
1135 /* update internal queue state */
1136 rxq->rx_nb_avail = (uint16_t)(rxq->rx_nb_avail - nb_pkts);
1137 rxq->rx_next_avail = (uint16_t)(rxq->rx_next_avail + nb_pkts);
1142 static inline uint16_t
1143 ngbe_rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
1146 struct ngbe_rx_queue *rxq = (struct ngbe_rx_queue *)rx_queue;
1147 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id];
1150 /* Any previously recv'd pkts will be returned from the Rx stage */
1151 if (rxq->rx_nb_avail)
1152 return ngbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
1154 /* Scan the H/W ring for packets to receive */
1155 nb_rx = (uint16_t)ngbe_rx_scan_hw_ring(rxq);
1157 /* update internal queue state */
1158 rxq->rx_next_avail = 0;
1159 rxq->rx_nb_avail = nb_rx;
1160 rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_rx);
1162 /* if required, allocate new buffers to replenish descriptors */
1163 if (rxq->rx_tail > rxq->rx_free_trigger) {
1164 uint16_t cur_free_trigger = rxq->rx_free_trigger;
1166 if (ngbe_rx_alloc_bufs(rxq, true) != 0) {
1169 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u "
1170 "queue_id=%u", (uint16_t)rxq->port_id,
1171 (uint16_t)rxq->queue_id);
1173 dev->data->rx_mbuf_alloc_failed +=
1174 rxq->rx_free_thresh;
1177 * Need to rewind any previous receives if we cannot
1178 * allocate new buffers to replenish the old ones.
1180 rxq->rx_nb_avail = 0;
1181 rxq->rx_tail = (uint16_t)(rxq->rx_tail - nb_rx);
1182 for (i = 0, j = rxq->rx_tail; i < nb_rx; ++i, ++j)
1183 rxq->sw_ring[j].mbuf = rxq->rx_stage[i];
1188 /* update tail pointer */
1190 ngbe_set32_relaxed(rxq->rdt_reg_addr, cur_free_trigger);
1193 if (rxq->rx_tail >= rxq->nb_rx_desc)
1196 /* received any packets this loop? */
1197 if (rxq->rx_nb_avail)
1198 return ngbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
1203 /* split requests into chunks of size RTE_PMD_NGBE_RX_MAX_BURST */
1205 ngbe_recv_pkts_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
1210 if (unlikely(nb_pkts == 0))
1213 if (likely(nb_pkts <= RTE_PMD_NGBE_RX_MAX_BURST))
1214 return ngbe_rx_recv_pkts(rx_queue, rx_pkts, nb_pkts);
1216 /* request is relatively large, chunk it up */
1221 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_NGBE_RX_MAX_BURST);
1222 ret = ngbe_rx_recv_pkts(rx_queue, &rx_pkts[nb_rx], n);
1223 nb_rx = (uint16_t)(nb_rx + ret);
1224 nb_pkts = (uint16_t)(nb_pkts - ret);
1233 ngbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
1236 struct ngbe_rx_queue *rxq;
1237 volatile struct ngbe_rx_desc *rx_ring;
1238 volatile struct ngbe_rx_desc *rxdp;
1239 struct ngbe_rx_entry *sw_ring;
1240 struct ngbe_rx_entry *rxe;
1241 struct rte_mbuf *rxm;
1242 struct rte_mbuf *nmb;
1243 struct ngbe_rx_desc rxd;
1256 rx_id = rxq->rx_tail;
1257 rx_ring = rxq->rx_ring;
1258 sw_ring = rxq->sw_ring;
1259 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id];
1260 while (nb_rx < nb_pkts) {
1262 * The order of operations here is important as the DD status
1263 * bit must not be read after any other descriptor fields.
1264 * rx_ring and rxdp are pointing to volatile data so the order
1265 * of accesses cannot be reordered by the compiler. If they were
1266 * not volatile, they could be reordered which could lead to
1267 * using invalid descriptor fields when read from rxd.
1269 rxdp = &rx_ring[rx_id];
1270 staterr = rxdp->qw1.lo.status;
1271 if (!(staterr & rte_cpu_to_le_32(NGBE_RXD_STAT_DD)))
1278 * If the NGBE_RXD_STAT_EOP flag is not set, the Rx packet
1279 * is likely to be invalid and to be dropped by the various
1280 * validation checks performed by the network stack.
1282 * Allocate a new mbuf to replenish the RX ring descriptor.
1283 * If the allocation fails:
1284 * - arrange for that Rx descriptor to be the first one
1285 * being parsed the next time the receive function is
1286 * invoked [on the same queue].
1288 * - Stop parsing the Rx ring and return immediately.
1290 * This policy do not drop the packet received in the Rx
1291 * descriptor for which the allocation of a new mbuf failed.
1292 * Thus, it allows that packet to be later retrieved if
1293 * mbuf have been freed in the mean time.
1294 * As a side effect, holding Rx descriptors instead of
1295 * systematically giving them back to the NIC may lead to
1296 * Rx ring exhaustion situations.
1297 * However, the NIC can gracefully prevent such situations
1298 * to happen by sending specific "back-pressure" flow control
1299 * frames to its peer(s).
1302 "port_id=%u queue_id=%u rx_id=%u ext_err_stat=0x%08x pkt_len=%u",
1303 (uint16_t)rxq->port_id, (uint16_t)rxq->queue_id,
1304 (uint16_t)rx_id, (uint32_t)staterr,
1305 (uint16_t)rte_le_to_cpu_16(rxd.qw1.hi.len));
1307 nmb = rte_mbuf_raw_alloc(rxq->mb_pool);
1310 "Rx mbuf alloc failed port_id=%u queue_id=%u",
1311 (uint16_t)rxq->port_id,
1312 (uint16_t)rxq->queue_id);
1313 dev->data->rx_mbuf_alloc_failed++;
1318 rxe = &sw_ring[rx_id];
1320 if (rx_id == rxq->nb_rx_desc)
1323 /* Prefetch next mbuf while processing current one. */
1324 rte_ngbe_prefetch(sw_ring[rx_id].mbuf);
1327 * When next Rx descriptor is on a cache-line boundary,
1328 * prefetch the next 4 Rx descriptors and the next 8 pointers
1331 if ((rx_id & 0x3) == 0) {
1332 rte_ngbe_prefetch(&rx_ring[rx_id]);
1333 rte_ngbe_prefetch(&sw_ring[rx_id]);
1338 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
1339 NGBE_RXD_HDRADDR(rxdp, 0);
1340 NGBE_RXD_PKTADDR(rxdp, dma_addr);
1343 * Initialize the returned mbuf.
1344 * 1) setup generic mbuf fields:
1345 * - number of segments,
1348 * - Rx port identifier.
1349 * 2) integrate hardware offload data, if any:
1350 * - RSS flag & hash,
1351 * - IP checksum flag,
1352 * - VLAN TCI, if any,
1355 pkt_len = (uint16_t)(rte_le_to_cpu_16(rxd.qw1.hi.len) -
1357 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1358 rte_packet_prefetch((char *)rxm->buf_addr + rxm->data_off);
1361 rxm->pkt_len = pkt_len;
1362 rxm->data_len = pkt_len;
1363 rxm->port = rxq->port_id;
1365 pkt_info = rte_le_to_cpu_32(rxd.qw0.dw0);
1366 /* Only valid if RTE_MBUF_F_RX_VLAN set in pkt_flags */
1367 rxm->vlan_tci = rte_le_to_cpu_16(rxd.qw1.hi.tag);
1369 pkt_flags = rx_desc_status_to_pkt_flags(staterr,
1371 pkt_flags |= rx_desc_error_to_pkt_flags(staterr);
1372 pkt_flags |= ngbe_rxd_pkt_info_to_pkt_flags(pkt_info);
1373 rxm->ol_flags = pkt_flags;
1374 rxm->packet_type = ngbe_rxd_pkt_info_to_pkt_type(pkt_info,
1377 if (likely(pkt_flags & RTE_MBUF_F_RX_RSS_HASH))
1378 rxm->hash.rss = rte_le_to_cpu_32(rxd.qw0.dw1);
1381 * Store the mbuf address into the next entry of the array
1382 * of returned packets.
1384 rx_pkts[nb_rx++] = rxm;
1386 rxq->rx_tail = rx_id;
1389 * If the number of free Rx descriptors is greater than the Rx free
1390 * threshold of the queue, advance the Receive Descriptor Tail (RDT)
1392 * Update the RDT with the value of the last processed Rx descriptor
1393 * minus 1, to guarantee that the RDT register is never equal to the
1394 * RDH register, which creates a "full" ring situation from the
1395 * hardware point of view...
1397 nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold);
1398 if (nb_hold > rxq->rx_free_thresh) {
1400 "port_id=%u queue_id=%u rx_tail=%u nb_hold=%u nb_rx=%u",
1401 (uint16_t)rxq->port_id, (uint16_t)rxq->queue_id,
1402 (uint16_t)rx_id, (uint16_t)nb_hold,
1404 rx_id = (uint16_t)((rx_id == 0) ?
1405 (rxq->nb_rx_desc - 1) : (rx_id - 1));
1406 ngbe_set32(rxq->rdt_reg_addr, rx_id);
1409 rxq->nb_rx_hold = nb_hold;
1414 * ngbe_fill_cluster_head_buf - fill the first mbuf of the returned packet
1416 * Fill the following info in the HEAD buffer of the Rx cluster:
1417 * - RX port identifier
1418 * - hardware offload data, if any:
1420 * - IP checksum flag
1421 * - VLAN TCI, if any
1423 * @head HEAD of the packet cluster
1424 * @desc HW descriptor to get data from
1425 * @rxq Pointer to the Rx queue
1428 ngbe_fill_cluster_head_buf(struct rte_mbuf *head, struct ngbe_rx_desc *desc,
1429 struct ngbe_rx_queue *rxq, uint32_t staterr)
1434 head->port = rxq->port_id;
1436 /* The vlan_tci field is only valid when RTE_MBUF_F_RX_VLAN is
1437 * set in the pkt_flags field.
1439 head->vlan_tci = rte_le_to_cpu_16(desc->qw1.hi.tag);
1440 pkt_info = rte_le_to_cpu_32(desc->qw0.dw0);
1441 pkt_flags = rx_desc_status_to_pkt_flags(staterr, rxq->vlan_flags);
1442 pkt_flags |= rx_desc_error_to_pkt_flags(staterr);
1443 pkt_flags |= ngbe_rxd_pkt_info_to_pkt_flags(pkt_info);
1444 head->ol_flags = pkt_flags;
1445 head->packet_type = ngbe_rxd_pkt_info_to_pkt_type(pkt_info,
1448 if (likely(pkt_flags & RTE_MBUF_F_RX_RSS_HASH))
1449 head->hash.rss = rte_le_to_cpu_32(desc->qw0.dw1);
1453 * ngbe_recv_pkts_sc - receive handler for scatter case.
1455 * @rx_queue Rx queue handle
1456 * @rx_pkts table of received packets
1457 * @nb_pkts size of rx_pkts table
1458 * @bulk_alloc if TRUE bulk allocation is used for a HW ring refilling
1460 * Returns the number of received packets/clusters (according to the "bulk
1461 * receive" interface).
1463 static inline uint16_t
1464 ngbe_recv_pkts_sc(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts,
1467 struct ngbe_rx_queue *rxq = rx_queue;
1468 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id];
1469 volatile struct ngbe_rx_desc *rx_ring = rxq->rx_ring;
1470 struct ngbe_rx_entry *sw_ring = rxq->sw_ring;
1471 struct ngbe_scattered_rx_entry *sw_sc_ring = rxq->sw_sc_ring;
1472 uint16_t rx_id = rxq->rx_tail;
1474 uint16_t nb_hold = rxq->nb_rx_hold;
1475 uint16_t prev_id = rxq->rx_tail;
1477 while (nb_rx < nb_pkts) {
1479 struct ngbe_rx_entry *rxe;
1480 struct ngbe_scattered_rx_entry *sc_entry;
1481 struct ngbe_scattered_rx_entry *next_sc_entry = NULL;
1482 struct ngbe_rx_entry *next_rxe = NULL;
1483 struct rte_mbuf *first_seg;
1484 struct rte_mbuf *rxm;
1485 struct rte_mbuf *nmb = NULL;
1486 struct ngbe_rx_desc rxd;
1489 volatile struct ngbe_rx_desc *rxdp;
1493 rxdp = &rx_ring[rx_id];
1494 staterr = rte_le_to_cpu_32(rxdp->qw1.lo.status);
1496 if (!(staterr & NGBE_RXD_STAT_DD))
1501 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u "
1502 "staterr=0x%x data_len=%u",
1503 rxq->port_id, rxq->queue_id, rx_id, staterr,
1504 rte_le_to_cpu_16(rxd.qw1.hi.len));
1507 nmb = rte_mbuf_raw_alloc(rxq->mb_pool);
1509 PMD_RX_LOG(DEBUG, "Rx mbuf alloc failed "
1510 "port_id=%u queue_id=%u",
1511 rxq->port_id, rxq->queue_id);
1513 dev->data->rx_mbuf_alloc_failed++;
1516 } else if (nb_hold > rxq->rx_free_thresh) {
1517 uint16_t next_rdt = rxq->rx_free_trigger;
1519 if (!ngbe_rx_alloc_bufs(rxq, false)) {
1521 ngbe_set32_relaxed(rxq->rdt_reg_addr,
1523 nb_hold -= rxq->rx_free_thresh;
1525 PMD_RX_LOG(DEBUG, "Rx bulk alloc failed "
1526 "port_id=%u queue_id=%u",
1527 rxq->port_id, rxq->queue_id);
1529 dev->data->rx_mbuf_alloc_failed++;
1535 rxe = &sw_ring[rx_id];
1536 eop = staterr & NGBE_RXD_STAT_EOP;
1538 next_id = rx_id + 1;
1539 if (next_id == rxq->nb_rx_desc)
1542 /* Prefetch next mbuf while processing current one. */
1543 rte_ngbe_prefetch(sw_ring[next_id].mbuf);
1546 * When next Rx descriptor is on a cache-line boundary,
1547 * prefetch the next 4 RX descriptors and the next 4 pointers
1550 if ((next_id & 0x3) == 0) {
1551 rte_ngbe_prefetch(&rx_ring[next_id]);
1552 rte_ngbe_prefetch(&sw_ring[next_id]);
1559 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
1561 * Update Rx descriptor with the physical address of the
1562 * new data buffer of the new allocated mbuf.
1566 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1567 NGBE_RXD_HDRADDR(rxdp, 0);
1568 NGBE_RXD_PKTADDR(rxdp, dma);
1574 * Set data length & data buffer address of mbuf.
1576 data_len = rte_le_to_cpu_16(rxd.qw1.hi.len);
1577 rxm->data_len = data_len;
1583 next_sc_entry = &sw_sc_ring[nextp_id];
1584 next_rxe = &sw_ring[nextp_id];
1585 rte_ngbe_prefetch(next_rxe);
1588 sc_entry = &sw_sc_ring[rx_id];
1589 first_seg = sc_entry->fbuf;
1590 sc_entry->fbuf = NULL;
1593 * If this is the first buffer of the received packet,
1594 * set the pointer to the first mbuf of the packet and
1595 * initialize its context.
1596 * Otherwise, update the total length and the number of segments
1597 * of the current scattered packet, and update the pointer to
1598 * the last mbuf of the current packet.
1600 if (first_seg == NULL) {
1602 first_seg->pkt_len = data_len;
1603 first_seg->nb_segs = 1;
1605 first_seg->pkt_len += data_len;
1606 first_seg->nb_segs++;
1613 * If this is not the last buffer of the received packet, update
1614 * the pointer to the first mbuf at the NEXTP entry in the
1615 * sw_sc_ring and continue to parse the Rx ring.
1617 if (!eop && next_rxe) {
1618 rxm->next = next_rxe->mbuf;
1619 next_sc_entry->fbuf = first_seg;
1623 /* Initialize the first mbuf of the returned packet */
1624 ngbe_fill_cluster_head_buf(first_seg, &rxd, rxq, staterr);
1626 /* Deal with the case, when HW CRC srip is disabled. */
1627 first_seg->pkt_len -= rxq->crc_len;
1628 if (unlikely(rxm->data_len <= rxq->crc_len)) {
1629 struct rte_mbuf *lp;
1631 for (lp = first_seg; lp->next != rxm; lp = lp->next)
1634 first_seg->nb_segs--;
1635 lp->data_len -= rxq->crc_len - rxm->data_len;
1637 rte_pktmbuf_free_seg(rxm);
1639 rxm->data_len -= rxq->crc_len;
1642 /* Prefetch data of first segment, if configured to do so. */
1643 rte_packet_prefetch((char *)first_seg->buf_addr +
1644 first_seg->data_off);
1647 * Store the mbuf address into the next entry of the array
1648 * of returned packets.
1650 rx_pkts[nb_rx++] = first_seg;
1654 * Record index of the next Rx descriptor to probe.
1656 rxq->rx_tail = rx_id;
1659 * If the number of free Rx descriptors is greater than the Rx free
1660 * threshold of the queue, advance the Receive Descriptor Tail (RDT)
1662 * Update the RDT with the value of the last processed Rx descriptor
1663 * minus 1, to guarantee that the RDT register is never equal to the
1664 * RDH register, which creates a "full" ring situation from the
1665 * hardware point of view...
1667 if (!bulk_alloc && nb_hold > rxq->rx_free_thresh) {
1668 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u "
1669 "nb_hold=%u nb_rx=%u",
1670 rxq->port_id, rxq->queue_id, rx_id, nb_hold, nb_rx);
1673 ngbe_set32_relaxed(rxq->rdt_reg_addr, prev_id);
1677 rxq->nb_rx_hold = nb_hold;
1682 ngbe_recv_pkts_sc_single_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
1685 return ngbe_recv_pkts_sc(rx_queue, rx_pkts, nb_pkts, false);
1689 ngbe_recv_pkts_sc_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
1692 return ngbe_recv_pkts_sc(rx_queue, rx_pkts, nb_pkts, true);
1695 /*********************************************************************
1697 * Queue management functions
1699 **********************************************************************/
1702 ngbe_tx_queue_release_mbufs(struct ngbe_tx_queue *txq)
1706 if (txq->sw_ring != NULL) {
1707 for (i = 0; i < txq->nb_tx_desc; i++) {
1708 if (txq->sw_ring[i].mbuf != NULL) {
1709 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
1710 txq->sw_ring[i].mbuf = NULL;
1717 ngbe_tx_free_swring(struct ngbe_tx_queue *txq)
1720 rte_free(txq->sw_ring);
1724 ngbe_tx_queue_release(struct ngbe_tx_queue *txq)
1727 if (txq->ops != NULL) {
1728 txq->ops->release_mbufs(txq);
1729 txq->ops->free_swring(txq);
1736 ngbe_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
1738 ngbe_tx_queue_release(dev->data->tx_queues[qid]);
1741 /* (Re)set dynamic ngbe_tx_queue fields to defaults */
1743 ngbe_reset_tx_queue(struct ngbe_tx_queue *txq)
1745 static const struct ngbe_tx_desc zeroed_desc = {0};
1746 struct ngbe_tx_entry *txe = txq->sw_ring;
1749 /* Zero out HW ring memory */
1750 for (i = 0; i < txq->nb_tx_desc; i++)
1751 txq->tx_ring[i] = zeroed_desc;
1753 /* Initialize SW ring entries */
1754 prev = (uint16_t)(txq->nb_tx_desc - 1);
1755 for (i = 0; i < txq->nb_tx_desc; i++) {
1756 /* the ring can also be modified by hardware */
1757 volatile struct ngbe_tx_desc *txd = &txq->tx_ring[i];
1759 txd->dw3 = rte_cpu_to_le_32(NGBE_TXD_DD);
1762 txe[prev].next_id = i;
1766 txq->tx_next_dd = (uint16_t)(txq->tx_free_thresh - 1);
1770 * Always allow 1 descriptor to be un-allocated to avoid
1771 * a H/W race condition
1773 txq->last_desc_cleaned = (uint16_t)(txq->nb_tx_desc - 1);
1774 txq->nb_tx_free = (uint16_t)(txq->nb_tx_desc - 1);
1776 memset((void *)&txq->ctx_cache, 0,
1777 NGBE_CTX_NUM * sizeof(struct ngbe_ctx_info));
1780 static const struct ngbe_txq_ops def_txq_ops = {
1781 .release_mbufs = ngbe_tx_queue_release_mbufs,
1782 .free_swring = ngbe_tx_free_swring,
1783 .reset = ngbe_reset_tx_queue,
1786 /* Takes an ethdev and a queue and sets up the tx function to be used based on
1787 * the queue parameters. Used in tx_queue_setup by primary process and then
1788 * in dev_init by secondary process when attaching to an existing ethdev.
1791 ngbe_set_tx_function(struct rte_eth_dev *dev, struct ngbe_tx_queue *txq)
1793 /* Use a simple Tx queue (no offloads, no multi segs) if possible */
1794 if (txq->offloads == 0 &&
1795 txq->tx_free_thresh >= RTE_PMD_NGBE_TX_MAX_BURST) {
1796 PMD_INIT_LOG(DEBUG, "Using simple tx code path");
1797 dev->tx_pkt_burst = ngbe_xmit_pkts_simple;
1798 dev->tx_pkt_prepare = NULL;
1800 PMD_INIT_LOG(DEBUG, "Using full-featured tx code path");
1802 " - offloads = 0x%" PRIx64,
1805 " - tx_free_thresh = %lu [RTE_PMD_NGBE_TX_MAX_BURST=%lu]",
1806 (unsigned long)txq->tx_free_thresh,
1807 (unsigned long)RTE_PMD_NGBE_TX_MAX_BURST);
1808 dev->tx_pkt_burst = ngbe_xmit_pkts;
1809 dev->tx_pkt_prepare = ngbe_prep_pkts;
1813 static const struct {
1814 eth_tx_burst_t pkt_burst;
1816 } ngbe_tx_burst_infos[] = {
1817 { ngbe_xmit_pkts_simple, "Scalar Simple"},
1818 { ngbe_xmit_pkts, "Scalar"},
1822 ngbe_tx_burst_mode_get(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id,
1823 struct rte_eth_burst_mode *mode)
1825 eth_tx_burst_t pkt_burst = dev->tx_pkt_burst;
1829 for (i = 0; i < RTE_DIM(ngbe_tx_burst_infos); ++i) {
1830 if (pkt_burst == ngbe_tx_burst_infos[i].pkt_burst) {
1831 snprintf(mode->info, sizeof(mode->info), "%s",
1832 ngbe_tx_burst_infos[i].info);
1842 ngbe_get_tx_port_offloads(struct rte_eth_dev *dev)
1844 uint64_t tx_offload_capa;
1845 struct ngbe_hw *hw = ngbe_dev_hw(dev);
1848 RTE_ETH_TX_OFFLOAD_VLAN_INSERT |
1849 RTE_ETH_TX_OFFLOAD_IPV4_CKSUM |
1850 RTE_ETH_TX_OFFLOAD_UDP_CKSUM |
1851 RTE_ETH_TX_OFFLOAD_TCP_CKSUM |
1852 RTE_ETH_TX_OFFLOAD_SCTP_CKSUM |
1853 RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM |
1854 RTE_ETH_TX_OFFLOAD_TCP_TSO |
1855 RTE_ETH_TX_OFFLOAD_UDP_TSO |
1856 RTE_ETH_TX_OFFLOAD_UDP_TNL_TSO |
1857 RTE_ETH_TX_OFFLOAD_IP_TNL_TSO |
1858 RTE_ETH_TX_OFFLOAD_IPIP_TNL_TSO |
1859 RTE_ETH_TX_OFFLOAD_MULTI_SEGS;
1862 tx_offload_capa |= RTE_ETH_TX_OFFLOAD_QINQ_INSERT;
1864 return tx_offload_capa;
1868 ngbe_dev_tx_queue_setup(struct rte_eth_dev *dev,
1871 unsigned int socket_id,
1872 const struct rte_eth_txconf *tx_conf)
1874 const struct rte_memzone *tz;
1875 struct ngbe_tx_queue *txq;
1877 uint16_t tx_free_thresh;
1880 PMD_INIT_FUNC_TRACE();
1881 hw = ngbe_dev_hw(dev);
1883 offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads;
1886 * The Tx descriptor ring will be cleaned after txq->tx_free_thresh
1887 * descriptors are used or if the number of descriptors required
1888 * to transmit a packet is greater than the number of free Tx
1890 * One descriptor in the Tx ring is used as a sentinel to avoid a
1891 * H/W race condition, hence the maximum threshold constraints.
1892 * When set to zero use default values.
1894 tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ?
1895 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH);
1896 if (tx_free_thresh >= (nb_desc - 3)) {
1898 "tx_free_thresh must be less than the number of TX descriptors minus 3. (tx_free_thresh=%u port=%d queue=%d)",
1899 (unsigned int)tx_free_thresh,
1900 (int)dev->data->port_id, (int)queue_idx);
1904 if (nb_desc % tx_free_thresh != 0) {
1906 "tx_free_thresh must be a divisor of the number of Tx descriptors. (tx_free_thresh=%u port=%d queue=%d)",
1907 (unsigned int)tx_free_thresh,
1908 (int)dev->data->port_id, (int)queue_idx);
1912 /* Free memory prior to re-allocation if needed... */
1913 if (dev->data->tx_queues[queue_idx] != NULL) {
1914 ngbe_tx_queue_release(dev->data->tx_queues[queue_idx]);
1915 dev->data->tx_queues[queue_idx] = NULL;
1918 /* First allocate the Tx queue data structure */
1919 txq = rte_zmalloc_socket("ethdev Tx queue",
1920 sizeof(struct ngbe_tx_queue),
1921 RTE_CACHE_LINE_SIZE, socket_id);
1926 * Allocate Tx ring hardware descriptors. A memzone large enough to
1927 * handle the maximum ring size is allocated in order to allow for
1928 * resizing in later calls to the queue setup function.
1930 tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx,
1931 sizeof(struct ngbe_tx_desc) * NGBE_RING_DESC_MAX,
1932 NGBE_ALIGN, socket_id);
1934 ngbe_tx_queue_release(txq);
1938 txq->nb_tx_desc = nb_desc;
1939 txq->tx_free_thresh = tx_free_thresh;
1940 txq->pthresh = tx_conf->tx_thresh.pthresh;
1941 txq->hthresh = tx_conf->tx_thresh.hthresh;
1942 txq->wthresh = tx_conf->tx_thresh.wthresh;
1943 txq->queue_id = queue_idx;
1944 txq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ?
1945 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx);
1946 txq->port_id = dev->data->port_id;
1947 txq->offloads = offloads;
1948 txq->ops = &def_txq_ops;
1949 txq->tx_deferred_start = tx_conf->tx_deferred_start;
1951 txq->tdt_reg_addr = NGBE_REG_ADDR(hw, NGBE_TXWP(txq->reg_idx));
1952 txq->tdc_reg_addr = NGBE_REG_ADDR(hw, NGBE_TXCFG(txq->reg_idx));
1954 txq->tx_ring_phys_addr = TMZ_PADDR(tz);
1955 txq->tx_ring = (struct ngbe_tx_desc *)TMZ_VADDR(tz);
1957 /* Allocate software ring */
1958 txq->sw_ring = rte_zmalloc_socket("txq->sw_ring",
1959 sizeof(struct ngbe_tx_entry) * nb_desc,
1960 RTE_CACHE_LINE_SIZE, socket_id);
1961 if (txq->sw_ring == NULL) {
1962 ngbe_tx_queue_release(txq);
1966 "sw_ring=%p hw_ring=%p dma_addr=0x%" PRIx64,
1967 txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr);
1969 /* set up scalar Tx function as appropriate */
1970 ngbe_set_tx_function(dev, txq);
1972 txq->ops->reset(txq);
1974 dev->data->tx_queues[queue_idx] = txq;
1980 * ngbe_free_sc_cluster - free the not-yet-completed scattered cluster
1982 * The "next" pointer of the last segment of (not-yet-completed) RSC clusters
1983 * in the sw_sc_ring is not set to NULL but rather points to the next
1984 * mbuf of this RSC aggregation (that has not been completed yet and still
1985 * resides on the HW ring). So, instead of calling for rte_pktmbuf_free() we
1986 * will just free first "nb_segs" segments of the cluster explicitly by calling
1987 * an rte_pktmbuf_free_seg().
1989 * @m scattered cluster head
1992 ngbe_free_sc_cluster(struct rte_mbuf *m)
1994 uint16_t i, nb_segs = m->nb_segs;
1995 struct rte_mbuf *next_seg;
1997 for (i = 0; i < nb_segs; i++) {
1999 rte_pktmbuf_free_seg(m);
2005 ngbe_rx_queue_release_mbufs(struct ngbe_rx_queue *rxq)
2009 if (rxq->sw_ring != NULL) {
2010 for (i = 0; i < rxq->nb_rx_desc; i++) {
2011 if (rxq->sw_ring[i].mbuf != NULL) {
2012 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
2013 rxq->sw_ring[i].mbuf = NULL;
2016 for (i = 0; i < rxq->rx_nb_avail; ++i) {
2017 struct rte_mbuf *mb;
2019 mb = rxq->rx_stage[rxq->rx_next_avail + i];
2020 rte_pktmbuf_free_seg(mb);
2022 rxq->rx_nb_avail = 0;
2025 if (rxq->sw_sc_ring != NULL)
2026 for (i = 0; i < rxq->nb_rx_desc; i++)
2027 if (rxq->sw_sc_ring[i].fbuf != NULL) {
2028 ngbe_free_sc_cluster(rxq->sw_sc_ring[i].fbuf);
2029 rxq->sw_sc_ring[i].fbuf = NULL;
2034 ngbe_rx_queue_release(struct ngbe_rx_queue *rxq)
2037 ngbe_rx_queue_release_mbufs(rxq);
2038 rte_free(rxq->sw_ring);
2039 rte_free(rxq->sw_sc_ring);
2045 ngbe_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
2047 ngbe_rx_queue_release(dev->data->rx_queues[qid]);
2051 * Check if Rx Burst Bulk Alloc function can be used.
2053 * 0: the preconditions are satisfied and the bulk allocation function
2055 * -EINVAL: the preconditions are NOT satisfied and the default Rx burst
2056 * function must be used.
2059 check_rx_burst_bulk_alloc_preconditions(struct ngbe_rx_queue *rxq)
2064 * Make sure the following pre-conditions are satisfied:
2065 * rxq->rx_free_thresh >= RTE_PMD_NGBE_RX_MAX_BURST
2066 * rxq->rx_free_thresh < rxq->nb_rx_desc
2067 * (rxq->nb_rx_desc % rxq->rx_free_thresh) == 0
2068 * Scattered packets are not supported. This should be checked
2069 * outside of this function.
2071 if (rxq->rx_free_thresh < RTE_PMD_NGBE_RX_MAX_BURST) {
2073 "Rx Burst Bulk Alloc Preconditions: rxq->rx_free_thresh=%d, RTE_PMD_NGBE_RX_MAX_BURST=%d",
2074 rxq->rx_free_thresh, RTE_PMD_NGBE_RX_MAX_BURST);
2076 } else if (rxq->rx_free_thresh >= rxq->nb_rx_desc) {
2078 "Rx Burst Bulk Alloc Preconditions: rxq->rx_free_thresh=%d, rxq->nb_rx_desc=%d",
2079 rxq->rx_free_thresh, rxq->nb_rx_desc);
2081 } else if ((rxq->nb_rx_desc % rxq->rx_free_thresh) != 0) {
2083 "Rx Burst Bulk Alloc Preconditions: rxq->nb_rx_desc=%d, rxq->rx_free_thresh=%d",
2084 rxq->nb_rx_desc, rxq->rx_free_thresh);
2091 /* Reset dynamic ngbe_rx_queue fields back to defaults */
2093 ngbe_reset_rx_queue(struct ngbe_adapter *adapter, struct ngbe_rx_queue *rxq)
2095 static const struct ngbe_rx_desc zeroed_desc = {
2096 {{0}, {0} }, {{0}, {0} } };
2098 uint16_t len = rxq->nb_rx_desc;
2101 * By default, the Rx queue setup function allocates enough memory for
2102 * NGBE_RING_DESC_MAX. The Rx Burst bulk allocation function requires
2103 * extra memory at the end of the descriptor ring to be zero'd out.
2105 if (adapter->rx_bulk_alloc_allowed)
2106 /* zero out extra memory */
2107 len += RTE_PMD_NGBE_RX_MAX_BURST;
2110 * Zero out HW ring memory. Zero out extra memory at the end of
2111 * the H/W ring so look-ahead logic in Rx Burst bulk alloc function
2112 * reads extra memory as zeros.
2114 for (i = 0; i < len; i++)
2115 rxq->rx_ring[i] = zeroed_desc;
2118 * initialize extra software ring entries. Space for these extra
2119 * entries is always allocated
2121 memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf));
2122 for (i = rxq->nb_rx_desc; i < len; ++i)
2123 rxq->sw_ring[i].mbuf = &rxq->fake_mbuf;
2125 rxq->rx_nb_avail = 0;
2126 rxq->rx_next_avail = 0;
2127 rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1);
2129 rxq->nb_rx_hold = 0;
2130 rxq->pkt_first_seg = NULL;
2131 rxq->pkt_last_seg = NULL;
2135 ngbe_get_rx_queue_offloads(struct rte_eth_dev *dev __rte_unused)
2137 return RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
2141 ngbe_get_rx_port_offloads(struct rte_eth_dev *dev)
2144 struct ngbe_hw *hw = ngbe_dev_hw(dev);
2146 offloads = RTE_ETH_RX_OFFLOAD_IPV4_CKSUM |
2147 RTE_ETH_RX_OFFLOAD_UDP_CKSUM |
2148 RTE_ETH_RX_OFFLOAD_TCP_CKSUM |
2149 RTE_ETH_RX_OFFLOAD_KEEP_CRC |
2150 RTE_ETH_RX_OFFLOAD_VLAN_FILTER |
2151 RTE_ETH_RX_OFFLOAD_SCATTER;
2154 offloads |= (RTE_ETH_RX_OFFLOAD_QINQ_STRIP |
2155 RTE_ETH_RX_OFFLOAD_VLAN_EXTEND);
2161 ngbe_dev_rx_queue_setup(struct rte_eth_dev *dev,
2164 unsigned int socket_id,
2165 const struct rte_eth_rxconf *rx_conf,
2166 struct rte_mempool *mp)
2168 const struct rte_memzone *rz;
2169 struct ngbe_rx_queue *rxq;
2172 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
2175 PMD_INIT_FUNC_TRACE();
2176 hw = ngbe_dev_hw(dev);
2178 offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads;
2180 /* Free memory prior to re-allocation if needed... */
2181 if (dev->data->rx_queues[queue_idx] != NULL) {
2182 ngbe_rx_queue_release(dev->data->rx_queues[queue_idx]);
2183 dev->data->rx_queues[queue_idx] = NULL;
2186 /* First allocate the Rx queue data structure */
2187 rxq = rte_zmalloc_socket("ethdev RX queue",
2188 sizeof(struct ngbe_rx_queue),
2189 RTE_CACHE_LINE_SIZE, socket_id);
2193 rxq->nb_rx_desc = nb_desc;
2194 rxq->rx_free_thresh = rx_conf->rx_free_thresh;
2195 rxq->queue_id = queue_idx;
2196 rxq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ?
2197 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx);
2198 rxq->port_id = dev->data->port_id;
2199 if (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC)
2200 rxq->crc_len = RTE_ETHER_CRC_LEN;
2203 rxq->drop_en = rx_conf->rx_drop_en;
2204 rxq->rx_deferred_start = rx_conf->rx_deferred_start;
2205 rxq->offloads = offloads;
2208 * Allocate Rx ring hardware descriptors. A memzone large enough to
2209 * handle the maximum ring size is allocated in order to allow for
2210 * resizing in later calls to the queue setup function.
2212 rz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx,
2213 RX_RING_SZ, NGBE_ALIGN, socket_id);
2215 ngbe_rx_queue_release(rxq);
2220 * Zero init all the descriptors in the ring.
2222 memset(rz->addr, 0, RX_RING_SZ);
2224 rxq->rdt_reg_addr = NGBE_REG_ADDR(hw, NGBE_RXWP(rxq->reg_idx));
2225 rxq->rdh_reg_addr = NGBE_REG_ADDR(hw, NGBE_RXRP(rxq->reg_idx));
2227 rxq->rx_ring_phys_addr = TMZ_PADDR(rz);
2228 rxq->rx_ring = (struct ngbe_rx_desc *)TMZ_VADDR(rz);
2231 * Certain constraints must be met in order to use the bulk buffer
2232 * allocation Rx burst function. If any of Rx queues doesn't meet them
2233 * the feature should be disabled for the whole port.
2235 if (check_rx_burst_bulk_alloc_preconditions(rxq)) {
2237 "queue[%d] doesn't meet Rx Bulk Alloc preconditions - canceling the feature for the whole port[%d]",
2238 rxq->queue_id, rxq->port_id);
2239 adapter->rx_bulk_alloc_allowed = false;
2243 * Allocate software ring. Allow for space at the end of the
2244 * S/W ring to make sure look-ahead logic in bulk alloc Rx burst
2245 * function does not access an invalid memory region.
2248 if (adapter->rx_bulk_alloc_allowed)
2249 len += RTE_PMD_NGBE_RX_MAX_BURST;
2251 rxq->sw_ring = rte_zmalloc_socket("rxq->sw_ring",
2252 sizeof(struct ngbe_rx_entry) * len,
2253 RTE_CACHE_LINE_SIZE, socket_id);
2254 if (rxq->sw_ring == NULL) {
2255 ngbe_rx_queue_release(rxq);
2260 * Always allocate even if it's not going to be needed in order to
2261 * simplify the code.
2263 * This ring is used in Scattered Rx cases and Scattered Rx may
2264 * be requested in ngbe_dev_rx_init(), which is called later from
2268 rte_zmalloc_socket("rxq->sw_sc_ring",
2269 sizeof(struct ngbe_scattered_rx_entry) * len,
2270 RTE_CACHE_LINE_SIZE, socket_id);
2271 if (rxq->sw_sc_ring == NULL) {
2272 ngbe_rx_queue_release(rxq);
2277 "sw_ring=%p sw_sc_ring=%p hw_ring=%p dma_addr=0x%" PRIx64,
2278 rxq->sw_ring, rxq->sw_sc_ring, rxq->rx_ring,
2279 rxq->rx_ring_phys_addr);
2281 dev->data->rx_queues[queue_idx] = rxq;
2283 ngbe_reset_rx_queue(adapter, rxq);
2289 ngbe_dev_clear_queues(struct rte_eth_dev *dev)
2292 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
2294 PMD_INIT_FUNC_TRACE();
2296 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2297 struct ngbe_tx_queue *txq = dev->data->tx_queues[i];
2300 txq->ops->release_mbufs(txq);
2301 txq->ops->reset(txq);
2305 for (i = 0; i < dev->data->nb_rx_queues; i++) {
2306 struct ngbe_rx_queue *rxq = dev->data->rx_queues[i];
2309 ngbe_rx_queue_release_mbufs(rxq);
2310 ngbe_reset_rx_queue(adapter, rxq);
2316 ngbe_dev_free_queues(struct rte_eth_dev *dev)
2320 PMD_INIT_FUNC_TRACE();
2322 for (i = 0; i < dev->data->nb_rx_queues; i++) {
2323 ngbe_dev_rx_queue_release(dev, i);
2324 dev->data->rx_queues[i] = NULL;
2326 dev->data->nb_rx_queues = 0;
2328 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2329 ngbe_dev_tx_queue_release(dev, i);
2330 dev->data->tx_queues[i] = NULL;
2332 dev->data->nb_tx_queues = 0;
2336 * Receive Side Scaling (RSS)
2339 * The source and destination IP addresses of the IP header and the source
2340 * and destination ports of TCP/UDP headers, if any, of received packets are
2341 * hashed against a configurable random key to compute a 32-bit RSS hash result.
2342 * The seven (7) LSBs of the 32-bit hash result are used as an index into a
2343 * 128-entry redirection table (RETA). Each entry of the RETA provides a 3-bit
2344 * RSS output index which is used as the Rx queue index where to store the
2346 * The following output is supplied in the Rx write-back descriptor:
2347 * - 32-bit result of the Microsoft RSS hash function,
2348 * - 4-bit RSS type field.
2352 * Used as the default key.
2354 static uint8_t rss_intel_key[40] = {
2355 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
2356 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
2357 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
2358 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
2359 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
2363 ngbe_rss_disable(struct rte_eth_dev *dev)
2365 struct ngbe_hw *hw = ngbe_dev_hw(dev);
2367 wr32m(hw, NGBE_RACTL, NGBE_RACTL_RSSENA, 0);
2371 ngbe_dev_rss_hash_update(struct rte_eth_dev *dev,
2372 struct rte_eth_rss_conf *rss_conf)
2374 struct ngbe_hw *hw = ngbe_dev_hw(dev);
2382 PMD_DRV_LOG(ERR, "RSS hash update is not supported on this "
2387 hash_key = rss_conf->rss_key;
2389 /* Fill in RSS hash key */
2390 for (i = 0; i < 10; i++) {
2391 rss_key = LS32(hash_key[(i * 4) + 0], 0, 0xFF);
2392 rss_key |= LS32(hash_key[(i * 4) + 1], 8, 0xFF);
2393 rss_key |= LS32(hash_key[(i * 4) + 2], 16, 0xFF);
2394 rss_key |= LS32(hash_key[(i * 4) + 3], 24, 0xFF);
2395 wr32a(hw, NGBE_REG_RSSKEY, i, rss_key);
2399 /* Set configured hashing protocols */
2400 rss_hf = rss_conf->rss_hf & NGBE_RSS_OFFLOAD_ALL;
2402 mrqc = rd32(hw, NGBE_RACTL);
2403 mrqc &= ~NGBE_RACTL_RSSMASK;
2404 if (rss_hf & RTE_ETH_RSS_IPV4)
2405 mrqc |= NGBE_RACTL_RSSIPV4;
2406 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_TCP)
2407 mrqc |= NGBE_RACTL_RSSIPV4TCP;
2408 if (rss_hf & RTE_ETH_RSS_IPV6 ||
2409 rss_hf & RTE_ETH_RSS_IPV6_EX)
2410 mrqc |= NGBE_RACTL_RSSIPV6;
2411 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV6_TCP ||
2412 rss_hf & RTE_ETH_RSS_IPV6_TCP_EX)
2413 mrqc |= NGBE_RACTL_RSSIPV6TCP;
2414 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_UDP)
2415 mrqc |= NGBE_RACTL_RSSIPV4UDP;
2416 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV6_UDP ||
2417 rss_hf & RTE_ETH_RSS_IPV6_UDP_EX)
2418 mrqc |= NGBE_RACTL_RSSIPV6UDP;
2421 mrqc |= NGBE_RACTL_RSSENA;
2423 mrqc &= ~NGBE_RACTL_RSSENA;
2425 wr32(hw, NGBE_RACTL, mrqc);
2431 ngbe_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
2432 struct rte_eth_rss_conf *rss_conf)
2434 struct ngbe_hw *hw = ngbe_dev_hw(dev);
2441 hash_key = rss_conf->rss_key;
2443 /* Return RSS hash key */
2444 for (i = 0; i < 10; i++) {
2445 rss_key = rd32a(hw, NGBE_REG_RSSKEY, i);
2446 hash_key[(i * 4) + 0] = RS32(rss_key, 0, 0xFF);
2447 hash_key[(i * 4) + 1] = RS32(rss_key, 8, 0xFF);
2448 hash_key[(i * 4) + 2] = RS32(rss_key, 16, 0xFF);
2449 hash_key[(i * 4) + 3] = RS32(rss_key, 24, 0xFF);
2455 mrqc = rd32(hw, NGBE_RACTL);
2456 if (mrqc & NGBE_RACTL_RSSIPV4)
2457 rss_hf |= RTE_ETH_RSS_IPV4;
2458 if (mrqc & NGBE_RACTL_RSSIPV4TCP)
2459 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_TCP;
2460 if (mrqc & NGBE_RACTL_RSSIPV6)
2461 rss_hf |= RTE_ETH_RSS_IPV6 |
2462 RTE_ETH_RSS_IPV6_EX;
2463 if (mrqc & NGBE_RACTL_RSSIPV6TCP)
2464 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_TCP |
2465 RTE_ETH_RSS_IPV6_TCP_EX;
2466 if (mrqc & NGBE_RACTL_RSSIPV4UDP)
2467 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_UDP;
2468 if (mrqc & NGBE_RACTL_RSSIPV6UDP)
2469 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_UDP |
2470 RTE_ETH_RSS_IPV6_UDP_EX;
2471 if (!(mrqc & NGBE_RACTL_RSSENA))
2474 rss_hf &= NGBE_RSS_OFFLOAD_ALL;
2476 rss_conf->rss_hf = rss_hf;
2481 ngbe_rss_configure(struct rte_eth_dev *dev)
2483 struct rte_eth_rss_conf rss_conf;
2484 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
2485 struct ngbe_hw *hw = ngbe_dev_hw(dev);
2490 PMD_INIT_FUNC_TRACE();
2493 * Fill in redirection table
2494 * The byte-swap is needed because NIC registers are in
2495 * little-endian order.
2497 if (adapter->rss_reta_updated == 0) {
2499 for (i = 0, j = 0; i < RTE_ETH_RSS_RETA_SIZE_128; i++, j++) {
2500 if (j == dev->data->nb_rx_queues)
2502 reta = (reta >> 8) | LS32(j, 24, 0xFF);
2504 wr32a(hw, NGBE_REG_RSSTBL, i >> 2, reta);
2508 * Configure the RSS key and the RSS protocols used to compute
2509 * the RSS hash of input packets.
2511 rss_conf = dev->data->dev_conf.rx_adv_conf.rss_conf;
2512 if (rss_conf.rss_key == NULL)
2513 rss_conf.rss_key = rss_intel_key; /* Default hash key */
2514 ngbe_dev_rss_hash_update(dev, &rss_conf);
2517 void ngbe_configure_port(struct rte_eth_dev *dev)
2519 struct ngbe_hw *hw = ngbe_dev_hw(dev);
2521 uint16_t tpids[8] = {RTE_ETHER_TYPE_VLAN, RTE_ETHER_TYPE_QINQ,
2526 PMD_INIT_FUNC_TRACE();
2528 /* default outer vlan tpid */
2529 wr32(hw, NGBE_EXTAG,
2530 NGBE_EXTAG_ETAG(RTE_ETHER_TYPE_ETAG) |
2531 NGBE_EXTAG_VLAN(RTE_ETHER_TYPE_QINQ));
2533 /* default inner vlan tpid */
2534 wr32m(hw, NGBE_VLANCTL,
2535 NGBE_VLANCTL_TPID_MASK,
2536 NGBE_VLANCTL_TPID(RTE_ETHER_TYPE_VLAN));
2537 wr32m(hw, NGBE_DMATXCTRL,
2538 NGBE_DMATXCTRL_TPID_MASK,
2539 NGBE_DMATXCTRL_TPID(RTE_ETHER_TYPE_VLAN));
2541 /* default vlan tpid filters */
2542 for (i = 0; i < 8; i++) {
2543 wr32m(hw, NGBE_TAGTPID(i / 2),
2544 (i % 2 ? NGBE_TAGTPID_MSB_MASK
2545 : NGBE_TAGTPID_LSB_MASK),
2546 (i % 2 ? NGBE_TAGTPID_MSB(tpids[i])
2547 : NGBE_TAGTPID_LSB(tpids[i])));
2552 ngbe_alloc_rx_queue_mbufs(struct ngbe_rx_queue *rxq)
2554 struct ngbe_rx_entry *rxe = rxq->sw_ring;
2558 /* Initialize software ring entries */
2559 for (i = 0; i < rxq->nb_rx_desc; i++) {
2560 /* the ring can also be modified by hardware */
2561 volatile struct ngbe_rx_desc *rxd;
2562 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mb_pool);
2565 PMD_INIT_LOG(ERR, "Rx mbuf alloc failed queue_id=%u port_id=%u",
2566 (unsigned int)rxq->queue_id,
2567 (unsigned int)rxq->port_id);
2571 mbuf->data_off = RTE_PKTMBUF_HEADROOM;
2572 mbuf->port = rxq->port_id;
2575 rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf));
2576 rxd = &rxq->rx_ring[i];
2577 NGBE_RXD_HDRADDR(rxd, 0);
2578 NGBE_RXD_PKTADDR(rxd, dma_addr);
2586 ngbe_dev_mq_rx_configure(struct rte_eth_dev *dev)
2588 if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
2589 switch (dev->data->dev_conf.rxmode.mq_mode) {
2590 case RTE_ETH_MQ_RX_RSS:
2591 ngbe_rss_configure(dev);
2594 case RTE_ETH_MQ_RX_NONE:
2596 /* if mq_mode is none, disable rss mode.*/
2597 ngbe_rss_disable(dev);
2606 ngbe_set_rx_function(struct rte_eth_dev *dev)
2608 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
2610 if (dev->data->scattered_rx) {
2612 * Set the scattered callback: there are bulk and
2613 * single allocation versions.
2615 if (adapter->rx_bulk_alloc_allowed) {
2616 PMD_INIT_LOG(DEBUG, "Using a Scattered with bulk "
2617 "allocation callback (port=%d).",
2618 dev->data->port_id);
2619 dev->rx_pkt_burst = ngbe_recv_pkts_sc_bulk_alloc;
2621 PMD_INIT_LOG(DEBUG, "Using Regular (non-vector, "
2622 "single allocation) "
2623 "Scattered Rx callback "
2625 dev->data->port_id);
2627 dev->rx_pkt_burst = ngbe_recv_pkts_sc_single_alloc;
2630 * Below we set "simple" callbacks according to port/queues parameters.
2631 * If parameters allow we are going to choose between the following
2634 * - Single buffer allocation (the simplest one)
2636 } else if (adapter->rx_bulk_alloc_allowed) {
2637 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are "
2638 "satisfied. Rx Burst Bulk Alloc function "
2639 "will be used on port=%d.",
2640 dev->data->port_id);
2642 dev->rx_pkt_burst = ngbe_recv_pkts_bulk_alloc;
2644 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are not "
2645 "satisfied, or Scattered Rx is requested "
2647 dev->data->port_id);
2649 dev->rx_pkt_burst = ngbe_recv_pkts;
2653 static const struct {
2654 eth_rx_burst_t pkt_burst;
2656 } ngbe_rx_burst_infos[] = {
2657 { ngbe_recv_pkts_sc_single_alloc, "Scalar Scattered"},
2658 { ngbe_recv_pkts_sc_bulk_alloc, "Scalar Scattered Bulk Alloc"},
2659 { ngbe_recv_pkts_bulk_alloc, "Scalar Bulk Alloc"},
2660 { ngbe_recv_pkts, "Scalar"},
2664 ngbe_rx_burst_mode_get(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id,
2665 struct rte_eth_burst_mode *mode)
2667 eth_rx_burst_t pkt_burst = dev->rx_pkt_burst;
2671 for (i = 0; i < RTE_DIM(ngbe_rx_burst_infos); ++i) {
2672 if (pkt_burst == ngbe_rx_burst_infos[i].pkt_burst) {
2673 snprintf(mode->info, sizeof(mode->info), "%s",
2674 ngbe_rx_burst_infos[i].info);
2684 * Initializes Receive Unit.
2687 ngbe_dev_rx_init(struct rte_eth_dev *dev)
2690 struct ngbe_rx_queue *rxq;
2699 struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode;
2701 PMD_INIT_FUNC_TRACE();
2702 hw = ngbe_dev_hw(dev);
2705 * Make sure receives are disabled while setting
2706 * up the Rx context (registers, descriptor rings, etc.).
2708 wr32m(hw, NGBE_MACRXCFG, NGBE_MACRXCFG_ENA, 0);
2709 wr32m(hw, NGBE_PBRXCTL, NGBE_PBRXCTL_ENA, 0);
2711 /* Enable receipt of broadcasted frames */
2712 fctrl = rd32(hw, NGBE_PSRCTL);
2713 fctrl |= NGBE_PSRCTL_BCA;
2714 wr32(hw, NGBE_PSRCTL, fctrl);
2717 * Configure CRC stripping, if any.
2719 hlreg0 = rd32(hw, NGBE_SECRXCTL);
2720 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC)
2721 hlreg0 &= ~NGBE_SECRXCTL_CRCSTRIP;
2723 hlreg0 |= NGBE_SECRXCTL_CRCSTRIP;
2724 hlreg0 &= ~NGBE_SECRXCTL_XDSA;
2725 wr32(hw, NGBE_SECRXCTL, hlreg0);
2728 * Configure jumbo frame support, if any.
2730 wr32m(hw, NGBE_FRMSZ, NGBE_FRMSZ_MAX_MASK,
2731 NGBE_FRMSZ_MAX(dev->data->mtu + NGBE_ETH_OVERHEAD));
2734 * If loopback mode is configured, set LPBK bit.
2736 hlreg0 = rd32(hw, NGBE_PSRCTL);
2737 if (hw->is_pf && dev->data->dev_conf.lpbk_mode)
2738 hlreg0 |= NGBE_PSRCTL_LBENA;
2740 hlreg0 &= ~NGBE_PSRCTL_LBENA;
2742 wr32(hw, NGBE_PSRCTL, hlreg0);
2745 * Assume no header split and no VLAN strip support
2746 * on any Rx queue first .
2748 rx_conf->offloads &= ~RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
2750 /* Setup Rx queues */
2751 for (i = 0; i < dev->data->nb_rx_queues; i++) {
2752 rxq = dev->data->rx_queues[i];
2755 * Reset crc_len in case it was changed after queue setup by a
2756 * call to configure.
2758 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC)
2759 rxq->crc_len = RTE_ETHER_CRC_LEN;
2763 /* Setup the Base and Length of the Rx Descriptor Rings */
2764 bus_addr = rxq->rx_ring_phys_addr;
2765 wr32(hw, NGBE_RXBAL(rxq->reg_idx),
2766 (uint32_t)(bus_addr & BIT_MASK32));
2767 wr32(hw, NGBE_RXBAH(rxq->reg_idx),
2768 (uint32_t)(bus_addr >> 32));
2769 wr32(hw, NGBE_RXRP(rxq->reg_idx), 0);
2770 wr32(hw, NGBE_RXWP(rxq->reg_idx), 0);
2772 srrctl = NGBE_RXCFG_RNGLEN(rxq->nb_rx_desc);
2774 /* Set if packets are dropped when no descriptors available */
2776 srrctl |= NGBE_RXCFG_DROP;
2779 * Configure the Rx buffer size in the PKTLEN field of
2780 * the RXCFG register of the queue.
2781 * The value is in 1 KB resolution. Valid values can be from
2784 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) -
2785 RTE_PKTMBUF_HEADROOM);
2786 buf_size = ROUND_DOWN(buf_size, 0x1 << 10);
2787 srrctl |= NGBE_RXCFG_PKTLEN(buf_size);
2789 wr32(hw, NGBE_RXCFG(rxq->reg_idx), srrctl);
2791 /* It adds dual VLAN length for supporting dual VLAN */
2792 if (dev->data->mtu + NGBE_ETH_OVERHEAD +
2793 2 * NGBE_VLAN_TAG_SIZE > buf_size)
2794 dev->data->scattered_rx = 1;
2795 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP)
2796 rx_conf->offloads |= RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
2799 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_SCATTER)
2800 dev->data->scattered_rx = 1;
2803 * Device configured with multiple RX queues.
2805 ngbe_dev_mq_rx_configure(dev);
2808 * Setup the Checksum Register.
2809 * Disable Full-Packet Checksum which is mutually exclusive with RSS.
2810 * Enable IP/L4 checksum computation by hardware if requested to do so.
2812 rxcsum = rd32(hw, NGBE_PSRCTL);
2813 rxcsum |= NGBE_PSRCTL_PCSD;
2814 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_CHECKSUM)
2815 rxcsum |= NGBE_PSRCTL_L4CSUM;
2817 rxcsum &= ~NGBE_PSRCTL_L4CSUM;
2819 wr32(hw, NGBE_PSRCTL, rxcsum);
2822 rdrxctl = rd32(hw, NGBE_SECRXCTL);
2823 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC)
2824 rdrxctl &= ~NGBE_SECRXCTL_CRCSTRIP;
2826 rdrxctl |= NGBE_SECRXCTL_CRCSTRIP;
2827 wr32(hw, NGBE_SECRXCTL, rdrxctl);
2830 ngbe_set_rx_function(dev);
2836 * Initializes Transmit Unit.
2839 ngbe_dev_tx_init(struct rte_eth_dev *dev)
2842 struct ngbe_tx_queue *txq;
2846 PMD_INIT_FUNC_TRACE();
2847 hw = ngbe_dev_hw(dev);
2849 wr32m(hw, NGBE_SECTXCTL, NGBE_SECTXCTL_ODSA, NGBE_SECTXCTL_ODSA);
2850 wr32m(hw, NGBE_SECTXCTL, NGBE_SECTXCTL_XDSA, 0);
2852 /* Setup the Base and Length of the Tx Descriptor Rings */
2853 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2854 txq = dev->data->tx_queues[i];
2856 bus_addr = txq->tx_ring_phys_addr;
2857 wr32(hw, NGBE_TXBAL(txq->reg_idx),
2858 (uint32_t)(bus_addr & BIT_MASK32));
2859 wr32(hw, NGBE_TXBAH(txq->reg_idx),
2860 (uint32_t)(bus_addr >> 32));
2861 wr32m(hw, NGBE_TXCFG(txq->reg_idx), NGBE_TXCFG_BUFLEN_MASK,
2862 NGBE_TXCFG_BUFLEN(txq->nb_tx_desc));
2863 /* Setup the HW Tx Head and TX Tail descriptor pointers */
2864 wr32(hw, NGBE_TXRP(txq->reg_idx), 0);
2865 wr32(hw, NGBE_TXWP(txq->reg_idx), 0);
2870 * Set up link loopback mode Tx->Rx.
2873 ngbe_setup_loopback_link(struct ngbe_hw *hw)
2875 PMD_INIT_FUNC_TRACE();
2877 wr32m(hw, NGBE_MACRXCFG, NGBE_MACRXCFG_LB, NGBE_MACRXCFG_LB);
2883 * Start Transmit and Receive Units.
2886 ngbe_dev_rxtx_start(struct rte_eth_dev *dev)
2889 struct ngbe_tx_queue *txq;
2890 struct ngbe_rx_queue *rxq;
2896 PMD_INIT_FUNC_TRACE();
2897 hw = ngbe_dev_hw(dev);
2899 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2900 txq = dev->data->tx_queues[i];
2901 /* Setup Transmit Threshold Registers */
2902 wr32m(hw, NGBE_TXCFG(txq->reg_idx),
2903 NGBE_TXCFG_HTHRESH_MASK |
2904 NGBE_TXCFG_WTHRESH_MASK,
2905 NGBE_TXCFG_HTHRESH(txq->hthresh) |
2906 NGBE_TXCFG_WTHRESH(txq->wthresh));
2909 dmatxctl = rd32(hw, NGBE_DMATXCTRL);
2910 dmatxctl |= NGBE_DMATXCTRL_ENA;
2911 wr32(hw, NGBE_DMATXCTRL, dmatxctl);
2913 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2914 txq = dev->data->tx_queues[i];
2915 if (txq->tx_deferred_start == 0) {
2916 ret = ngbe_dev_tx_queue_start(dev, i);
2922 for (i = 0; i < dev->data->nb_rx_queues; i++) {
2923 rxq = dev->data->rx_queues[i];
2924 if (rxq->rx_deferred_start == 0) {
2925 ret = ngbe_dev_rx_queue_start(dev, i);
2931 /* Enable Receive engine */
2932 rxctrl = rd32(hw, NGBE_PBRXCTL);
2933 rxctrl |= NGBE_PBRXCTL_ENA;
2934 hw->mac.enable_rx_dma(hw, rxctrl);
2936 /* If loopback mode is enabled, set up the link accordingly */
2937 if (hw->is_pf && dev->data->dev_conf.lpbk_mode)
2938 ngbe_setup_loopback_link(hw);
2944 ngbe_dev_save_rx_queue(struct ngbe_hw *hw, uint16_t rx_queue_id)
2946 u32 *reg = &hw->q_rx_regs[rx_queue_id * 8];
2947 *(reg++) = rd32(hw, NGBE_RXBAL(rx_queue_id));
2948 *(reg++) = rd32(hw, NGBE_RXBAH(rx_queue_id));
2949 *(reg++) = rd32(hw, NGBE_RXCFG(rx_queue_id));
2953 ngbe_dev_store_rx_queue(struct ngbe_hw *hw, uint16_t rx_queue_id)
2955 u32 *reg = &hw->q_rx_regs[rx_queue_id * 8];
2956 wr32(hw, NGBE_RXBAL(rx_queue_id), *(reg++));
2957 wr32(hw, NGBE_RXBAH(rx_queue_id), *(reg++));
2958 wr32(hw, NGBE_RXCFG(rx_queue_id), *(reg++) & ~NGBE_RXCFG_ENA);
2962 ngbe_dev_save_tx_queue(struct ngbe_hw *hw, uint16_t tx_queue_id)
2964 u32 *reg = &hw->q_tx_regs[tx_queue_id * 8];
2965 *(reg++) = rd32(hw, NGBE_TXBAL(tx_queue_id));
2966 *(reg++) = rd32(hw, NGBE_TXBAH(tx_queue_id));
2967 *(reg++) = rd32(hw, NGBE_TXCFG(tx_queue_id));
2971 ngbe_dev_store_tx_queue(struct ngbe_hw *hw, uint16_t tx_queue_id)
2973 u32 *reg = &hw->q_tx_regs[tx_queue_id * 8];
2974 wr32(hw, NGBE_TXBAL(tx_queue_id), *(reg++));
2975 wr32(hw, NGBE_TXBAH(tx_queue_id), *(reg++));
2976 wr32(hw, NGBE_TXCFG(tx_queue_id), *(reg++) & ~NGBE_TXCFG_ENA);
2980 * Start Receive Units for specified queue.
2983 ngbe_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
2985 struct ngbe_hw *hw = ngbe_dev_hw(dev);
2986 struct ngbe_rx_queue *rxq;
2990 PMD_INIT_FUNC_TRACE();
2992 rxq = dev->data->rx_queues[rx_queue_id];
2994 /* Allocate buffers for descriptor rings */
2995 if (ngbe_alloc_rx_queue_mbufs(rxq) != 0) {
2996 PMD_INIT_LOG(ERR, "Could not alloc mbuf for queue:%d",
3000 rxdctl = rd32(hw, NGBE_RXCFG(rxq->reg_idx));
3001 rxdctl |= NGBE_RXCFG_ENA;
3002 wr32(hw, NGBE_RXCFG(rxq->reg_idx), rxdctl);
3004 /* Wait until Rx Enable ready */
3005 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS;
3008 rxdctl = rd32(hw, NGBE_RXCFG(rxq->reg_idx));
3009 } while (--poll_ms && !(rxdctl & NGBE_RXCFG_ENA));
3011 PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", rx_queue_id);
3013 wr32(hw, NGBE_RXRP(rxq->reg_idx), 0);
3014 wr32(hw, NGBE_RXWP(rxq->reg_idx), rxq->nb_rx_desc - 1);
3015 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
3021 * Stop Receive Units for specified queue.
3024 ngbe_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
3026 struct ngbe_hw *hw = ngbe_dev_hw(dev);
3027 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
3028 struct ngbe_rx_queue *rxq;
3032 PMD_INIT_FUNC_TRACE();
3034 rxq = dev->data->rx_queues[rx_queue_id];
3036 ngbe_dev_save_rx_queue(hw, rxq->reg_idx);
3037 wr32m(hw, NGBE_RXCFG(rxq->reg_idx), NGBE_RXCFG_ENA, 0);
3039 /* Wait until Rx Enable bit clear */
3040 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS;
3043 rxdctl = rd32(hw, NGBE_RXCFG(rxq->reg_idx));
3044 } while (--poll_ms && (rxdctl & NGBE_RXCFG_ENA));
3046 PMD_INIT_LOG(ERR, "Could not disable Rx Queue %d", rx_queue_id);
3048 rte_delay_us(RTE_NGBE_WAIT_100_US);
3049 ngbe_dev_store_rx_queue(hw, rxq->reg_idx);
3051 ngbe_rx_queue_release_mbufs(rxq);
3052 ngbe_reset_rx_queue(adapter, rxq);
3053 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
3059 * Start Transmit Units for specified queue.
3062 ngbe_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
3064 struct ngbe_hw *hw = ngbe_dev_hw(dev);
3065 struct ngbe_tx_queue *txq;
3069 PMD_INIT_FUNC_TRACE();
3071 txq = dev->data->tx_queues[tx_queue_id];
3072 wr32m(hw, NGBE_TXCFG(txq->reg_idx), NGBE_TXCFG_ENA, NGBE_TXCFG_ENA);
3074 /* Wait until Tx Enable ready */
3075 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS;
3078 txdctl = rd32(hw, NGBE_TXCFG(txq->reg_idx));
3079 } while (--poll_ms && !(txdctl & NGBE_TXCFG_ENA));
3081 PMD_INIT_LOG(ERR, "Could not enable Tx Queue %d",
3085 wr32(hw, NGBE_TXWP(txq->reg_idx), txq->tx_tail);
3086 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
3092 * Stop Transmit Units for specified queue.
3095 ngbe_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
3097 struct ngbe_hw *hw = ngbe_dev_hw(dev);
3098 struct ngbe_tx_queue *txq;
3100 uint32_t txtdh, txtdt;
3103 PMD_INIT_FUNC_TRACE();
3105 txq = dev->data->tx_queues[tx_queue_id];
3107 /* Wait until Tx queue is empty */
3108 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS;
3110 rte_delay_us(RTE_NGBE_WAIT_100_US);
3111 txtdh = rd32(hw, NGBE_TXRP(txq->reg_idx));
3112 txtdt = rd32(hw, NGBE_TXWP(txq->reg_idx));
3113 } while (--poll_ms && (txtdh != txtdt));
3115 PMD_INIT_LOG(ERR, "Tx Queue %d is not empty when stopping.",
3118 ngbe_dev_save_tx_queue(hw, txq->reg_idx);
3119 wr32m(hw, NGBE_TXCFG(txq->reg_idx), NGBE_TXCFG_ENA, 0);
3121 /* Wait until Tx Enable bit clear */
3122 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS;
3125 txdctl = rd32(hw, NGBE_TXCFG(txq->reg_idx));
3126 } while (--poll_ms && (txdctl & NGBE_TXCFG_ENA));
3128 PMD_INIT_LOG(ERR, "Could not disable Tx Queue %d",
3131 rte_delay_us(RTE_NGBE_WAIT_100_US);
3132 ngbe_dev_store_tx_queue(hw, txq->reg_idx);
3134 if (txq->ops != NULL) {
3135 txq->ops->release_mbufs(txq);
3136 txq->ops->reset(txq);
3138 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
3144 ngbe_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
3145 struct rte_eth_rxq_info *qinfo)
3147 struct ngbe_rx_queue *rxq;
3149 rxq = dev->data->rx_queues[queue_id];
3151 qinfo->mp = rxq->mb_pool;
3152 qinfo->scattered_rx = dev->data->scattered_rx;
3153 qinfo->nb_desc = rxq->nb_rx_desc;
3155 qinfo->conf.rx_free_thresh = rxq->rx_free_thresh;
3156 qinfo->conf.rx_drop_en = rxq->drop_en;
3157 qinfo->conf.rx_deferred_start = rxq->rx_deferred_start;
3158 qinfo->conf.offloads = rxq->offloads;
3162 ngbe_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
3163 struct rte_eth_txq_info *qinfo)
3165 struct ngbe_tx_queue *txq;
3167 txq = dev->data->tx_queues[queue_id];
3169 qinfo->nb_desc = txq->nb_tx_desc;
3171 qinfo->conf.tx_thresh.pthresh = txq->pthresh;
3172 qinfo->conf.tx_thresh.hthresh = txq->hthresh;
3173 qinfo->conf.tx_thresh.wthresh = txq->wthresh;
3175 qinfo->conf.tx_free_thresh = txq->tx_free_thresh;
3176 qinfo->conf.offloads = txq->offloads;
3177 qinfo->conf.tx_deferred_start = txq->tx_deferred_start;