1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2015-2020
16 #include <rte_byteorder.h>
17 #include <rte_common.h>
18 #include <rte_cycles.h>
20 #include <rte_debug.h>
21 #include <rte_ethdev.h>
22 #include <rte_ethdev_driver.h>
23 #include <rte_memzone.h>
24 #include <rte_atomic.h>
25 #include <rte_mempool.h>
26 #include <rte_malloc.h>
28 #include <rte_ether.h>
29 #include <rte_prefetch.h>
33 #include <rte_string_fns.h>
34 #include <rte_errno.h>
38 #include "txgbe_logs.h"
39 #include "base/txgbe.h"
40 #include "txgbe_ethdev.h"
41 #include "txgbe_rxtx.h"
43 #ifdef RTE_LIBRTE_IEEE1588
44 #define TXGBE_TX_IEEE1588_TMST PKT_TX_IEEE1588_TMST
46 #define TXGBE_TX_IEEE1588_TMST 0
49 /* Bit Mask to indicate what bits required for building TX context */
50 static const u64 TXGBE_TX_OFFLOAD_MASK = (PKT_TX_IP_CKSUM |
59 PKT_TX_OUTER_IP_CKSUM |
60 TXGBE_TX_IEEE1588_TMST);
62 #define TXGBE_TX_OFFLOAD_NOTSUP_MASK \
63 (PKT_TX_OFFLOAD_MASK ^ TXGBE_TX_OFFLOAD_MASK)
66 * Prefetch a cache line into all cache levels.
68 #define rte_txgbe_prefetch(p) rte_prefetch0(p)
71 txgbe_is_vf(struct rte_eth_dev *dev)
73 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
75 switch (hw->mac.type) {
76 case txgbe_mac_raptor_vf:
83 /*********************************************************************
87 **********************************************************************/
90 * Check for descriptors with their DD bit set and free mbufs.
91 * Return the total number of buffers freed.
93 static __rte_always_inline int
94 txgbe_tx_free_bufs(struct txgbe_tx_queue *txq)
96 struct txgbe_tx_entry *txep;
99 struct rte_mbuf *m, *free[RTE_TXGBE_TX_MAX_FREE_BUF_SZ];
101 /* check DD bit on threshold descriptor */
102 status = txq->tx_ring[txq->tx_next_dd].dw3;
103 if (!(status & rte_cpu_to_le_32(TXGBE_TXD_DD))) {
104 if (txq->nb_tx_free >> 1 < txq->tx_free_thresh)
105 txgbe_set32_masked(txq->tdc_reg_addr,
106 TXGBE_TXCFG_FLUSH, TXGBE_TXCFG_FLUSH);
111 * first buffer to free from S/W ring is at index
112 * tx_next_dd - (tx_free_thresh-1)
114 txep = &txq->sw_ring[txq->tx_next_dd - (txq->tx_free_thresh - 1)];
115 for (i = 0; i < txq->tx_free_thresh; ++i, ++txep) {
116 /* free buffers one at a time */
117 m = rte_pktmbuf_prefree_seg(txep->mbuf);
120 if (unlikely(m == NULL))
123 if (nb_free >= RTE_TXGBE_TX_MAX_FREE_BUF_SZ ||
124 (nb_free > 0 && m->pool != free[0]->pool)) {
125 rte_mempool_put_bulk(free[0]->pool,
126 (void **)free, nb_free);
134 rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free);
136 /* buffers were freed, update counters */
137 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_free_thresh);
138 txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_free_thresh);
139 if (txq->tx_next_dd >= txq->nb_tx_desc)
140 txq->tx_next_dd = (uint16_t)(txq->tx_free_thresh - 1);
142 return txq->tx_free_thresh;
145 /* Populate 4 descriptors with data from 4 mbufs */
147 tx4(volatile struct txgbe_tx_desc *txdp, struct rte_mbuf **pkts)
149 uint64_t buf_dma_addr;
153 for (i = 0; i < 4; ++i, ++txdp, ++pkts) {
154 buf_dma_addr = rte_mbuf_data_iova(*pkts);
155 pkt_len = (*pkts)->data_len;
157 /* write data to descriptor */
158 txdp->qw0 = rte_cpu_to_le_64(buf_dma_addr);
159 txdp->dw2 = cpu_to_le32(TXGBE_TXD_FLAGS |
160 TXGBE_TXD_DATLEN(pkt_len));
161 txdp->dw3 = cpu_to_le32(TXGBE_TXD_PAYLEN(pkt_len));
163 rte_prefetch0(&(*pkts)->pool);
167 /* Populate 1 descriptor with data from 1 mbuf */
169 tx1(volatile struct txgbe_tx_desc *txdp, struct rte_mbuf **pkts)
171 uint64_t buf_dma_addr;
174 buf_dma_addr = rte_mbuf_data_iova(*pkts);
175 pkt_len = (*pkts)->data_len;
177 /* write data to descriptor */
178 txdp->qw0 = cpu_to_le64(buf_dma_addr);
179 txdp->dw2 = cpu_to_le32(TXGBE_TXD_FLAGS |
180 TXGBE_TXD_DATLEN(pkt_len));
181 txdp->dw3 = cpu_to_le32(TXGBE_TXD_PAYLEN(pkt_len));
183 rte_prefetch0(&(*pkts)->pool);
187 * Fill H/W descriptor ring with mbuf data.
188 * Copy mbuf pointers to the S/W ring.
191 txgbe_tx_fill_hw_ring(struct txgbe_tx_queue *txq, struct rte_mbuf **pkts,
194 volatile struct txgbe_tx_desc *txdp = &txq->tx_ring[txq->tx_tail];
195 struct txgbe_tx_entry *txep = &txq->sw_ring[txq->tx_tail];
196 const int N_PER_LOOP = 4;
197 const int N_PER_LOOP_MASK = N_PER_LOOP - 1;
198 int mainpart, leftover;
202 * Process most of the packets in chunks of N pkts. Any
203 * leftover packets will get processed one at a time.
205 mainpart = (nb_pkts & ((uint32_t)~N_PER_LOOP_MASK));
206 leftover = (nb_pkts & ((uint32_t)N_PER_LOOP_MASK));
207 for (i = 0; i < mainpart; i += N_PER_LOOP) {
208 /* Copy N mbuf pointers to the S/W ring */
209 for (j = 0; j < N_PER_LOOP; ++j)
210 (txep + i + j)->mbuf = *(pkts + i + j);
211 tx4(txdp + i, pkts + i);
214 if (unlikely(leftover > 0)) {
215 for (i = 0; i < leftover; ++i) {
216 (txep + mainpart + i)->mbuf = *(pkts + mainpart + i);
217 tx1(txdp + mainpart + i, pkts + mainpart + i);
222 static inline uint16_t
223 tx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
226 struct txgbe_tx_queue *txq = (struct txgbe_tx_queue *)tx_queue;
230 * Begin scanning the H/W ring for done descriptors when the
231 * number of available descriptors drops below tx_free_thresh. For
232 * each done descriptor, free the associated buffer.
234 if (txq->nb_tx_free < txq->tx_free_thresh)
235 txgbe_tx_free_bufs(txq);
237 /* Only use descriptors that are available */
238 nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts);
239 if (unlikely(nb_pkts == 0))
242 /* Use exactly nb_pkts descriptors */
243 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts);
246 * At this point, we know there are enough descriptors in the
247 * ring to transmit all the packets. This assumes that each
248 * mbuf contains a single segment, and that no new offloads
249 * are expected, which would require a new context descriptor.
253 * See if we're going to wrap-around. If so, handle the top
254 * of the descriptor ring first, then do the bottom. If not,
255 * the processing looks just like the "bottom" part anyway...
257 if ((txq->tx_tail + nb_pkts) > txq->nb_tx_desc) {
258 n = (uint16_t)(txq->nb_tx_desc - txq->tx_tail);
259 txgbe_tx_fill_hw_ring(txq, tx_pkts, n);
263 /* Fill H/W descriptor ring with mbuf data */
264 txgbe_tx_fill_hw_ring(txq, tx_pkts + n, (uint16_t)(nb_pkts - n));
265 txq->tx_tail = (uint16_t)(txq->tx_tail + (nb_pkts - n));
268 * Check for wrap-around. This would only happen if we used
269 * up to the last descriptor in the ring, no more, no less.
271 if (txq->tx_tail >= txq->nb_tx_desc)
274 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u",
275 (uint16_t)txq->port_id, (uint16_t)txq->queue_id,
276 (uint16_t)txq->tx_tail, (uint16_t)nb_pkts);
278 /* update tail pointer */
280 txgbe_set32_relaxed(txq->tdt_reg_addr, txq->tx_tail);
286 txgbe_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts,
291 /* Try to transmit at least chunks of TX_MAX_BURST pkts */
292 if (likely(nb_pkts <= RTE_PMD_TXGBE_TX_MAX_BURST))
293 return tx_xmit_pkts(tx_queue, tx_pkts, nb_pkts);
295 /* transmit more than the max burst, in chunks of TX_MAX_BURST */
300 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_TXGBE_TX_MAX_BURST);
301 ret = tx_xmit_pkts(tx_queue, &tx_pkts[nb_tx], n);
302 nb_tx = (uint16_t)(nb_tx + ret);
303 nb_pkts = (uint16_t)(nb_pkts - ret);
312 txgbe_set_xmit_ctx(struct txgbe_tx_queue *txq,
313 volatile struct txgbe_tx_ctx_desc *ctx_txd,
314 uint64_t ol_flags, union txgbe_tx_offload tx_offload)
316 union txgbe_tx_offload tx_offload_mask;
317 uint32_t type_tucmd_mlhl;
318 uint32_t mss_l4len_idx;
320 uint32_t vlan_macip_lens;
321 uint32_t tunnel_seed;
323 ctx_idx = txq->ctx_curr;
324 tx_offload_mask.data[0] = 0;
325 tx_offload_mask.data[1] = 0;
327 /* Specify which HW CTX to upload. */
328 mss_l4len_idx = TXGBE_TXD_IDX(ctx_idx);
329 type_tucmd_mlhl = TXGBE_TXD_CTXT;
331 tx_offload_mask.ptid |= ~0;
332 type_tucmd_mlhl |= TXGBE_TXD_PTID(tx_offload.ptid);
334 /* check if TCP segmentation required for this packet */
335 if (ol_flags & PKT_TX_TCP_SEG) {
336 tx_offload_mask.l2_len |= ~0;
337 tx_offload_mask.l3_len |= ~0;
338 tx_offload_mask.l4_len |= ~0;
339 tx_offload_mask.tso_segsz |= ~0;
340 mss_l4len_idx |= TXGBE_TXD_MSS(tx_offload.tso_segsz);
341 mss_l4len_idx |= TXGBE_TXD_L4LEN(tx_offload.l4_len);
342 } else { /* no TSO, check if hardware checksum is needed */
343 if (ol_flags & PKT_TX_IP_CKSUM) {
344 tx_offload_mask.l2_len |= ~0;
345 tx_offload_mask.l3_len |= ~0;
348 switch (ol_flags & PKT_TX_L4_MASK) {
349 case PKT_TX_UDP_CKSUM:
351 TXGBE_TXD_L4LEN(sizeof(struct rte_udp_hdr));
352 tx_offload_mask.l2_len |= ~0;
353 tx_offload_mask.l3_len |= ~0;
355 case PKT_TX_TCP_CKSUM:
357 TXGBE_TXD_L4LEN(sizeof(struct rte_tcp_hdr));
358 tx_offload_mask.l2_len |= ~0;
359 tx_offload_mask.l3_len |= ~0;
361 case PKT_TX_SCTP_CKSUM:
363 TXGBE_TXD_L4LEN(sizeof(struct rte_sctp_hdr));
364 tx_offload_mask.l2_len |= ~0;
365 tx_offload_mask.l3_len |= ~0;
372 vlan_macip_lens = TXGBE_TXD_IPLEN(tx_offload.l3_len >> 1);
374 if (ol_flags & PKT_TX_TUNNEL_MASK) {
375 tx_offload_mask.outer_tun_len |= ~0;
376 tx_offload_mask.outer_l2_len |= ~0;
377 tx_offload_mask.outer_l3_len |= ~0;
378 tx_offload_mask.l2_len |= ~0;
379 tunnel_seed = TXGBE_TXD_ETUNLEN(tx_offload.outer_tun_len >> 1);
380 tunnel_seed |= TXGBE_TXD_EIPLEN(tx_offload.outer_l3_len >> 2);
382 switch (ol_flags & PKT_TX_TUNNEL_MASK) {
383 case PKT_TX_TUNNEL_IPIP:
384 /* for non UDP / GRE tunneling, set to 0b */
386 case PKT_TX_TUNNEL_VXLAN:
387 case PKT_TX_TUNNEL_GENEVE:
388 tunnel_seed |= TXGBE_TXD_ETYPE_UDP;
390 case PKT_TX_TUNNEL_GRE:
391 tunnel_seed |= TXGBE_TXD_ETYPE_GRE;
394 PMD_TX_LOG(ERR, "Tunnel type not supported");
397 vlan_macip_lens |= TXGBE_TXD_MACLEN(tx_offload.outer_l2_len);
400 vlan_macip_lens |= TXGBE_TXD_MACLEN(tx_offload.l2_len);
403 if (ol_flags & PKT_TX_VLAN_PKT) {
404 tx_offload_mask.vlan_tci |= ~0;
405 vlan_macip_lens |= TXGBE_TXD_VLAN(tx_offload.vlan_tci);
408 txq->ctx_cache[ctx_idx].flags = ol_flags;
409 txq->ctx_cache[ctx_idx].tx_offload.data[0] =
410 tx_offload_mask.data[0] & tx_offload.data[0];
411 txq->ctx_cache[ctx_idx].tx_offload.data[1] =
412 tx_offload_mask.data[1] & tx_offload.data[1];
413 txq->ctx_cache[ctx_idx].tx_offload_mask = tx_offload_mask;
415 ctx_txd->dw0 = rte_cpu_to_le_32(vlan_macip_lens);
416 ctx_txd->dw1 = rte_cpu_to_le_32(tunnel_seed);
417 ctx_txd->dw2 = rte_cpu_to_le_32(type_tucmd_mlhl);
418 ctx_txd->dw3 = rte_cpu_to_le_32(mss_l4len_idx);
422 * Check which hardware context can be used. Use the existing match
423 * or create a new context descriptor.
425 static inline uint32_t
426 what_ctx_update(struct txgbe_tx_queue *txq, uint64_t flags,
427 union txgbe_tx_offload tx_offload)
429 /* If match with the current used context */
430 if (likely(txq->ctx_cache[txq->ctx_curr].flags == flags &&
431 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] ==
432 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0]
433 & tx_offload.data[0])) &&
434 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] ==
435 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1]
436 & tx_offload.data[1]))))
437 return txq->ctx_curr;
439 /* What if match with the next context */
441 if (likely(txq->ctx_cache[txq->ctx_curr].flags == flags &&
442 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] ==
443 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0]
444 & tx_offload.data[0])) &&
445 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] ==
446 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1]
447 & tx_offload.data[1]))))
448 return txq->ctx_curr;
450 /* Mismatch, use the previous context */
451 return TXGBE_CTX_NUM;
454 static inline uint32_t
455 tx_desc_cksum_flags_to_olinfo(uint64_t ol_flags)
459 if ((ol_flags & PKT_TX_L4_MASK) != PKT_TX_L4_NO_CKSUM) {
461 tmp |= TXGBE_TXD_L4CS;
463 if (ol_flags & PKT_TX_IP_CKSUM) {
465 tmp |= TXGBE_TXD_IPCS;
467 if (ol_flags & PKT_TX_OUTER_IP_CKSUM) {
469 tmp |= TXGBE_TXD_EIPCS;
471 if (ol_flags & PKT_TX_TCP_SEG) {
473 /* implies IPv4 cksum */
474 if (ol_flags & PKT_TX_IPV4)
475 tmp |= TXGBE_TXD_IPCS;
476 tmp |= TXGBE_TXD_L4CS;
478 if (ol_flags & PKT_TX_VLAN_PKT)
484 static inline uint32_t
485 tx_desc_ol_flags_to_cmdtype(uint64_t ol_flags)
487 uint32_t cmdtype = 0;
489 if (ol_flags & PKT_TX_VLAN_PKT)
490 cmdtype |= TXGBE_TXD_VLE;
491 if (ol_flags & PKT_TX_TCP_SEG)
492 cmdtype |= TXGBE_TXD_TSE;
493 if (ol_flags & PKT_TX_MACSEC)
494 cmdtype |= TXGBE_TXD_LINKSEC;
498 static inline uint8_t
499 tx_desc_ol_flags_to_ptid(uint64_t oflags, uint32_t ptype)
504 return txgbe_encode_ptype(ptype);
506 /* Only support flags in TXGBE_TX_OFFLOAD_MASK */
507 tun = !!(oflags & PKT_TX_TUNNEL_MASK);
510 ptype = RTE_PTYPE_L2_ETHER;
511 if (oflags & PKT_TX_VLAN)
512 ptype |= RTE_PTYPE_L2_ETHER_VLAN;
515 if (oflags & (PKT_TX_OUTER_IPV4 | PKT_TX_OUTER_IP_CKSUM))
516 ptype |= RTE_PTYPE_L3_IPV4;
517 else if (oflags & (PKT_TX_OUTER_IPV6))
518 ptype |= RTE_PTYPE_L3_IPV6;
520 if (oflags & (PKT_TX_IPV4 | PKT_TX_IP_CKSUM))
521 ptype |= (tun ? RTE_PTYPE_INNER_L3_IPV4 : RTE_PTYPE_L3_IPV4);
522 else if (oflags & (PKT_TX_IPV6))
523 ptype |= (tun ? RTE_PTYPE_INNER_L3_IPV6 : RTE_PTYPE_L3_IPV6);
526 switch (oflags & (PKT_TX_L4_MASK)) {
527 case PKT_TX_TCP_CKSUM:
528 ptype |= (tun ? RTE_PTYPE_INNER_L4_TCP : RTE_PTYPE_L4_TCP);
530 case PKT_TX_UDP_CKSUM:
531 ptype |= (tun ? RTE_PTYPE_INNER_L4_UDP : RTE_PTYPE_L4_UDP);
533 case PKT_TX_SCTP_CKSUM:
534 ptype |= (tun ? RTE_PTYPE_INNER_L4_SCTP : RTE_PTYPE_L4_SCTP);
538 if (oflags & PKT_TX_TCP_SEG)
539 ptype |= (tun ? RTE_PTYPE_INNER_L4_TCP : RTE_PTYPE_L4_TCP);
542 switch (oflags & PKT_TX_TUNNEL_MASK) {
543 case PKT_TX_TUNNEL_VXLAN:
544 ptype |= RTE_PTYPE_L2_ETHER |
546 RTE_PTYPE_TUNNEL_VXLAN;
547 ptype |= RTE_PTYPE_INNER_L2_ETHER;
549 case PKT_TX_TUNNEL_GRE:
550 ptype |= RTE_PTYPE_L2_ETHER |
552 RTE_PTYPE_TUNNEL_GRE;
553 ptype |= RTE_PTYPE_INNER_L2_ETHER;
555 case PKT_TX_TUNNEL_GENEVE:
556 ptype |= RTE_PTYPE_L2_ETHER |
558 RTE_PTYPE_TUNNEL_GENEVE;
559 ptype |= RTE_PTYPE_INNER_L2_ETHER;
561 case PKT_TX_TUNNEL_VXLAN_GPE:
562 ptype |= RTE_PTYPE_L2_ETHER |
564 RTE_PTYPE_TUNNEL_VXLAN_GPE;
565 ptype |= RTE_PTYPE_INNER_L2_ETHER;
567 case PKT_TX_TUNNEL_IPIP:
568 case PKT_TX_TUNNEL_IP:
569 ptype |= RTE_PTYPE_L2_ETHER |
575 return txgbe_encode_ptype(ptype);
578 #ifndef DEFAULT_TX_FREE_THRESH
579 #define DEFAULT_TX_FREE_THRESH 32
582 /* Reset transmit descriptors after they have been used */
584 txgbe_xmit_cleanup(struct txgbe_tx_queue *txq)
586 struct txgbe_tx_entry *sw_ring = txq->sw_ring;
587 volatile struct txgbe_tx_desc *txr = txq->tx_ring;
588 uint16_t last_desc_cleaned = txq->last_desc_cleaned;
589 uint16_t nb_tx_desc = txq->nb_tx_desc;
590 uint16_t desc_to_clean_to;
591 uint16_t nb_tx_to_clean;
594 /* Determine the last descriptor needing to be cleaned */
595 desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->tx_free_thresh);
596 if (desc_to_clean_to >= nb_tx_desc)
597 desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc);
599 /* Check to make sure the last descriptor to clean is done */
600 desc_to_clean_to = sw_ring[desc_to_clean_to].last_id;
601 status = txr[desc_to_clean_to].dw3;
602 if (!(status & rte_cpu_to_le_32(TXGBE_TXD_DD))) {
603 PMD_TX_FREE_LOG(DEBUG,
604 "TX descriptor %4u is not done"
605 "(port=%d queue=%d)",
607 txq->port_id, txq->queue_id);
608 if (txq->nb_tx_free >> 1 < txq->tx_free_thresh)
609 txgbe_set32_masked(txq->tdc_reg_addr,
610 TXGBE_TXCFG_FLUSH, TXGBE_TXCFG_FLUSH);
611 /* Failed to clean any descriptors, better luck next time */
615 /* Figure out how many descriptors will be cleaned */
616 if (last_desc_cleaned > desc_to_clean_to)
617 nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) +
620 nb_tx_to_clean = (uint16_t)(desc_to_clean_to -
623 PMD_TX_FREE_LOG(DEBUG,
624 "Cleaning %4u TX descriptors: %4u to %4u "
625 "(port=%d queue=%d)",
626 nb_tx_to_clean, last_desc_cleaned, desc_to_clean_to,
627 txq->port_id, txq->queue_id);
630 * The last descriptor to clean is done, so that means all the
631 * descriptors from the last descriptor that was cleaned
632 * up to the last descriptor with the RS bit set
633 * are done. Only reset the threshold descriptor.
635 txr[desc_to_clean_to].dw3 = 0;
637 /* Update the txq to reflect the last descriptor that was cleaned */
638 txq->last_desc_cleaned = desc_to_clean_to;
639 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + nb_tx_to_clean);
645 static inline uint8_t
646 txgbe_get_tun_len(struct rte_mbuf *mbuf)
648 struct txgbe_genevehdr genevehdr;
649 const struct txgbe_genevehdr *gh;
652 switch (mbuf->ol_flags & PKT_TX_TUNNEL_MASK) {
653 case PKT_TX_TUNNEL_IPIP:
656 case PKT_TX_TUNNEL_VXLAN:
657 case PKT_TX_TUNNEL_VXLAN_GPE:
658 tun_len = sizeof(struct txgbe_udphdr)
659 + sizeof(struct txgbe_vxlanhdr);
661 case PKT_TX_TUNNEL_GRE:
662 tun_len = sizeof(struct txgbe_nvgrehdr);
664 case PKT_TX_TUNNEL_GENEVE:
665 gh = rte_pktmbuf_read(mbuf,
666 mbuf->outer_l2_len + mbuf->outer_l3_len,
667 sizeof(genevehdr), &genevehdr);
668 tun_len = sizeof(struct txgbe_udphdr)
669 + sizeof(struct txgbe_genevehdr)
670 + (gh->opt_len << 2);
680 txgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
683 struct txgbe_tx_queue *txq;
684 struct txgbe_tx_entry *sw_ring;
685 struct txgbe_tx_entry *txe, *txn;
686 volatile struct txgbe_tx_desc *txr;
687 volatile struct txgbe_tx_desc *txd;
688 struct rte_mbuf *tx_pkt;
689 struct rte_mbuf *m_seg;
690 uint64_t buf_dma_addr;
691 uint32_t olinfo_status;
692 uint32_t cmd_type_len;
703 union txgbe_tx_offload tx_offload;
705 tx_offload.data[0] = 0;
706 tx_offload.data[1] = 0;
708 sw_ring = txq->sw_ring;
710 tx_id = txq->tx_tail;
711 txe = &sw_ring[tx_id];
713 /* Determine if the descriptor ring needs to be cleaned. */
714 if (txq->nb_tx_free < txq->tx_free_thresh)
715 txgbe_xmit_cleanup(txq);
717 rte_prefetch0(&txe->mbuf->pool);
720 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
723 pkt_len = tx_pkt->pkt_len;
726 * Determine how many (if any) context descriptors
727 * are needed for offload functionality.
729 ol_flags = tx_pkt->ol_flags;
731 /* If hardware offload required */
732 tx_ol_req = ol_flags & TXGBE_TX_OFFLOAD_MASK;
734 tx_offload.ptid = tx_desc_ol_flags_to_ptid(tx_ol_req,
735 tx_pkt->packet_type);
736 tx_offload.l2_len = tx_pkt->l2_len;
737 tx_offload.l3_len = tx_pkt->l3_len;
738 tx_offload.l4_len = tx_pkt->l4_len;
739 tx_offload.vlan_tci = tx_pkt->vlan_tci;
740 tx_offload.tso_segsz = tx_pkt->tso_segsz;
741 tx_offload.outer_l2_len = tx_pkt->outer_l2_len;
742 tx_offload.outer_l3_len = tx_pkt->outer_l3_len;
743 tx_offload.outer_tun_len = txgbe_get_tun_len(tx_pkt);
745 /* If new context need be built or reuse the exist ctx*/
746 ctx = what_ctx_update(txq, tx_ol_req, tx_offload);
747 /* Only allocate context descriptor if required */
748 new_ctx = (ctx == TXGBE_CTX_NUM);
753 * Keep track of how many descriptors are used this loop
754 * This will always be the number of segments + the number of
755 * Context descriptors required to transmit the packet
757 nb_used = (uint16_t)(tx_pkt->nb_segs + new_ctx);
760 * The number of descriptors that must be allocated for a
761 * packet is the number of segments of that packet, plus 1
762 * Context Descriptor for the hardware offload, if any.
763 * Determine the last TX descriptor to allocate in the TX ring
764 * for the packet, starting from the current position (tx_id)
767 tx_last = (uint16_t)(tx_id + nb_used - 1);
770 if (tx_last >= txq->nb_tx_desc)
771 tx_last = (uint16_t)(tx_last - txq->nb_tx_desc);
773 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u pktlen=%u"
774 " tx_first=%u tx_last=%u",
775 (uint16_t)txq->port_id,
776 (uint16_t)txq->queue_id,
782 * Make sure there are enough TX descriptors available to
783 * transmit the entire packet.
784 * nb_used better be less than or equal to txq->tx_free_thresh
786 if (nb_used > txq->nb_tx_free) {
787 PMD_TX_FREE_LOG(DEBUG,
788 "Not enough free TX descriptors "
789 "nb_used=%4u nb_free=%4u "
790 "(port=%d queue=%d)",
791 nb_used, txq->nb_tx_free,
792 txq->port_id, txq->queue_id);
794 if (txgbe_xmit_cleanup(txq) != 0) {
795 /* Could not clean any descriptors */
801 /* nb_used better be <= txq->tx_free_thresh */
802 if (unlikely(nb_used > txq->tx_free_thresh)) {
803 PMD_TX_FREE_LOG(DEBUG,
804 "The number of descriptors needed to "
805 "transmit the packet exceeds the "
806 "RS bit threshold. This will impact "
808 "nb_used=%4u nb_free=%4u "
809 "tx_free_thresh=%4u. "
810 "(port=%d queue=%d)",
811 nb_used, txq->nb_tx_free,
813 txq->port_id, txq->queue_id);
815 * Loop here until there are enough TX
816 * descriptors or until the ring cannot be
819 while (nb_used > txq->nb_tx_free) {
820 if (txgbe_xmit_cleanup(txq) != 0) {
822 * Could not clean any
834 * By now there are enough free TX descriptors to transmit
839 * Set common flags of all TX Data Descriptors.
841 * The following bits must be set in all Data Descriptors:
842 * - TXGBE_TXD_DTYP_DATA
843 * - TXGBE_TXD_DCMD_DEXT
845 * The following bits must be set in the first Data Descriptor
846 * and are ignored in the other ones:
847 * - TXGBE_TXD_DCMD_IFCS
848 * - TXGBE_TXD_MAC_1588
849 * - TXGBE_TXD_DCMD_VLE
851 * The following bits must only be set in the last Data
853 * - TXGBE_TXD_CMD_EOP
855 * The following bits can be set in any Data Descriptor, but
856 * are only set in the last Data Descriptor:
859 cmd_type_len = TXGBE_TXD_FCS;
861 #ifdef RTE_LIBRTE_IEEE1588
862 if (ol_flags & PKT_TX_IEEE1588_TMST)
863 cmd_type_len |= TXGBE_TXD_1588;
868 if (ol_flags & PKT_TX_TCP_SEG) {
869 /* when TSO is on, paylen in descriptor is the
870 * not the packet len but the tcp payload len
872 pkt_len -= (tx_offload.l2_len +
873 tx_offload.l3_len + tx_offload.l4_len);
875 (tx_pkt->ol_flags & PKT_TX_TUNNEL_MASK)
876 ? tx_offload.outer_l2_len +
877 tx_offload.outer_l3_len : 0;
881 * Setup the TX Advanced Context Descriptor if required
884 volatile struct txgbe_tx_ctx_desc *ctx_txd;
886 ctx_txd = (volatile struct txgbe_tx_ctx_desc *)
889 txn = &sw_ring[txe->next_id];
890 rte_prefetch0(&txn->mbuf->pool);
892 if (txe->mbuf != NULL) {
893 rte_pktmbuf_free_seg(txe->mbuf);
897 txgbe_set_xmit_ctx(txq, ctx_txd, tx_ol_req,
900 txe->last_id = tx_last;
901 tx_id = txe->next_id;
906 * Setup the TX Advanced Data Descriptor,
907 * This path will go through
908 * whatever new/reuse the context descriptor
910 cmd_type_len |= tx_desc_ol_flags_to_cmdtype(ol_flags);
912 tx_desc_cksum_flags_to_olinfo(ol_flags);
913 olinfo_status |= TXGBE_TXD_IDX(ctx);
916 olinfo_status |= TXGBE_TXD_PAYLEN(pkt_len);
921 txn = &sw_ring[txe->next_id];
922 rte_prefetch0(&txn->mbuf->pool);
924 if (txe->mbuf != NULL)
925 rte_pktmbuf_free_seg(txe->mbuf);
929 * Set up Transmit Data Descriptor.
931 slen = m_seg->data_len;
932 buf_dma_addr = rte_mbuf_data_iova(m_seg);
933 txd->qw0 = rte_cpu_to_le_64(buf_dma_addr);
934 txd->dw2 = rte_cpu_to_le_32(cmd_type_len | slen);
935 txd->dw3 = rte_cpu_to_le_32(olinfo_status);
936 txe->last_id = tx_last;
937 tx_id = txe->next_id;
940 } while (m_seg != NULL);
943 * The last packet data descriptor needs End Of Packet (EOP)
945 cmd_type_len |= TXGBE_TXD_EOP;
946 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_used);
948 txd->dw2 |= rte_cpu_to_le_32(cmd_type_len);
956 * Set the Transmit Descriptor Tail (TDT)
958 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u",
959 (uint16_t)txq->port_id, (uint16_t)txq->queue_id,
960 (uint16_t)tx_id, (uint16_t)nb_tx);
961 txgbe_set32_relaxed(txq->tdt_reg_addr, tx_id);
962 txq->tx_tail = tx_id;
967 /*********************************************************************
971 **********************************************************************/
973 txgbe_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
978 struct txgbe_tx_queue *txq = (struct txgbe_tx_queue *)tx_queue;
980 for (i = 0; i < nb_pkts; i++) {
982 ol_flags = m->ol_flags;
985 * Check if packet meets requirements for number of segments
987 * NOTE: for txgbe it's always (40 - WTHRESH) for both TSO and
991 if (m->nb_segs > TXGBE_TX_MAX_SEG - txq->wthresh) {
996 if (ol_flags & TXGBE_TX_OFFLOAD_NOTSUP_MASK) {
997 rte_errno = -ENOTSUP;
1001 #ifdef RTE_LIBRTE_ETHDEV_DEBUG
1002 ret = rte_validate_tx_offload(m);
1008 ret = rte_net_intel_cksum_prepare(m);
1018 /*********************************************************************
1022 **********************************************************************/
1023 /* @note: fix txgbe_dev_supported_ptypes_get() if any change here. */
1024 static inline uint32_t
1025 txgbe_rxd_pkt_info_to_pkt_type(uint32_t pkt_info, uint16_t ptid_mask)
1027 uint16_t ptid = TXGBE_RXD_PTID(pkt_info);
1031 return txgbe_decode_ptype(ptid);
1034 static inline uint64_t
1035 txgbe_rxd_pkt_info_to_pkt_flags(uint32_t pkt_info)
1037 static uint64_t ip_rss_types_map[16] __rte_cache_aligned = {
1038 0, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH,
1039 0, PKT_RX_RSS_HASH, 0, PKT_RX_RSS_HASH,
1040 PKT_RX_RSS_HASH, 0, 0, 0,
1041 0, 0, 0, PKT_RX_FDIR,
1043 #ifdef RTE_LIBRTE_IEEE1588
1044 static uint64_t ip_pkt_etqf_map[8] = {
1045 0, 0, 0, PKT_RX_IEEE1588_PTP,
1048 int etfid = txgbe_etflt_id(TXGBE_RXD_PTID(pkt_info));
1049 if (likely(-1 != etfid))
1050 return ip_pkt_etqf_map[etfid] |
1051 ip_rss_types_map[TXGBE_RXD_RSSTYPE(pkt_info)];
1053 return ip_rss_types_map[TXGBE_RXD_RSSTYPE(pkt_info)];
1055 return ip_rss_types_map[TXGBE_RXD_RSSTYPE(pkt_info)];
1059 static inline uint64_t
1060 rx_desc_status_to_pkt_flags(uint32_t rx_status, uint64_t vlan_flags)
1065 * Check if VLAN present only.
1066 * Do not check whether L3/L4 rx checksum done by NIC or not,
1067 * That can be found from rte_eth_rxmode.offloads flag
1069 pkt_flags = (rx_status & TXGBE_RXD_STAT_VLAN &&
1070 vlan_flags & PKT_RX_VLAN_STRIPPED)
1073 #ifdef RTE_LIBRTE_IEEE1588
1074 if (rx_status & TXGBE_RXD_STAT_1588)
1075 pkt_flags = pkt_flags | PKT_RX_IEEE1588_TMST;
1080 static inline uint64_t
1081 rx_desc_error_to_pkt_flags(uint32_t rx_status)
1083 uint64_t pkt_flags = 0;
1085 /* checksum offload can't be disabled */
1086 if (rx_status & TXGBE_RXD_STAT_IPCS) {
1087 pkt_flags |= (rx_status & TXGBE_RXD_ERR_IPCS
1088 ? PKT_RX_IP_CKSUM_BAD : PKT_RX_IP_CKSUM_GOOD);
1091 if (rx_status & TXGBE_RXD_STAT_L4CS) {
1092 pkt_flags |= (rx_status & TXGBE_RXD_ERR_L4CS
1093 ? PKT_RX_L4_CKSUM_BAD : PKT_RX_L4_CKSUM_GOOD);
1096 if (rx_status & TXGBE_RXD_STAT_EIPCS &&
1097 rx_status & TXGBE_RXD_ERR_EIPCS) {
1098 pkt_flags |= PKT_RX_EIP_CKSUM_BAD;
1105 * LOOK_AHEAD defines how many desc statuses to check beyond the
1106 * current descriptor.
1107 * It must be a pound define for optimal performance.
1108 * Do not change the value of LOOK_AHEAD, as the txgbe_rx_scan_hw_ring
1109 * function only works with LOOK_AHEAD=8.
1111 #define LOOK_AHEAD 8
1112 #if (LOOK_AHEAD != 8)
1113 #error "PMD TXGBE: LOOK_AHEAD must be 8\n"
1116 txgbe_rx_scan_hw_ring(struct txgbe_rx_queue *rxq)
1118 volatile struct txgbe_rx_desc *rxdp;
1119 struct txgbe_rx_entry *rxep;
1120 struct rte_mbuf *mb;
1124 uint32_t s[LOOK_AHEAD];
1125 uint32_t pkt_info[LOOK_AHEAD];
1126 int i, j, nb_rx = 0;
1129 /* get references to current descriptor and S/W ring entry */
1130 rxdp = &rxq->rx_ring[rxq->rx_tail];
1131 rxep = &rxq->sw_ring[rxq->rx_tail];
1133 status = rxdp->qw1.lo.status;
1134 /* check to make sure there is at least 1 packet to receive */
1135 if (!(status & rte_cpu_to_le_32(TXGBE_RXD_STAT_DD)))
1139 * Scan LOOK_AHEAD descriptors at a time to determine which descriptors
1140 * reference packets that are ready to be received.
1142 for (i = 0; i < RTE_PMD_TXGBE_RX_MAX_BURST;
1143 i += LOOK_AHEAD, rxdp += LOOK_AHEAD, rxep += LOOK_AHEAD) {
1144 /* Read desc statuses backwards to avoid race condition */
1145 for (j = 0; j < LOOK_AHEAD; j++)
1146 s[j] = rte_le_to_cpu_32(rxdp[j].qw1.lo.status);
1148 rte_atomic_thread_fence(__ATOMIC_ACQUIRE);
1150 /* Compute how many status bits were set */
1151 for (nb_dd = 0; nb_dd < LOOK_AHEAD &&
1152 (s[nb_dd] & TXGBE_RXD_STAT_DD); nb_dd++)
1155 for (j = 0; j < nb_dd; j++)
1156 pkt_info[j] = rte_le_to_cpu_32(rxdp[j].qw0.dw0);
1160 /* Translate descriptor info to mbuf format */
1161 for (j = 0; j < nb_dd; ++j) {
1163 pkt_len = rte_le_to_cpu_16(rxdp[j].qw1.hi.len) -
1165 mb->data_len = pkt_len;
1166 mb->pkt_len = pkt_len;
1167 mb->vlan_tci = rte_le_to_cpu_16(rxdp[j].qw1.hi.tag);
1169 /* convert descriptor fields to rte mbuf flags */
1170 pkt_flags = rx_desc_status_to_pkt_flags(s[j],
1172 pkt_flags |= rx_desc_error_to_pkt_flags(s[j]);
1174 txgbe_rxd_pkt_info_to_pkt_flags(pkt_info[j]);
1175 mb->ol_flags = pkt_flags;
1177 txgbe_rxd_pkt_info_to_pkt_type(pkt_info[j],
1178 rxq->pkt_type_mask);
1180 if (likely(pkt_flags & PKT_RX_RSS_HASH))
1182 rte_le_to_cpu_32(rxdp[j].qw0.dw1);
1183 else if (pkt_flags & PKT_RX_FDIR) {
1184 mb->hash.fdir.hash =
1185 rte_le_to_cpu_16(rxdp[j].qw0.hi.csum) &
1186 TXGBE_ATR_HASH_MASK;
1188 rte_le_to_cpu_16(rxdp[j].qw0.hi.ipid);
1192 /* Move mbuf pointers from the S/W ring to the stage */
1193 for (j = 0; j < LOOK_AHEAD; ++j)
1194 rxq->rx_stage[i + j] = rxep[j].mbuf;
1196 /* stop if all requested packets could not be received */
1197 if (nb_dd != LOOK_AHEAD)
1201 /* clear software ring entries so we can cleanup correctly */
1202 for (i = 0; i < nb_rx; ++i)
1203 rxq->sw_ring[rxq->rx_tail + i].mbuf = NULL;
1209 txgbe_rx_alloc_bufs(struct txgbe_rx_queue *rxq, bool reset_mbuf)
1211 volatile struct txgbe_rx_desc *rxdp;
1212 struct txgbe_rx_entry *rxep;
1213 struct rte_mbuf *mb;
1218 /* allocate buffers in bulk directly into the S/W ring */
1219 alloc_idx = rxq->rx_free_trigger - (rxq->rx_free_thresh - 1);
1220 rxep = &rxq->sw_ring[alloc_idx];
1221 diag = rte_mempool_get_bulk(rxq->mb_pool, (void *)rxep,
1222 rxq->rx_free_thresh);
1223 if (unlikely(diag != 0))
1226 rxdp = &rxq->rx_ring[alloc_idx];
1227 for (i = 0; i < rxq->rx_free_thresh; ++i) {
1228 /* populate the static rte mbuf fields */
1231 mb->port = rxq->port_id;
1233 rte_mbuf_refcnt_set(mb, 1);
1234 mb->data_off = RTE_PKTMBUF_HEADROOM;
1236 /* populate the descriptors */
1237 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mb));
1238 TXGBE_RXD_HDRADDR(&rxdp[i], 0);
1239 TXGBE_RXD_PKTADDR(&rxdp[i], dma_addr);
1242 /* update state of internal queue structure */
1243 rxq->rx_free_trigger = rxq->rx_free_trigger + rxq->rx_free_thresh;
1244 if (rxq->rx_free_trigger >= rxq->nb_rx_desc)
1245 rxq->rx_free_trigger = rxq->rx_free_thresh - 1;
1251 static inline uint16_t
1252 txgbe_rx_fill_from_stage(struct txgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
1255 struct rte_mbuf **stage = &rxq->rx_stage[rxq->rx_next_avail];
1258 /* how many packets are ready to return? */
1259 nb_pkts = (uint16_t)RTE_MIN(nb_pkts, rxq->rx_nb_avail);
1261 /* copy mbuf pointers to the application's packet list */
1262 for (i = 0; i < nb_pkts; ++i)
1263 rx_pkts[i] = stage[i];
1265 /* update internal queue state */
1266 rxq->rx_nb_avail = (uint16_t)(rxq->rx_nb_avail - nb_pkts);
1267 rxq->rx_next_avail = (uint16_t)(rxq->rx_next_avail + nb_pkts);
1272 static inline uint16_t
1273 txgbe_rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
1276 struct txgbe_rx_queue *rxq = (struct txgbe_rx_queue *)rx_queue;
1277 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id];
1280 /* Any previously recv'd pkts will be returned from the Rx stage */
1281 if (rxq->rx_nb_avail)
1282 return txgbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
1284 /* Scan the H/W ring for packets to receive */
1285 nb_rx = (uint16_t)txgbe_rx_scan_hw_ring(rxq);
1287 /* update internal queue state */
1288 rxq->rx_next_avail = 0;
1289 rxq->rx_nb_avail = nb_rx;
1290 rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_rx);
1292 /* if required, allocate new buffers to replenish descriptors */
1293 if (rxq->rx_tail > rxq->rx_free_trigger) {
1294 uint16_t cur_free_trigger = rxq->rx_free_trigger;
1296 if (txgbe_rx_alloc_bufs(rxq, true) != 0) {
1299 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u "
1300 "queue_id=%u", (uint16_t)rxq->port_id,
1301 (uint16_t)rxq->queue_id);
1303 dev->data->rx_mbuf_alloc_failed +=
1304 rxq->rx_free_thresh;
1307 * Need to rewind any previous receives if we cannot
1308 * allocate new buffers to replenish the old ones.
1310 rxq->rx_nb_avail = 0;
1311 rxq->rx_tail = (uint16_t)(rxq->rx_tail - nb_rx);
1312 for (i = 0, j = rxq->rx_tail; i < nb_rx; ++i, ++j)
1313 rxq->sw_ring[j].mbuf = rxq->rx_stage[i];
1318 /* update tail pointer */
1320 txgbe_set32_relaxed(rxq->rdt_reg_addr, cur_free_trigger);
1323 if (rxq->rx_tail >= rxq->nb_rx_desc)
1326 /* received any packets this loop? */
1327 if (rxq->rx_nb_avail)
1328 return txgbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
1333 /* split requests into chunks of size RTE_PMD_TXGBE_RX_MAX_BURST */
1335 txgbe_recv_pkts_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
1340 if (unlikely(nb_pkts == 0))
1343 if (likely(nb_pkts <= RTE_PMD_TXGBE_RX_MAX_BURST))
1344 return txgbe_rx_recv_pkts(rx_queue, rx_pkts, nb_pkts);
1346 /* request is relatively large, chunk it up */
1351 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_TXGBE_RX_MAX_BURST);
1352 ret = txgbe_rx_recv_pkts(rx_queue, &rx_pkts[nb_rx], n);
1353 nb_rx = (uint16_t)(nb_rx + ret);
1354 nb_pkts = (uint16_t)(nb_pkts - ret);
1363 txgbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
1366 struct txgbe_rx_queue *rxq;
1367 volatile struct txgbe_rx_desc *rx_ring;
1368 volatile struct txgbe_rx_desc *rxdp;
1369 struct txgbe_rx_entry *sw_ring;
1370 struct txgbe_rx_entry *rxe;
1371 struct rte_mbuf *rxm;
1372 struct rte_mbuf *nmb;
1373 struct txgbe_rx_desc rxd;
1386 rx_id = rxq->rx_tail;
1387 rx_ring = rxq->rx_ring;
1388 sw_ring = rxq->sw_ring;
1389 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id];
1390 while (nb_rx < nb_pkts) {
1392 * The order of operations here is important as the DD status
1393 * bit must not be read after any other descriptor fields.
1394 * rx_ring and rxdp are pointing to volatile data so the order
1395 * of accesses cannot be reordered by the compiler. If they were
1396 * not volatile, they could be reordered which could lead to
1397 * using invalid descriptor fields when read from rxd.
1399 rxdp = &rx_ring[rx_id];
1400 staterr = rxdp->qw1.lo.status;
1401 if (!(staterr & rte_cpu_to_le_32(TXGBE_RXD_STAT_DD)))
1408 * If the TXGBE_RXD_STAT_EOP flag is not set, the RX packet
1409 * is likely to be invalid and to be dropped by the various
1410 * validation checks performed by the network stack.
1412 * Allocate a new mbuf to replenish the RX ring descriptor.
1413 * If the allocation fails:
1414 * - arrange for that RX descriptor to be the first one
1415 * being parsed the next time the receive function is
1416 * invoked [on the same queue].
1418 * - Stop parsing the RX ring and return immediately.
1420 * This policy do not drop the packet received in the RX
1421 * descriptor for which the allocation of a new mbuf failed.
1422 * Thus, it allows that packet to be later retrieved if
1423 * mbuf have been freed in the mean time.
1424 * As a side effect, holding RX descriptors instead of
1425 * systematically giving them back to the NIC may lead to
1426 * RX ring exhaustion situations.
1427 * However, the NIC can gracefully prevent such situations
1428 * to happen by sending specific "back-pressure" flow control
1429 * frames to its peer(s).
1431 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u "
1432 "ext_err_stat=0x%08x pkt_len=%u",
1433 (uint16_t)rxq->port_id, (uint16_t)rxq->queue_id,
1434 (uint16_t)rx_id, (uint32_t)staterr,
1435 (uint16_t)rte_le_to_cpu_16(rxd.qw1.hi.len));
1437 nmb = rte_mbuf_raw_alloc(rxq->mb_pool);
1439 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u "
1440 "queue_id=%u", (uint16_t)rxq->port_id,
1441 (uint16_t)rxq->queue_id);
1442 dev->data->rx_mbuf_alloc_failed++;
1447 rxe = &sw_ring[rx_id];
1449 if (rx_id == rxq->nb_rx_desc)
1452 /* Prefetch next mbuf while processing current one. */
1453 rte_txgbe_prefetch(sw_ring[rx_id].mbuf);
1456 * When next RX descriptor is on a cache-line boundary,
1457 * prefetch the next 4 RX descriptors and the next 8 pointers
1460 if ((rx_id & 0x3) == 0) {
1461 rte_txgbe_prefetch(&rx_ring[rx_id]);
1462 rte_txgbe_prefetch(&sw_ring[rx_id]);
1467 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
1468 TXGBE_RXD_HDRADDR(rxdp, 0);
1469 TXGBE_RXD_PKTADDR(rxdp, dma_addr);
1472 * Initialize the returned mbuf.
1473 * 1) setup generic mbuf fields:
1474 * - number of segments,
1477 * - RX port identifier.
1478 * 2) integrate hardware offload data, if any:
1479 * - RSS flag & hash,
1480 * - IP checksum flag,
1481 * - VLAN TCI, if any,
1484 pkt_len = (uint16_t)(rte_le_to_cpu_16(rxd.qw1.hi.len) -
1486 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1487 rte_packet_prefetch((char *)rxm->buf_addr + rxm->data_off);
1490 rxm->pkt_len = pkt_len;
1491 rxm->data_len = pkt_len;
1492 rxm->port = rxq->port_id;
1494 pkt_info = rte_le_to_cpu_32(rxd.qw0.dw0);
1495 /* Only valid if PKT_RX_VLAN set in pkt_flags */
1496 rxm->vlan_tci = rte_le_to_cpu_16(rxd.qw1.hi.tag);
1498 pkt_flags = rx_desc_status_to_pkt_flags(staterr,
1500 pkt_flags |= rx_desc_error_to_pkt_flags(staterr);
1501 pkt_flags |= txgbe_rxd_pkt_info_to_pkt_flags(pkt_info);
1502 rxm->ol_flags = pkt_flags;
1503 rxm->packet_type = txgbe_rxd_pkt_info_to_pkt_type(pkt_info,
1504 rxq->pkt_type_mask);
1506 if (likely(pkt_flags & PKT_RX_RSS_HASH)) {
1507 rxm->hash.rss = rte_le_to_cpu_32(rxd.qw0.dw1);
1508 } else if (pkt_flags & PKT_RX_FDIR) {
1509 rxm->hash.fdir.hash =
1510 rte_le_to_cpu_16(rxd.qw0.hi.csum) &
1511 TXGBE_ATR_HASH_MASK;
1512 rxm->hash.fdir.id = rte_le_to_cpu_16(rxd.qw0.hi.ipid);
1515 * Store the mbuf address into the next entry of the array
1516 * of returned packets.
1518 rx_pkts[nb_rx++] = rxm;
1520 rxq->rx_tail = rx_id;
1523 * If the number of free RX descriptors is greater than the RX free
1524 * threshold of the queue, advance the Receive Descriptor Tail (RDT)
1526 * Update the RDT with the value of the last processed RX descriptor
1527 * minus 1, to guarantee that the RDT register is never equal to the
1528 * RDH register, which creates a "full" ring situation from the
1529 * hardware point of view...
1531 nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold);
1532 if (nb_hold > rxq->rx_free_thresh) {
1533 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u "
1534 "nb_hold=%u nb_rx=%u",
1535 (uint16_t)rxq->port_id, (uint16_t)rxq->queue_id,
1536 (uint16_t)rx_id, (uint16_t)nb_hold,
1538 rx_id = (uint16_t)((rx_id == 0) ?
1539 (rxq->nb_rx_desc - 1) : (rx_id - 1));
1540 txgbe_set32(rxq->rdt_reg_addr, rx_id);
1543 rxq->nb_rx_hold = nb_hold;
1548 * txgbe_fill_cluster_head_buf - fill the first mbuf of the returned packet
1550 * Fill the following info in the HEAD buffer of the Rx cluster:
1551 * - RX port identifier
1552 * - hardware offload data, if any:
1554 * - IP checksum flag
1555 * - VLAN TCI, if any
1557 * @head HEAD of the packet cluster
1558 * @desc HW descriptor to get data from
1559 * @rxq Pointer to the Rx queue
1562 txgbe_fill_cluster_head_buf(struct rte_mbuf *head, struct txgbe_rx_desc *desc,
1563 struct txgbe_rx_queue *rxq, uint32_t staterr)
1568 head->port = rxq->port_id;
1570 /* The vlan_tci field is only valid when PKT_RX_VLAN is
1571 * set in the pkt_flags field.
1573 head->vlan_tci = rte_le_to_cpu_16(desc->qw1.hi.tag);
1574 pkt_info = rte_le_to_cpu_32(desc->qw0.dw0);
1575 pkt_flags = rx_desc_status_to_pkt_flags(staterr, rxq->vlan_flags);
1576 pkt_flags |= rx_desc_error_to_pkt_flags(staterr);
1577 pkt_flags |= txgbe_rxd_pkt_info_to_pkt_flags(pkt_info);
1578 head->ol_flags = pkt_flags;
1579 head->packet_type = txgbe_rxd_pkt_info_to_pkt_type(pkt_info,
1580 rxq->pkt_type_mask);
1582 if (likely(pkt_flags & PKT_RX_RSS_HASH)) {
1583 head->hash.rss = rte_le_to_cpu_32(desc->qw0.dw1);
1584 } else if (pkt_flags & PKT_RX_FDIR) {
1585 head->hash.fdir.hash = rte_le_to_cpu_16(desc->qw0.hi.csum)
1586 & TXGBE_ATR_HASH_MASK;
1587 head->hash.fdir.id = rte_le_to_cpu_16(desc->qw0.hi.ipid);
1592 * txgbe_recv_pkts_lro - receive handler for and LRO case.
1594 * @rx_queue Rx queue handle
1595 * @rx_pkts table of received packets
1596 * @nb_pkts size of rx_pkts table
1597 * @bulk_alloc if TRUE bulk allocation is used for a HW ring refilling
1599 * Handles the Rx HW ring completions when RSC feature is configured. Uses an
1600 * additional ring of txgbe_rsc_entry's that will hold the relevant RSC info.
1602 * We use the same logic as in Linux and in FreeBSD txgbe drivers:
1603 * 1) When non-EOP RSC completion arrives:
1604 * a) Update the HEAD of the current RSC aggregation cluster with the new
1605 * segment's data length.
1606 * b) Set the "next" pointer of the current segment to point to the segment
1607 * at the NEXTP index.
1608 * c) Pass the HEAD of RSC aggregation cluster on to the next NEXTP entry
1609 * in the sw_rsc_ring.
1610 * 2) When EOP arrives we just update the cluster's total length and offload
1611 * flags and deliver the cluster up to the upper layers. In our case - put it
1612 * in the rx_pkts table.
1614 * Returns the number of received packets/clusters (according to the "bulk
1615 * receive" interface).
1617 static inline uint16_t
1618 txgbe_recv_pkts_lro(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts,
1621 struct txgbe_rx_queue *rxq = rx_queue;
1622 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id];
1623 volatile struct txgbe_rx_desc *rx_ring = rxq->rx_ring;
1624 struct txgbe_rx_entry *sw_ring = rxq->sw_ring;
1625 struct txgbe_scattered_rx_entry *sw_sc_ring = rxq->sw_sc_ring;
1626 uint16_t rx_id = rxq->rx_tail;
1628 uint16_t nb_hold = rxq->nb_rx_hold;
1629 uint16_t prev_id = rxq->rx_tail;
1631 while (nb_rx < nb_pkts) {
1633 struct txgbe_rx_entry *rxe;
1634 struct txgbe_scattered_rx_entry *sc_entry;
1635 struct txgbe_scattered_rx_entry *next_sc_entry = NULL;
1636 struct txgbe_rx_entry *next_rxe = NULL;
1637 struct rte_mbuf *first_seg;
1638 struct rte_mbuf *rxm;
1639 struct rte_mbuf *nmb = NULL;
1640 struct txgbe_rx_desc rxd;
1643 volatile struct txgbe_rx_desc *rxdp;
1648 * The code in this whole file uses the volatile pointer to
1649 * ensure the read ordering of the status and the rest of the
1650 * descriptor fields (on the compiler level only!!!). This is so
1651 * UGLY - why not to just use the compiler barrier instead? DPDK
1652 * even has the rte_compiler_barrier() for that.
1654 * But most importantly this is just wrong because this doesn't
1655 * ensure memory ordering in a general case at all. For
1656 * instance, DPDK is supposed to work on Power CPUs where
1657 * compiler barrier may just not be enough!
1659 * I tried to write only this function properly to have a
1660 * starting point (as a part of an LRO/RSC series) but the
1661 * compiler cursed at me when I tried to cast away the
1662 * "volatile" from rx_ring (yes, it's volatile too!!!). So, I'm
1663 * keeping it the way it is for now.
1665 * The code in this file is broken in so many other places and
1666 * will just not work on a big endian CPU anyway therefore the
1667 * lines below will have to be revisited together with the rest
1671 * - Get rid of "volatile" and let the compiler do its job.
1672 * - Use the proper memory barrier (rte_rmb()) to ensure the
1673 * memory ordering below.
1675 rxdp = &rx_ring[rx_id];
1676 staterr = rte_le_to_cpu_32(rxdp->qw1.lo.status);
1678 if (!(staterr & TXGBE_RXD_STAT_DD))
1683 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u "
1684 "staterr=0x%x data_len=%u",
1685 rxq->port_id, rxq->queue_id, rx_id, staterr,
1686 rte_le_to_cpu_16(rxd.qw1.hi.len));
1689 nmb = rte_mbuf_raw_alloc(rxq->mb_pool);
1691 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed "
1692 "port_id=%u queue_id=%u",
1693 rxq->port_id, rxq->queue_id);
1695 dev->data->rx_mbuf_alloc_failed++;
1698 } else if (nb_hold > rxq->rx_free_thresh) {
1699 uint16_t next_rdt = rxq->rx_free_trigger;
1701 if (!txgbe_rx_alloc_bufs(rxq, false)) {
1703 txgbe_set32_relaxed(rxq->rdt_reg_addr,
1705 nb_hold -= rxq->rx_free_thresh;
1707 PMD_RX_LOG(DEBUG, "RX bulk alloc failed "
1708 "port_id=%u queue_id=%u",
1709 rxq->port_id, rxq->queue_id);
1711 dev->data->rx_mbuf_alloc_failed++;
1717 rxe = &sw_ring[rx_id];
1718 eop = staterr & TXGBE_RXD_STAT_EOP;
1720 next_id = rx_id + 1;
1721 if (next_id == rxq->nb_rx_desc)
1724 /* Prefetch next mbuf while processing current one. */
1725 rte_txgbe_prefetch(sw_ring[next_id].mbuf);
1728 * When next RX descriptor is on a cache-line boundary,
1729 * prefetch the next 4 RX descriptors and the next 4 pointers
1732 if ((next_id & 0x3) == 0) {
1733 rte_txgbe_prefetch(&rx_ring[next_id]);
1734 rte_txgbe_prefetch(&sw_ring[next_id]);
1741 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
1743 * Update RX descriptor with the physical address of the
1744 * new data buffer of the new allocated mbuf.
1748 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1749 TXGBE_RXD_HDRADDR(rxdp, 0);
1750 TXGBE_RXD_PKTADDR(rxdp, dma);
1756 * Set data length & data buffer address of mbuf.
1758 data_len = rte_le_to_cpu_16(rxd.qw1.hi.len);
1759 rxm->data_len = data_len;
1764 * Get next descriptor index:
1765 * - For RSC it's in the NEXTP field.
1766 * - For a scattered packet - it's just a following
1769 if (TXGBE_RXD_RSCCNT(rxd.qw0.dw0))
1770 nextp_id = TXGBE_RXD_NEXTP(staterr);
1774 next_sc_entry = &sw_sc_ring[nextp_id];
1775 next_rxe = &sw_ring[nextp_id];
1776 rte_txgbe_prefetch(next_rxe);
1779 sc_entry = &sw_sc_ring[rx_id];
1780 first_seg = sc_entry->fbuf;
1781 sc_entry->fbuf = NULL;
1784 * If this is the first buffer of the received packet,
1785 * set the pointer to the first mbuf of the packet and
1786 * initialize its context.
1787 * Otherwise, update the total length and the number of segments
1788 * of the current scattered packet, and update the pointer to
1789 * the last mbuf of the current packet.
1791 if (first_seg == NULL) {
1793 first_seg->pkt_len = data_len;
1794 first_seg->nb_segs = 1;
1796 first_seg->pkt_len += data_len;
1797 first_seg->nb_segs++;
1804 * If this is not the last buffer of the received packet, update
1805 * the pointer to the first mbuf at the NEXTP entry in the
1806 * sw_sc_ring and continue to parse the RX ring.
1808 if (!eop && next_rxe) {
1809 rxm->next = next_rxe->mbuf;
1810 next_sc_entry->fbuf = first_seg;
1814 /* Initialize the first mbuf of the returned packet */
1815 txgbe_fill_cluster_head_buf(first_seg, &rxd, rxq, staterr);
1818 * Deal with the case, when HW CRC srip is disabled.
1819 * That can't happen when LRO is enabled, but still could
1820 * happen for scattered RX mode.
1822 first_seg->pkt_len -= rxq->crc_len;
1823 if (unlikely(rxm->data_len <= rxq->crc_len)) {
1824 struct rte_mbuf *lp;
1826 for (lp = first_seg; lp->next != rxm; lp = lp->next)
1829 first_seg->nb_segs--;
1830 lp->data_len -= rxq->crc_len - rxm->data_len;
1832 rte_pktmbuf_free_seg(rxm);
1834 rxm->data_len -= rxq->crc_len;
1837 /* Prefetch data of first segment, if configured to do so. */
1838 rte_packet_prefetch((char *)first_seg->buf_addr +
1839 first_seg->data_off);
1842 * Store the mbuf address into the next entry of the array
1843 * of returned packets.
1845 rx_pkts[nb_rx++] = first_seg;
1849 * Record index of the next RX descriptor to probe.
1851 rxq->rx_tail = rx_id;
1854 * If the number of free RX descriptors is greater than the RX free
1855 * threshold of the queue, advance the Receive Descriptor Tail (RDT)
1857 * Update the RDT with the value of the last processed RX descriptor
1858 * minus 1, to guarantee that the RDT register is never equal to the
1859 * RDH register, which creates a "full" ring situation from the
1860 * hardware point of view...
1862 if (!bulk_alloc && nb_hold > rxq->rx_free_thresh) {
1863 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u "
1864 "nb_hold=%u nb_rx=%u",
1865 rxq->port_id, rxq->queue_id, rx_id, nb_hold, nb_rx);
1868 txgbe_set32_relaxed(rxq->rdt_reg_addr, prev_id);
1872 rxq->nb_rx_hold = nb_hold;
1877 txgbe_recv_pkts_lro_single_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
1880 return txgbe_recv_pkts_lro(rx_queue, rx_pkts, nb_pkts, false);
1884 txgbe_recv_pkts_lro_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
1887 return txgbe_recv_pkts_lro(rx_queue, rx_pkts, nb_pkts, true);
1891 txgbe_get_rx_queue_offloads(struct rte_eth_dev *dev __rte_unused)
1893 return DEV_RX_OFFLOAD_VLAN_STRIP;
1897 txgbe_get_rx_port_offloads(struct rte_eth_dev *dev)
1900 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
1901 struct rte_eth_dev_sriov *sriov = &RTE_ETH_DEV_SRIOV(dev);
1903 offloads = DEV_RX_OFFLOAD_IPV4_CKSUM |
1904 DEV_RX_OFFLOAD_UDP_CKSUM |
1905 DEV_RX_OFFLOAD_TCP_CKSUM |
1906 DEV_RX_OFFLOAD_KEEP_CRC |
1907 DEV_RX_OFFLOAD_JUMBO_FRAME |
1908 DEV_RX_OFFLOAD_VLAN_FILTER |
1909 DEV_RX_OFFLOAD_RSS_HASH |
1910 DEV_RX_OFFLOAD_SCATTER;
1912 if (!txgbe_is_vf(dev))
1913 offloads |= (DEV_RX_OFFLOAD_VLAN_FILTER |
1914 DEV_RX_OFFLOAD_QINQ_STRIP |
1915 DEV_RX_OFFLOAD_VLAN_EXTEND);
1918 * RSC is only supported by PF devices in a non-SR-IOV
1921 if (hw->mac.type == txgbe_mac_raptor && !sriov->active)
1922 offloads |= DEV_RX_OFFLOAD_TCP_LRO;
1924 if (hw->mac.type == txgbe_mac_raptor)
1925 offloads |= DEV_RX_OFFLOAD_MACSEC_STRIP;
1927 offloads |= DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM;
1932 static void __rte_cold
1933 txgbe_tx_queue_release_mbufs(struct txgbe_tx_queue *txq)
1937 if (txq->sw_ring != NULL) {
1938 for (i = 0; i < txq->nb_tx_desc; i++) {
1939 if (txq->sw_ring[i].mbuf != NULL) {
1940 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
1941 txq->sw_ring[i].mbuf = NULL;
1948 txgbe_tx_done_cleanup_full(struct txgbe_tx_queue *txq, uint32_t free_cnt)
1950 struct txgbe_tx_entry *swr_ring = txq->sw_ring;
1951 uint16_t i, tx_last, tx_id;
1952 uint16_t nb_tx_free_last;
1953 uint16_t nb_tx_to_clean;
1956 /* Start free mbuf from the next of tx_tail */
1957 tx_last = txq->tx_tail;
1958 tx_id = swr_ring[tx_last].next_id;
1960 if (txq->nb_tx_free == 0 && txgbe_xmit_cleanup(txq))
1963 nb_tx_to_clean = txq->nb_tx_free;
1964 nb_tx_free_last = txq->nb_tx_free;
1966 free_cnt = txq->nb_tx_desc;
1968 /* Loop through swr_ring to count the amount of
1969 * freeable mubfs and packets.
1971 for (pkt_cnt = 0; pkt_cnt < free_cnt; ) {
1972 for (i = 0; i < nb_tx_to_clean &&
1973 pkt_cnt < free_cnt &&
1974 tx_id != tx_last; i++) {
1975 if (swr_ring[tx_id].mbuf != NULL) {
1976 rte_pktmbuf_free_seg(swr_ring[tx_id].mbuf);
1977 swr_ring[tx_id].mbuf = NULL;
1980 * last segment in the packet,
1981 * increment packet count
1983 pkt_cnt += (swr_ring[tx_id].last_id == tx_id);
1986 tx_id = swr_ring[tx_id].next_id;
1989 if (pkt_cnt < free_cnt) {
1990 if (txgbe_xmit_cleanup(txq))
1993 nb_tx_to_clean = txq->nb_tx_free - nb_tx_free_last;
1994 nb_tx_free_last = txq->nb_tx_free;
1998 return (int)pkt_cnt;
2002 txgbe_tx_done_cleanup_simple(struct txgbe_tx_queue *txq,
2007 if (free_cnt == 0 || free_cnt > txq->nb_tx_desc)
2008 free_cnt = txq->nb_tx_desc;
2010 cnt = free_cnt - free_cnt % txq->tx_free_thresh;
2012 for (i = 0; i < cnt; i += n) {
2013 if (txq->nb_tx_desc - txq->nb_tx_free < txq->tx_free_thresh)
2016 n = txgbe_tx_free_bufs(txq);
2026 txgbe_dev_tx_done_cleanup(void *tx_queue, uint32_t free_cnt)
2028 struct txgbe_tx_queue *txq = (struct txgbe_tx_queue *)tx_queue;
2029 if (txq->offloads == 0 &&
2030 txq->tx_free_thresh >= RTE_PMD_TXGBE_TX_MAX_BURST)
2031 return txgbe_tx_done_cleanup_simple(txq, free_cnt);
2033 return txgbe_tx_done_cleanup_full(txq, free_cnt);
2036 static void __rte_cold
2037 txgbe_tx_free_swring(struct txgbe_tx_queue *txq)
2040 txq->sw_ring != NULL)
2041 rte_free(txq->sw_ring);
2044 static void __rte_cold
2045 txgbe_tx_queue_release(struct txgbe_tx_queue *txq)
2047 if (txq != NULL && txq->ops != NULL) {
2048 txq->ops->release_mbufs(txq);
2049 txq->ops->free_swring(txq);
2055 txgbe_dev_tx_queue_release(void *txq)
2057 txgbe_tx_queue_release(txq);
2060 /* (Re)set dynamic txgbe_tx_queue fields to defaults */
2061 static void __rte_cold
2062 txgbe_reset_tx_queue(struct txgbe_tx_queue *txq)
2064 static const struct txgbe_tx_desc zeroed_desc = {0};
2065 struct txgbe_tx_entry *txe = txq->sw_ring;
2068 /* Zero out HW ring memory */
2069 for (i = 0; i < txq->nb_tx_desc; i++)
2070 txq->tx_ring[i] = zeroed_desc;
2072 /* Initialize SW ring entries */
2073 prev = (uint16_t)(txq->nb_tx_desc - 1);
2074 for (i = 0; i < txq->nb_tx_desc; i++) {
2075 volatile struct txgbe_tx_desc *txd = &txq->tx_ring[i];
2077 txd->dw3 = rte_cpu_to_le_32(TXGBE_TXD_DD);
2080 txe[prev].next_id = i;
2084 txq->tx_next_dd = (uint16_t)(txq->tx_free_thresh - 1);
2088 * Always allow 1 descriptor to be un-allocated to avoid
2089 * a H/W race condition
2091 txq->last_desc_cleaned = (uint16_t)(txq->nb_tx_desc - 1);
2092 txq->nb_tx_free = (uint16_t)(txq->nb_tx_desc - 1);
2094 memset((void *)&txq->ctx_cache, 0,
2095 TXGBE_CTX_NUM * sizeof(struct txgbe_ctx_info));
2098 static const struct txgbe_txq_ops def_txq_ops = {
2099 .release_mbufs = txgbe_tx_queue_release_mbufs,
2100 .free_swring = txgbe_tx_free_swring,
2101 .reset = txgbe_reset_tx_queue,
2104 /* Takes an ethdev and a queue and sets up the tx function to be used based on
2105 * the queue parameters. Used in tx_queue_setup by primary process and then
2106 * in dev_init by secondary process when attaching to an existing ethdev.
2109 txgbe_set_tx_function(struct rte_eth_dev *dev, struct txgbe_tx_queue *txq)
2111 /* Use a simple Tx queue (no offloads, no multi segs) if possible */
2112 if (txq->offloads == 0 &&
2113 txq->tx_free_thresh >= RTE_PMD_TXGBE_TX_MAX_BURST) {
2114 PMD_INIT_LOG(DEBUG, "Using simple tx code path");
2115 dev->tx_pkt_burst = txgbe_xmit_pkts_simple;
2116 dev->tx_pkt_prepare = NULL;
2118 PMD_INIT_LOG(DEBUG, "Using full-featured tx code path");
2120 " - offloads = 0x%" PRIx64,
2123 " - tx_free_thresh = %lu [RTE_PMD_TXGBE_TX_MAX_BURST=%lu]",
2124 (unsigned long)txq->tx_free_thresh,
2125 (unsigned long)RTE_PMD_TXGBE_TX_MAX_BURST);
2126 dev->tx_pkt_burst = txgbe_xmit_pkts;
2127 dev->tx_pkt_prepare = txgbe_prep_pkts;
2132 txgbe_get_tx_queue_offloads(struct rte_eth_dev *dev)
2140 txgbe_get_tx_port_offloads(struct rte_eth_dev *dev)
2142 uint64_t tx_offload_capa;
2145 DEV_TX_OFFLOAD_VLAN_INSERT |
2146 DEV_TX_OFFLOAD_IPV4_CKSUM |
2147 DEV_TX_OFFLOAD_UDP_CKSUM |
2148 DEV_TX_OFFLOAD_TCP_CKSUM |
2149 DEV_TX_OFFLOAD_SCTP_CKSUM |
2150 DEV_TX_OFFLOAD_TCP_TSO |
2151 DEV_TX_OFFLOAD_UDP_TSO |
2152 DEV_TX_OFFLOAD_UDP_TNL_TSO |
2153 DEV_TX_OFFLOAD_IP_TNL_TSO |
2154 DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
2155 DEV_TX_OFFLOAD_GRE_TNL_TSO |
2156 DEV_TX_OFFLOAD_IPIP_TNL_TSO |
2157 DEV_TX_OFFLOAD_GENEVE_TNL_TSO |
2158 DEV_TX_OFFLOAD_MULTI_SEGS;
2160 if (!txgbe_is_vf(dev))
2161 tx_offload_capa |= DEV_TX_OFFLOAD_QINQ_INSERT;
2163 tx_offload_capa |= DEV_TX_OFFLOAD_MACSEC_INSERT;
2165 tx_offload_capa |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM;
2167 return tx_offload_capa;
2171 txgbe_dev_tx_queue_setup(struct rte_eth_dev *dev,
2174 unsigned int socket_id,
2175 const struct rte_eth_txconf *tx_conf)
2177 const struct rte_memzone *tz;
2178 struct txgbe_tx_queue *txq;
2179 struct txgbe_hw *hw;
2180 uint16_t tx_free_thresh;
2183 PMD_INIT_FUNC_TRACE();
2184 hw = TXGBE_DEV_HW(dev);
2186 offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads;
2189 * Validate number of transmit descriptors.
2190 * It must not exceed hardware maximum, and must be multiple
2193 if (nb_desc % TXGBE_TXD_ALIGN != 0 ||
2194 nb_desc > TXGBE_RING_DESC_MAX ||
2195 nb_desc < TXGBE_RING_DESC_MIN) {
2200 * The TX descriptor ring will be cleaned after txq->tx_free_thresh
2201 * descriptors are used or if the number of descriptors required
2202 * to transmit a packet is greater than the number of free TX
2204 * One descriptor in the TX ring is used as a sentinel to avoid a
2205 * H/W race condition, hence the maximum threshold constraints.
2206 * When set to zero use default values.
2208 tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ?
2209 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH);
2210 if (tx_free_thresh >= (nb_desc - 3)) {
2211 PMD_INIT_LOG(ERR, "tx_free_thresh must be less than the number of "
2212 "TX descriptors minus 3. (tx_free_thresh=%u "
2213 "port=%d queue=%d)",
2214 (unsigned int)tx_free_thresh,
2215 (int)dev->data->port_id, (int)queue_idx);
2219 if ((nb_desc % tx_free_thresh) != 0) {
2220 PMD_INIT_LOG(ERR, "tx_free_thresh must be a divisor of the "
2221 "number of TX descriptors. (tx_free_thresh=%u "
2222 "port=%d queue=%d)", (unsigned int)tx_free_thresh,
2223 (int)dev->data->port_id, (int)queue_idx);
2227 /* Free memory prior to re-allocation if needed... */
2228 if (dev->data->tx_queues[queue_idx] != NULL) {
2229 txgbe_tx_queue_release(dev->data->tx_queues[queue_idx]);
2230 dev->data->tx_queues[queue_idx] = NULL;
2233 /* First allocate the tx queue data structure */
2234 txq = rte_zmalloc_socket("ethdev TX queue",
2235 sizeof(struct txgbe_tx_queue),
2236 RTE_CACHE_LINE_SIZE, socket_id);
2241 * Allocate TX ring hardware descriptors. A memzone large enough to
2242 * handle the maximum ring size is allocated in order to allow for
2243 * resizing in later calls to the queue setup function.
2245 tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx,
2246 sizeof(struct txgbe_tx_desc) * TXGBE_RING_DESC_MAX,
2247 TXGBE_ALIGN, socket_id);
2249 txgbe_tx_queue_release(txq);
2253 txq->nb_tx_desc = nb_desc;
2254 txq->tx_free_thresh = tx_free_thresh;
2255 txq->pthresh = tx_conf->tx_thresh.pthresh;
2256 txq->hthresh = tx_conf->tx_thresh.hthresh;
2257 txq->wthresh = tx_conf->tx_thresh.wthresh;
2258 txq->queue_id = queue_idx;
2259 txq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ?
2260 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx);
2261 txq->port_id = dev->data->port_id;
2262 txq->offloads = offloads;
2263 txq->ops = &def_txq_ops;
2264 txq->tx_deferred_start = tx_conf->tx_deferred_start;
2266 /* Modification to set tail pointer for virtual function
2267 * if vf is detected.
2269 if (hw->mac.type == txgbe_mac_raptor_vf) {
2270 txq->tdt_reg_addr = TXGBE_REG_ADDR(hw, TXGBE_TXWP(queue_idx));
2271 txq->tdc_reg_addr = TXGBE_REG_ADDR(hw, TXGBE_TXCFG(queue_idx));
2273 txq->tdt_reg_addr = TXGBE_REG_ADDR(hw,
2274 TXGBE_TXWP(txq->reg_idx));
2275 txq->tdc_reg_addr = TXGBE_REG_ADDR(hw,
2276 TXGBE_TXCFG(txq->reg_idx));
2279 txq->tx_ring_phys_addr = TMZ_PADDR(tz);
2280 txq->tx_ring = (struct txgbe_tx_desc *)TMZ_VADDR(tz);
2282 /* Allocate software ring */
2283 txq->sw_ring = rte_zmalloc_socket("txq->sw_ring",
2284 sizeof(struct txgbe_tx_entry) * nb_desc,
2285 RTE_CACHE_LINE_SIZE, socket_id);
2286 if (txq->sw_ring == NULL) {
2287 txgbe_tx_queue_release(txq);
2290 PMD_INIT_LOG(DEBUG, "sw_ring=%p hw_ring=%p dma_addr=0x%" PRIx64,
2291 txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr);
2293 /* set up scalar TX function as appropriate */
2294 txgbe_set_tx_function(dev, txq);
2296 txq->ops->reset(txq);
2298 dev->data->tx_queues[queue_idx] = txq;
2304 * txgbe_free_sc_cluster - free the not-yet-completed scattered cluster
2306 * The "next" pointer of the last segment of (not-yet-completed) RSC clusters
2307 * in the sw_rsc_ring is not set to NULL but rather points to the next
2308 * mbuf of this RSC aggregation (that has not been completed yet and still
2309 * resides on the HW ring). So, instead of calling for rte_pktmbuf_free() we
2310 * will just free first "nb_segs" segments of the cluster explicitly by calling
2311 * an rte_pktmbuf_free_seg().
2313 * @m scattered cluster head
2315 static void __rte_cold
2316 txgbe_free_sc_cluster(struct rte_mbuf *m)
2318 uint16_t i, nb_segs = m->nb_segs;
2319 struct rte_mbuf *next_seg;
2321 for (i = 0; i < nb_segs; i++) {
2323 rte_pktmbuf_free_seg(m);
2328 static void __rte_cold
2329 txgbe_rx_queue_release_mbufs(struct txgbe_rx_queue *rxq)
2333 if (rxq->sw_ring != NULL) {
2334 for (i = 0; i < rxq->nb_rx_desc; i++) {
2335 if (rxq->sw_ring[i].mbuf != NULL) {
2336 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
2337 rxq->sw_ring[i].mbuf = NULL;
2340 if (rxq->rx_nb_avail) {
2341 for (i = 0; i < rxq->rx_nb_avail; ++i) {
2342 struct rte_mbuf *mb;
2344 mb = rxq->rx_stage[rxq->rx_next_avail + i];
2345 rte_pktmbuf_free_seg(mb);
2347 rxq->rx_nb_avail = 0;
2351 if (rxq->sw_sc_ring)
2352 for (i = 0; i < rxq->nb_rx_desc; i++)
2353 if (rxq->sw_sc_ring[i].fbuf) {
2354 txgbe_free_sc_cluster(rxq->sw_sc_ring[i].fbuf);
2355 rxq->sw_sc_ring[i].fbuf = NULL;
2359 static void __rte_cold
2360 txgbe_rx_queue_release(struct txgbe_rx_queue *rxq)
2363 txgbe_rx_queue_release_mbufs(rxq);
2364 rte_free(rxq->sw_ring);
2365 rte_free(rxq->sw_sc_ring);
2371 txgbe_dev_rx_queue_release(void *rxq)
2373 txgbe_rx_queue_release(rxq);
2377 * Check if Rx Burst Bulk Alloc function can be used.
2379 * 0: the preconditions are satisfied and the bulk allocation function
2381 * -EINVAL: the preconditions are NOT satisfied and the default Rx burst
2382 * function must be used.
2384 static inline int __rte_cold
2385 check_rx_burst_bulk_alloc_preconditions(struct txgbe_rx_queue *rxq)
2390 * Make sure the following pre-conditions are satisfied:
2391 * rxq->rx_free_thresh >= RTE_PMD_TXGBE_RX_MAX_BURST
2392 * rxq->rx_free_thresh < rxq->nb_rx_desc
2393 * (rxq->nb_rx_desc % rxq->rx_free_thresh) == 0
2394 * Scattered packets are not supported. This should be checked
2395 * outside of this function.
2397 if (!(rxq->rx_free_thresh >= RTE_PMD_TXGBE_RX_MAX_BURST)) {
2398 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
2399 "rxq->rx_free_thresh=%d, "
2400 "RTE_PMD_TXGBE_RX_MAX_BURST=%d",
2401 rxq->rx_free_thresh, RTE_PMD_TXGBE_RX_MAX_BURST);
2403 } else if (!(rxq->rx_free_thresh < rxq->nb_rx_desc)) {
2404 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
2405 "rxq->rx_free_thresh=%d, "
2406 "rxq->nb_rx_desc=%d",
2407 rxq->rx_free_thresh, rxq->nb_rx_desc);
2409 } else if (!((rxq->nb_rx_desc % rxq->rx_free_thresh) == 0)) {
2410 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
2411 "rxq->nb_rx_desc=%d, "
2412 "rxq->rx_free_thresh=%d",
2413 rxq->nb_rx_desc, rxq->rx_free_thresh);
2420 /* Reset dynamic txgbe_rx_queue fields back to defaults */
2421 static void __rte_cold
2422 txgbe_reset_rx_queue(struct txgbe_adapter *adapter, struct txgbe_rx_queue *rxq)
2424 static const struct txgbe_rx_desc zeroed_desc = {
2425 {{0}, {0} }, {{0}, {0} } };
2427 uint16_t len = rxq->nb_rx_desc;
2430 * By default, the Rx queue setup function allocates enough memory for
2431 * TXGBE_RING_DESC_MAX. The Rx Burst bulk allocation function requires
2432 * extra memory at the end of the descriptor ring to be zero'd out.
2434 if (adapter->rx_bulk_alloc_allowed)
2435 /* zero out extra memory */
2436 len += RTE_PMD_TXGBE_RX_MAX_BURST;
2439 * Zero out HW ring memory. Zero out extra memory at the end of
2440 * the H/W ring so look-ahead logic in Rx Burst bulk alloc function
2441 * reads extra memory as zeros.
2443 for (i = 0; i < len; i++)
2444 rxq->rx_ring[i] = zeroed_desc;
2447 * initialize extra software ring entries. Space for these extra
2448 * entries is always allocated
2450 memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf));
2451 for (i = rxq->nb_rx_desc; i < len; ++i)
2452 rxq->sw_ring[i].mbuf = &rxq->fake_mbuf;
2454 rxq->rx_nb_avail = 0;
2455 rxq->rx_next_avail = 0;
2456 rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1);
2458 rxq->nb_rx_hold = 0;
2459 rxq->pkt_first_seg = NULL;
2460 rxq->pkt_last_seg = NULL;
2464 txgbe_dev_rx_queue_setup(struct rte_eth_dev *dev,
2467 unsigned int socket_id,
2468 const struct rte_eth_rxconf *rx_conf,
2469 struct rte_mempool *mp)
2471 const struct rte_memzone *rz;
2472 struct txgbe_rx_queue *rxq;
2473 struct txgbe_hw *hw;
2475 struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev);
2478 PMD_INIT_FUNC_TRACE();
2479 hw = TXGBE_DEV_HW(dev);
2481 offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads;
2484 * Validate number of receive descriptors.
2485 * It must not exceed hardware maximum, and must be multiple
2488 if (nb_desc % TXGBE_RXD_ALIGN != 0 ||
2489 nb_desc > TXGBE_RING_DESC_MAX ||
2490 nb_desc < TXGBE_RING_DESC_MIN) {
2494 /* Free memory prior to re-allocation if needed... */
2495 if (dev->data->rx_queues[queue_idx] != NULL) {
2496 txgbe_rx_queue_release(dev->data->rx_queues[queue_idx]);
2497 dev->data->rx_queues[queue_idx] = NULL;
2500 /* First allocate the rx queue data structure */
2501 rxq = rte_zmalloc_socket("ethdev RX queue",
2502 sizeof(struct txgbe_rx_queue),
2503 RTE_CACHE_LINE_SIZE, socket_id);
2507 rxq->nb_rx_desc = nb_desc;
2508 rxq->rx_free_thresh = rx_conf->rx_free_thresh;
2509 rxq->queue_id = queue_idx;
2510 rxq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ?
2511 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx);
2512 rxq->port_id = dev->data->port_id;
2513 if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC)
2514 rxq->crc_len = RTE_ETHER_CRC_LEN;
2517 rxq->drop_en = rx_conf->rx_drop_en;
2518 rxq->rx_deferred_start = rx_conf->rx_deferred_start;
2519 rxq->offloads = offloads;
2522 * The packet type in RX descriptor is different for different NICs.
2523 * So set different masks for different NICs.
2525 rxq->pkt_type_mask = TXGBE_PTID_MASK;
2528 * Allocate RX ring hardware descriptors. A memzone large enough to
2529 * handle the maximum ring size is allocated in order to allow for
2530 * resizing in later calls to the queue setup function.
2532 rz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx,
2533 RX_RING_SZ, TXGBE_ALIGN, socket_id);
2535 txgbe_rx_queue_release(rxq);
2540 * Zero init all the descriptors in the ring.
2542 memset(rz->addr, 0, RX_RING_SZ);
2545 * Modified to setup VFRDT for Virtual Function
2547 if (hw->mac.type == txgbe_mac_raptor_vf) {
2549 TXGBE_REG_ADDR(hw, TXGBE_RXWP(queue_idx));
2551 TXGBE_REG_ADDR(hw, TXGBE_RXRP(queue_idx));
2554 TXGBE_REG_ADDR(hw, TXGBE_RXWP(rxq->reg_idx));
2556 TXGBE_REG_ADDR(hw, TXGBE_RXRP(rxq->reg_idx));
2559 rxq->rx_ring_phys_addr = TMZ_PADDR(rz);
2560 rxq->rx_ring = (struct txgbe_rx_desc *)TMZ_VADDR(rz);
2563 * Certain constraints must be met in order to use the bulk buffer
2564 * allocation Rx burst function. If any of Rx queues doesn't meet them
2565 * the feature should be disabled for the whole port.
2567 if (check_rx_burst_bulk_alloc_preconditions(rxq)) {
2568 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Rx Bulk Alloc "
2569 "preconditions - canceling the feature for "
2570 "the whole port[%d]",
2571 rxq->queue_id, rxq->port_id);
2572 adapter->rx_bulk_alloc_allowed = false;
2576 * Allocate software ring. Allow for space at the end of the
2577 * S/W ring to make sure look-ahead logic in bulk alloc Rx burst
2578 * function does not access an invalid memory region.
2581 if (adapter->rx_bulk_alloc_allowed)
2582 len += RTE_PMD_TXGBE_RX_MAX_BURST;
2584 rxq->sw_ring = rte_zmalloc_socket("rxq->sw_ring",
2585 sizeof(struct txgbe_rx_entry) * len,
2586 RTE_CACHE_LINE_SIZE, socket_id);
2587 if (!rxq->sw_ring) {
2588 txgbe_rx_queue_release(rxq);
2593 * Always allocate even if it's not going to be needed in order to
2594 * simplify the code.
2596 * This ring is used in LRO and Scattered Rx cases and Scattered Rx may
2597 * be requested in txgbe_dev_rx_init(), which is called later from
2601 rte_zmalloc_socket("rxq->sw_sc_ring",
2602 sizeof(struct txgbe_scattered_rx_entry) * len,
2603 RTE_CACHE_LINE_SIZE, socket_id);
2604 if (!rxq->sw_sc_ring) {
2605 txgbe_rx_queue_release(rxq);
2609 PMD_INIT_LOG(DEBUG, "sw_ring=%p sw_sc_ring=%p hw_ring=%p "
2610 "dma_addr=0x%" PRIx64,
2611 rxq->sw_ring, rxq->sw_sc_ring, rxq->rx_ring,
2612 rxq->rx_ring_phys_addr);
2614 dev->data->rx_queues[queue_idx] = rxq;
2616 txgbe_reset_rx_queue(adapter, rxq);
2622 txgbe_dev_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id)
2624 #define TXGBE_RXQ_SCAN_INTERVAL 4
2625 volatile struct txgbe_rx_desc *rxdp;
2626 struct txgbe_rx_queue *rxq;
2629 rxq = dev->data->rx_queues[rx_queue_id];
2630 rxdp = &rxq->rx_ring[rxq->rx_tail];
2632 while ((desc < rxq->nb_rx_desc) &&
2633 (rxdp->qw1.lo.status &
2634 rte_cpu_to_le_32(TXGBE_RXD_STAT_DD))) {
2635 desc += TXGBE_RXQ_SCAN_INTERVAL;
2636 rxdp += TXGBE_RXQ_SCAN_INTERVAL;
2637 if (rxq->rx_tail + desc >= rxq->nb_rx_desc)
2638 rxdp = &(rxq->rx_ring[rxq->rx_tail +
2639 desc - rxq->nb_rx_desc]);
2646 txgbe_dev_rx_descriptor_status(void *rx_queue, uint16_t offset)
2648 struct txgbe_rx_queue *rxq = rx_queue;
2649 volatile uint32_t *status;
2650 uint32_t nb_hold, desc;
2652 if (unlikely(offset >= rxq->nb_rx_desc))
2655 nb_hold = rxq->nb_rx_hold;
2656 if (offset >= rxq->nb_rx_desc - nb_hold)
2657 return RTE_ETH_RX_DESC_UNAVAIL;
2659 desc = rxq->rx_tail + offset;
2660 if (desc >= rxq->nb_rx_desc)
2661 desc -= rxq->nb_rx_desc;
2663 status = &rxq->rx_ring[desc].qw1.lo.status;
2664 if (*status & rte_cpu_to_le_32(TXGBE_RXD_STAT_DD))
2665 return RTE_ETH_RX_DESC_DONE;
2667 return RTE_ETH_RX_DESC_AVAIL;
2671 txgbe_dev_tx_descriptor_status(void *tx_queue, uint16_t offset)
2673 struct txgbe_tx_queue *txq = tx_queue;
2674 volatile uint32_t *status;
2677 if (unlikely(offset >= txq->nb_tx_desc))
2680 desc = txq->tx_tail + offset;
2681 if (desc >= txq->nb_tx_desc) {
2682 desc -= txq->nb_tx_desc;
2683 if (desc >= txq->nb_tx_desc)
2684 desc -= txq->nb_tx_desc;
2687 status = &txq->tx_ring[desc].dw3;
2688 if (*status & rte_cpu_to_le_32(TXGBE_TXD_DD))
2689 return RTE_ETH_TX_DESC_DONE;
2691 return RTE_ETH_TX_DESC_FULL;
2695 txgbe_dev_clear_queues(struct rte_eth_dev *dev)
2698 struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev);
2700 PMD_INIT_FUNC_TRACE();
2702 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2703 struct txgbe_tx_queue *txq = dev->data->tx_queues[i];
2706 txq->ops->release_mbufs(txq);
2707 txq->ops->reset(txq);
2711 for (i = 0; i < dev->data->nb_rx_queues; i++) {
2712 struct txgbe_rx_queue *rxq = dev->data->rx_queues[i];
2715 txgbe_rx_queue_release_mbufs(rxq);
2716 txgbe_reset_rx_queue(adapter, rxq);
2722 txgbe_dev_free_queues(struct rte_eth_dev *dev)
2726 PMD_INIT_FUNC_TRACE();
2728 for (i = 0; i < dev->data->nb_rx_queues; i++) {
2729 txgbe_dev_rx_queue_release(dev->data->rx_queues[i]);
2730 dev->data->rx_queues[i] = NULL;
2732 dev->data->nb_rx_queues = 0;
2734 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2735 txgbe_dev_tx_queue_release(dev->data->tx_queues[i]);
2736 dev->data->tx_queues[i] = NULL;
2738 dev->data->nb_tx_queues = 0;
2742 * Receive Side Scaling (RSS)
2745 * The source and destination IP addresses of the IP header and the source
2746 * and destination ports of TCP/UDP headers, if any, of received packets are
2747 * hashed against a configurable random key to compute a 32-bit RSS hash result.
2748 * The seven (7) LSBs of the 32-bit hash result are used as an index into a
2749 * 128-entry redirection table (RETA). Each entry of the RETA provides a 3-bit
2750 * RSS output index which is used as the RX queue index where to store the
2752 * The following output is supplied in the RX write-back descriptor:
2753 * - 32-bit result of the Microsoft RSS hash function,
2754 * - 4-bit RSS type field.
2758 * Used as the default key.
2760 static uint8_t rss_intel_key[40] = {
2761 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
2762 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
2763 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
2764 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
2765 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
2769 txgbe_rss_disable(struct rte_eth_dev *dev)
2771 struct txgbe_hw *hw;
2773 hw = TXGBE_DEV_HW(dev);
2775 wr32m(hw, TXGBE_RACTL, TXGBE_RACTL_RSSENA, 0);
2779 txgbe_dev_rss_hash_update(struct rte_eth_dev *dev,
2780 struct rte_eth_rss_conf *rss_conf)
2782 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
2789 if (!txgbe_rss_update_sp(hw->mac.type)) {
2790 PMD_DRV_LOG(ERR, "RSS hash update is not supported on this "
2795 hash_key = rss_conf->rss_key;
2797 /* Fill in RSS hash key */
2798 for (i = 0; i < 10; i++) {
2799 rss_key = LS32(hash_key[(i * 4) + 0], 0, 0xFF);
2800 rss_key |= LS32(hash_key[(i * 4) + 1], 8, 0xFF);
2801 rss_key |= LS32(hash_key[(i * 4) + 2], 16, 0xFF);
2802 rss_key |= LS32(hash_key[(i * 4) + 3], 24, 0xFF);
2803 wr32a(hw, TXGBE_REG_RSSKEY, i, rss_key);
2807 /* Set configured hashing protocols */
2808 rss_hf = rss_conf->rss_hf & TXGBE_RSS_OFFLOAD_ALL;
2809 mrqc = rd32(hw, TXGBE_RACTL);
2810 mrqc &= ~TXGBE_RACTL_RSSMASK;
2811 if (rss_hf & ETH_RSS_IPV4)
2812 mrqc |= TXGBE_RACTL_RSSIPV4;
2813 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
2814 mrqc |= TXGBE_RACTL_RSSIPV4TCP;
2815 if (rss_hf & ETH_RSS_IPV6 ||
2816 rss_hf & ETH_RSS_IPV6_EX)
2817 mrqc |= TXGBE_RACTL_RSSIPV6;
2818 if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP ||
2819 rss_hf & ETH_RSS_IPV6_TCP_EX)
2820 mrqc |= TXGBE_RACTL_RSSIPV6TCP;
2821 if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP)
2822 mrqc |= TXGBE_RACTL_RSSIPV4UDP;
2823 if (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP ||
2824 rss_hf & ETH_RSS_IPV6_UDP_EX)
2825 mrqc |= TXGBE_RACTL_RSSIPV6UDP;
2828 mrqc |= TXGBE_RACTL_RSSENA;
2830 mrqc &= ~TXGBE_RACTL_RSSENA;
2832 wr32(hw, TXGBE_RACTL, mrqc);
2838 txgbe_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
2839 struct rte_eth_rss_conf *rss_conf)
2841 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
2848 hash_key = rss_conf->rss_key;
2850 /* Return RSS hash key */
2851 for (i = 0; i < 10; i++) {
2852 rss_key = rd32a(hw, TXGBE_REG_RSSKEY, i);
2853 hash_key[(i * 4) + 0] = RS32(rss_key, 0, 0xFF);
2854 hash_key[(i * 4) + 1] = RS32(rss_key, 8, 0xFF);
2855 hash_key[(i * 4) + 2] = RS32(rss_key, 16, 0xFF);
2856 hash_key[(i * 4) + 3] = RS32(rss_key, 24, 0xFF);
2861 mrqc = rd32(hw, TXGBE_RACTL);
2862 if (mrqc & TXGBE_RACTL_RSSIPV4)
2863 rss_hf |= ETH_RSS_IPV4;
2864 if (mrqc & TXGBE_RACTL_RSSIPV4TCP)
2865 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP;
2866 if (mrqc & TXGBE_RACTL_RSSIPV6)
2867 rss_hf |= ETH_RSS_IPV6 |
2869 if (mrqc & TXGBE_RACTL_RSSIPV6TCP)
2870 rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP |
2871 ETH_RSS_IPV6_TCP_EX;
2872 if (mrqc & TXGBE_RACTL_RSSIPV4UDP)
2873 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP;
2874 if (mrqc & TXGBE_RACTL_RSSIPV6UDP)
2875 rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP |
2876 ETH_RSS_IPV6_UDP_EX;
2877 if (!(mrqc & TXGBE_RACTL_RSSENA))
2880 rss_hf &= TXGBE_RSS_OFFLOAD_ALL;
2882 rss_conf->rss_hf = rss_hf;
2887 txgbe_rss_configure(struct rte_eth_dev *dev)
2889 struct rte_eth_rss_conf rss_conf;
2890 struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev);
2891 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
2896 PMD_INIT_FUNC_TRACE();
2899 * Fill in redirection table
2900 * The byte-swap is needed because NIC registers are in
2901 * little-endian order.
2903 if (adapter->rss_reta_updated == 0) {
2905 for (i = 0, j = 0; i < ETH_RSS_RETA_SIZE_128; i++, j++) {
2906 if (j == dev->data->nb_rx_queues)
2908 reta = (reta >> 8) | LS32(j, 24, 0xFF);
2910 wr32a(hw, TXGBE_REG_RSSTBL, i >> 2, reta);
2914 * Configure the RSS key and the RSS protocols used to compute
2915 * the RSS hash of input packets.
2917 rss_conf = dev->data->dev_conf.rx_adv_conf.rss_conf;
2918 if (rss_conf.rss_key == NULL)
2919 rss_conf.rss_key = rss_intel_key; /* Default hash key */
2920 txgbe_dev_rss_hash_update(dev, &rss_conf);
2923 #define NUM_VFTA_REGISTERS 128
2924 #define NIC_RX_BUFFER_SIZE 0x200
2927 txgbe_vmdq_dcb_configure(struct rte_eth_dev *dev)
2929 struct rte_eth_vmdq_dcb_conf *cfg;
2930 struct txgbe_hw *hw;
2931 enum rte_eth_nb_pools num_pools;
2932 uint32_t mrqc, vt_ctl, queue_mapping, vlanctrl;
2934 uint8_t nb_tcs; /* number of traffic classes */
2937 PMD_INIT_FUNC_TRACE();
2938 hw = TXGBE_DEV_HW(dev);
2939 cfg = &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf;
2940 num_pools = cfg->nb_queue_pools;
2941 /* Check we have a valid number of pools */
2942 if (num_pools != ETH_16_POOLS && num_pools != ETH_32_POOLS) {
2943 txgbe_rss_disable(dev);
2946 /* 16 pools -> 8 traffic classes, 32 pools -> 4 traffic classes */
2947 nb_tcs = (uint8_t)(ETH_VMDQ_DCB_NUM_QUEUES / (int)num_pools);
2950 * split rx buffer up into sections, each for 1 traffic class
2952 pbsize = (uint16_t)(NIC_RX_BUFFER_SIZE / nb_tcs);
2953 for (i = 0; i < nb_tcs; i++) {
2954 uint32_t rxpbsize = rd32(hw, TXGBE_PBRXSIZE(i));
2956 rxpbsize &= (~(0x3FF << 10));
2957 /* clear 10 bits. */
2958 rxpbsize |= (pbsize << 10); /* set value */
2959 wr32(hw, TXGBE_PBRXSIZE(i), rxpbsize);
2961 /* zero alloc all unused TCs */
2962 for (i = nb_tcs; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
2963 uint32_t rxpbsize = rd32(hw, TXGBE_PBRXSIZE(i));
2965 rxpbsize &= (~(0x3FF << 10));
2966 /* clear 10 bits. */
2967 wr32(hw, TXGBE_PBRXSIZE(i), rxpbsize);
2970 if (num_pools == ETH_16_POOLS) {
2971 mrqc = TXGBE_PORTCTL_NUMTC_8;
2972 mrqc |= TXGBE_PORTCTL_NUMVT_16;
2974 mrqc = TXGBE_PORTCTL_NUMTC_4;
2975 mrqc |= TXGBE_PORTCTL_NUMVT_32;
2977 wr32m(hw, TXGBE_PORTCTL,
2978 TXGBE_PORTCTL_NUMTC_MASK | TXGBE_PORTCTL_NUMVT_MASK, mrqc);
2980 vt_ctl = TXGBE_POOLCTL_RPLEN;
2981 if (cfg->enable_default_pool)
2982 vt_ctl |= TXGBE_POOLCTL_DEFPL(cfg->default_pool);
2984 vt_ctl |= TXGBE_POOLCTL_DEFDSA;
2986 wr32(hw, TXGBE_POOLCTL, vt_ctl);
2989 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++)
2991 * mapping is done with 3 bits per priority,
2992 * so shift by i*3 each time
2994 queue_mapping |= ((cfg->dcb_tc[i] & 0x07) << (i * 3));
2996 wr32(hw, TXGBE_RPUP2TC, queue_mapping);
2998 wr32(hw, TXGBE_ARBRXCTL, TXGBE_ARBRXCTL_RRM);
3000 /* enable vlan filtering and allow all vlan tags through */
3001 vlanctrl = rd32(hw, TXGBE_VLANCTL);
3002 vlanctrl |= TXGBE_VLANCTL_VFE; /* enable vlan filters */
3003 wr32(hw, TXGBE_VLANCTL, vlanctrl);
3005 /* enable all vlan filters */
3006 for (i = 0; i < NUM_VFTA_REGISTERS; i++)
3007 wr32(hw, TXGBE_VLANTBL(i), 0xFFFFFFFF);
3009 wr32(hw, TXGBE_POOLRXENA(0),
3010 num_pools == ETH_16_POOLS ? 0xFFFF : 0xFFFFFFFF);
3012 wr32(hw, TXGBE_ETHADDRIDX, 0);
3013 wr32(hw, TXGBE_ETHADDRASSL, 0xFFFFFFFF);
3014 wr32(hw, TXGBE_ETHADDRASSH, 0xFFFFFFFF);
3016 /* set up filters for vlan tags as configured */
3017 for (i = 0; i < cfg->nb_pool_maps; i++) {
3018 /* set vlan id in VF register and set the valid bit */
3019 wr32(hw, TXGBE_PSRVLANIDX, i);
3020 wr32(hw, TXGBE_PSRVLAN, (TXGBE_PSRVLAN_EA |
3021 (cfg->pool_map[i].vlan_id & 0xFFF)));
3023 wr32(hw, TXGBE_PSRVLANPLM(0), cfg->pool_map[i].pools);
3028 * txgbe_dcb_config_tx_hw_config - Configure general DCB TX parameters
3029 * @dev: pointer to eth_dev structure
3030 * @dcb_config: pointer to txgbe_dcb_config structure
3033 txgbe_dcb_tx_hw_config(struct rte_eth_dev *dev,
3034 struct txgbe_dcb_config *dcb_config)
3037 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
3039 PMD_INIT_FUNC_TRACE();
3041 /* Disable the Tx desc arbiter */
3042 reg = rd32(hw, TXGBE_ARBTXCTL);
3043 reg |= TXGBE_ARBTXCTL_DIA;
3044 wr32(hw, TXGBE_ARBTXCTL, reg);
3046 /* Enable DCB for Tx with 8 TCs */
3047 reg = rd32(hw, TXGBE_PORTCTL);
3048 reg &= TXGBE_PORTCTL_NUMTC_MASK;
3049 reg |= TXGBE_PORTCTL_DCB;
3050 if (dcb_config->num_tcs.pg_tcs == 8)
3051 reg |= TXGBE_PORTCTL_NUMTC_8;
3053 reg |= TXGBE_PORTCTL_NUMTC_4;
3055 wr32(hw, TXGBE_PORTCTL, reg);
3057 /* Enable the Tx desc arbiter */
3058 reg = rd32(hw, TXGBE_ARBTXCTL);
3059 reg &= ~TXGBE_ARBTXCTL_DIA;
3060 wr32(hw, TXGBE_ARBTXCTL, reg);
3064 * txgbe_vmdq_dcb_hw_tx_config - Configure general VMDQ+DCB TX parameters
3065 * @dev: pointer to rte_eth_dev structure
3066 * @dcb_config: pointer to txgbe_dcb_config structure
3069 txgbe_vmdq_dcb_hw_tx_config(struct rte_eth_dev *dev,
3070 struct txgbe_dcb_config *dcb_config)
3072 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3073 &dev->data->dev_conf.tx_adv_conf.vmdq_dcb_tx_conf;
3074 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
3076 PMD_INIT_FUNC_TRACE();
3077 /*PF VF Transmit Enable*/
3078 wr32(hw, TXGBE_POOLTXENA(0),
3079 vmdq_tx_conf->nb_queue_pools ==
3080 ETH_16_POOLS ? 0xFFFF : 0xFFFFFFFF);
3082 /*Configure general DCB TX parameters*/
3083 txgbe_dcb_tx_hw_config(dev, dcb_config);
3087 txgbe_vmdq_dcb_rx_config(struct rte_eth_dev *dev,
3088 struct txgbe_dcb_config *dcb_config)
3090 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3091 &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf;
3092 struct txgbe_dcb_tc_config *tc;
3095 /* convert rte_eth_conf.rx_adv_conf to struct txgbe_dcb_config */
3096 if (vmdq_rx_conf->nb_queue_pools == ETH_16_POOLS) {
3097 dcb_config->num_tcs.pg_tcs = ETH_8_TCS;
3098 dcb_config->num_tcs.pfc_tcs = ETH_8_TCS;
3100 dcb_config->num_tcs.pg_tcs = ETH_4_TCS;
3101 dcb_config->num_tcs.pfc_tcs = ETH_4_TCS;
3104 /* Initialize User Priority to Traffic Class mapping */
3105 for (j = 0; j < TXGBE_DCB_TC_MAX; j++) {
3106 tc = &dcb_config->tc_config[j];
3107 tc->path[TXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0;
3110 /* User Priority to Traffic Class mapping */
3111 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3112 j = vmdq_rx_conf->dcb_tc[i];
3113 tc = &dcb_config->tc_config[j];
3114 tc->path[TXGBE_DCB_RX_CONFIG].up_to_tc_bitmap |=
3120 txgbe_dcb_vt_tx_config(struct rte_eth_dev *dev,
3121 struct txgbe_dcb_config *dcb_config)
3123 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3124 &dev->data->dev_conf.tx_adv_conf.vmdq_dcb_tx_conf;
3125 struct txgbe_dcb_tc_config *tc;
3128 /* convert rte_eth_conf.rx_adv_conf to struct txgbe_dcb_config */
3129 if (vmdq_tx_conf->nb_queue_pools == ETH_16_POOLS) {
3130 dcb_config->num_tcs.pg_tcs = ETH_8_TCS;
3131 dcb_config->num_tcs.pfc_tcs = ETH_8_TCS;
3133 dcb_config->num_tcs.pg_tcs = ETH_4_TCS;
3134 dcb_config->num_tcs.pfc_tcs = ETH_4_TCS;
3137 /* Initialize User Priority to Traffic Class mapping */
3138 for (j = 0; j < TXGBE_DCB_TC_MAX; j++) {
3139 tc = &dcb_config->tc_config[j];
3140 tc->path[TXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0;
3143 /* User Priority to Traffic Class mapping */
3144 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3145 j = vmdq_tx_conf->dcb_tc[i];
3146 tc = &dcb_config->tc_config[j];
3147 tc->path[TXGBE_DCB_TX_CONFIG].up_to_tc_bitmap |=
3153 txgbe_dcb_rx_config(struct rte_eth_dev *dev,
3154 struct txgbe_dcb_config *dcb_config)
3156 struct rte_eth_dcb_rx_conf *rx_conf =
3157 &dev->data->dev_conf.rx_adv_conf.dcb_rx_conf;
3158 struct txgbe_dcb_tc_config *tc;
3161 dcb_config->num_tcs.pg_tcs = (uint8_t)rx_conf->nb_tcs;
3162 dcb_config->num_tcs.pfc_tcs = (uint8_t)rx_conf->nb_tcs;
3164 /* Initialize User Priority to Traffic Class mapping */
3165 for (j = 0; j < TXGBE_DCB_TC_MAX; j++) {
3166 tc = &dcb_config->tc_config[j];
3167 tc->path[TXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0;
3170 /* User Priority to Traffic Class mapping */
3171 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3172 j = rx_conf->dcb_tc[i];
3173 tc = &dcb_config->tc_config[j];
3174 tc->path[TXGBE_DCB_RX_CONFIG].up_to_tc_bitmap |=
3180 txgbe_dcb_tx_config(struct rte_eth_dev *dev,
3181 struct txgbe_dcb_config *dcb_config)
3183 struct rte_eth_dcb_tx_conf *tx_conf =
3184 &dev->data->dev_conf.tx_adv_conf.dcb_tx_conf;
3185 struct txgbe_dcb_tc_config *tc;
3188 dcb_config->num_tcs.pg_tcs = (uint8_t)tx_conf->nb_tcs;
3189 dcb_config->num_tcs.pfc_tcs = (uint8_t)tx_conf->nb_tcs;
3191 /* Initialize User Priority to Traffic Class mapping */
3192 for (j = 0; j < TXGBE_DCB_TC_MAX; j++) {
3193 tc = &dcb_config->tc_config[j];
3194 tc->path[TXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0;
3197 /* User Priority to Traffic Class mapping */
3198 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3199 j = tx_conf->dcb_tc[i];
3200 tc = &dcb_config->tc_config[j];
3201 tc->path[TXGBE_DCB_TX_CONFIG].up_to_tc_bitmap |=
3207 * txgbe_dcb_rx_hw_config - Configure general DCB RX HW parameters
3208 * @dev: pointer to eth_dev structure
3209 * @dcb_config: pointer to txgbe_dcb_config structure
3212 txgbe_dcb_rx_hw_config(struct rte_eth_dev *dev,
3213 struct txgbe_dcb_config *dcb_config)
3219 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
3221 PMD_INIT_FUNC_TRACE();
3223 * Disable the arbiter before changing parameters
3224 * (always enable recycle mode; WSP)
3226 reg = TXGBE_ARBRXCTL_RRM | TXGBE_ARBRXCTL_WSP | TXGBE_ARBRXCTL_DIA;
3227 wr32(hw, TXGBE_ARBRXCTL, reg);
3229 reg = rd32(hw, TXGBE_PORTCTL);
3230 reg &= ~(TXGBE_PORTCTL_NUMTC_MASK | TXGBE_PORTCTL_NUMVT_MASK);
3231 if (dcb_config->num_tcs.pg_tcs == 4) {
3232 reg |= TXGBE_PORTCTL_NUMTC_4;
3233 if (dcb_config->vt_mode)
3234 reg |= TXGBE_PORTCTL_NUMVT_32;
3236 wr32(hw, TXGBE_POOLCTL, 0);
3239 if (dcb_config->num_tcs.pg_tcs == 8) {
3240 reg |= TXGBE_PORTCTL_NUMTC_8;
3241 if (dcb_config->vt_mode)
3242 reg |= TXGBE_PORTCTL_NUMVT_16;
3244 wr32(hw, TXGBE_POOLCTL, 0);
3247 wr32(hw, TXGBE_PORTCTL, reg);
3249 if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
3250 /* Disable drop for all queues in VMDQ mode*/
3251 for (q = 0; q < TXGBE_MAX_RX_QUEUE_NUM; q++) {
3252 u32 val = 1 << (q % 32);
3253 wr32m(hw, TXGBE_QPRXDROP(q / 32), val, val);
3256 /* Enable drop for all queues in SRIOV mode */
3257 for (q = 0; q < TXGBE_MAX_RX_QUEUE_NUM; q++) {
3258 u32 val = 1 << (q % 32);
3259 wr32m(hw, TXGBE_QPRXDROP(q / 32), val, val);
3263 /* VLNCTL: enable vlan filtering and allow all vlan tags through */
3264 vlanctrl = rd32(hw, TXGBE_VLANCTL);
3265 vlanctrl |= TXGBE_VLANCTL_VFE; /* enable vlan filters */
3266 wr32(hw, TXGBE_VLANCTL, vlanctrl);
3268 /* VLANTBL - enable all vlan filters */
3269 for (i = 0; i < NUM_VFTA_REGISTERS; i++)
3270 wr32(hw, TXGBE_VLANTBL(i), 0xFFFFFFFF);
3273 * Configure Rx packet plane (recycle mode; WSP) and
3276 reg = TXGBE_ARBRXCTL_RRM | TXGBE_ARBRXCTL_WSP;
3277 wr32(hw, TXGBE_ARBRXCTL, reg);
3281 txgbe_dcb_hw_arbite_rx_config(struct txgbe_hw *hw, uint16_t *refill,
3282 uint16_t *max, uint8_t *bwg_id, uint8_t *tsa, uint8_t *map)
3284 txgbe_dcb_config_rx_arbiter_raptor(hw, refill, max, bwg_id,
3289 txgbe_dcb_hw_arbite_tx_config(struct txgbe_hw *hw, uint16_t *refill,
3290 uint16_t *max, uint8_t *bwg_id, uint8_t *tsa, uint8_t *map)
3292 switch (hw->mac.type) {
3293 case txgbe_mac_raptor:
3294 txgbe_dcb_config_tx_desc_arbiter_raptor(hw, refill,
3296 txgbe_dcb_config_tx_data_arbiter_raptor(hw, refill,
3297 max, bwg_id, tsa, map);
3304 #define DCB_RX_CONFIG 1
3305 #define DCB_TX_CONFIG 1
3306 #define DCB_TX_PB 1024
3308 * txgbe_dcb_hw_configure - Enable DCB and configure
3309 * general DCB in VT mode and non-VT mode parameters
3310 * @dev: pointer to rte_eth_dev structure
3311 * @dcb_config: pointer to txgbe_dcb_config structure
3314 txgbe_dcb_hw_configure(struct rte_eth_dev *dev,
3315 struct txgbe_dcb_config *dcb_config)
3318 uint8_t i, pfc_en, nb_tcs;
3319 uint16_t pbsize, rx_buffer_size;
3320 uint8_t config_dcb_rx = 0;
3321 uint8_t config_dcb_tx = 0;
3322 uint8_t tsa[TXGBE_DCB_TC_MAX] = {0};
3323 uint8_t bwgid[TXGBE_DCB_TC_MAX] = {0};
3324 uint16_t refill[TXGBE_DCB_TC_MAX] = {0};
3325 uint16_t max[TXGBE_DCB_TC_MAX] = {0};
3326 uint8_t map[TXGBE_DCB_TC_MAX] = {0};
3327 struct txgbe_dcb_tc_config *tc;
3328 uint32_t max_frame = dev->data->mtu +
3329 RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN;
3330 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
3331 struct txgbe_bw_conf *bw_conf = TXGBE_DEV_BW_CONF(dev);
3333 switch (dev->data->dev_conf.rxmode.mq_mode) {
3334 case ETH_MQ_RX_VMDQ_DCB:
3335 dcb_config->vt_mode = true;
3336 config_dcb_rx = DCB_RX_CONFIG;
3338 * get dcb and VT rx configuration parameters
3341 txgbe_vmdq_dcb_rx_config(dev, dcb_config);
3342 /*Configure general VMDQ and DCB RX parameters*/
3343 txgbe_vmdq_dcb_configure(dev);
3346 case ETH_MQ_RX_DCB_RSS:
3347 dcb_config->vt_mode = false;
3348 config_dcb_rx = DCB_RX_CONFIG;
3349 /* Get dcb TX configuration parameters from rte_eth_conf */
3350 txgbe_dcb_rx_config(dev, dcb_config);
3351 /*Configure general DCB RX parameters*/
3352 txgbe_dcb_rx_hw_config(dev, dcb_config);
3355 PMD_INIT_LOG(ERR, "Incorrect DCB RX mode configuration");
3358 switch (dev->data->dev_conf.txmode.mq_mode) {
3359 case ETH_MQ_TX_VMDQ_DCB:
3360 dcb_config->vt_mode = true;
3361 config_dcb_tx = DCB_TX_CONFIG;
3362 /* get DCB and VT TX configuration parameters
3365 txgbe_dcb_vt_tx_config(dev, dcb_config);
3366 /* Configure general VMDQ and DCB TX parameters */
3367 txgbe_vmdq_dcb_hw_tx_config(dev, dcb_config);
3371 dcb_config->vt_mode = false;
3372 config_dcb_tx = DCB_TX_CONFIG;
3373 /* get DCB TX configuration parameters from rte_eth_conf */
3374 txgbe_dcb_tx_config(dev, dcb_config);
3375 /* Configure general DCB TX parameters */
3376 txgbe_dcb_tx_hw_config(dev, dcb_config);
3379 PMD_INIT_LOG(ERR, "Incorrect DCB TX mode configuration");
3383 nb_tcs = dcb_config->num_tcs.pfc_tcs;
3385 txgbe_dcb_unpack_map_cee(dcb_config, TXGBE_DCB_RX_CONFIG, map);
3386 if (nb_tcs == ETH_4_TCS) {
3387 /* Avoid un-configured priority mapping to TC0 */
3389 uint8_t mask = 0xFF;
3391 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES - 4; i++)
3392 mask = (uint8_t)(mask & (~(1 << map[i])));
3393 for (i = 0; mask && (i < TXGBE_DCB_TC_MAX); i++) {
3394 if ((mask & 0x1) && j < ETH_DCB_NUM_USER_PRIORITIES)
3398 /* Re-configure 4 TCs BW */
3399 for (i = 0; i < nb_tcs; i++) {
3400 tc = &dcb_config->tc_config[i];
3401 if (bw_conf->tc_num != nb_tcs)
3402 tc->path[TXGBE_DCB_TX_CONFIG].bwg_percent =
3403 (uint8_t)(100 / nb_tcs);
3404 tc->path[TXGBE_DCB_RX_CONFIG].bwg_percent =
3405 (uint8_t)(100 / nb_tcs);
3407 for (; i < TXGBE_DCB_TC_MAX; i++) {
3408 tc = &dcb_config->tc_config[i];
3409 tc->path[TXGBE_DCB_TX_CONFIG].bwg_percent = 0;
3410 tc->path[TXGBE_DCB_RX_CONFIG].bwg_percent = 0;
3413 /* Re-configure 8 TCs BW */
3414 for (i = 0; i < nb_tcs; i++) {
3415 tc = &dcb_config->tc_config[i];
3416 if (bw_conf->tc_num != nb_tcs)
3417 tc->path[TXGBE_DCB_TX_CONFIG].bwg_percent =
3418 (uint8_t)(100 / nb_tcs + (i & 1));
3419 tc->path[TXGBE_DCB_RX_CONFIG].bwg_percent =
3420 (uint8_t)(100 / nb_tcs + (i & 1));
3424 rx_buffer_size = NIC_RX_BUFFER_SIZE;
3426 if (config_dcb_rx) {
3427 /* Set RX buffer size */
3428 pbsize = (uint16_t)(rx_buffer_size / nb_tcs);
3429 uint32_t rxpbsize = pbsize << 10;
3431 for (i = 0; i < nb_tcs; i++)
3432 wr32(hw, TXGBE_PBRXSIZE(i), rxpbsize);
3434 /* zero alloc all unused TCs */
3435 for (; i < ETH_DCB_NUM_USER_PRIORITIES; i++)
3436 wr32(hw, TXGBE_PBRXSIZE(i), 0);
3438 if (config_dcb_tx) {
3439 /* Only support an equally distributed
3440 * Tx packet buffer strategy.
3442 uint32_t txpktsize = TXGBE_PBTXSIZE_MAX / nb_tcs;
3443 uint32_t txpbthresh = (txpktsize / DCB_TX_PB) -
3444 TXGBE_TXPKT_SIZE_MAX;
3446 for (i = 0; i < nb_tcs; i++) {
3447 wr32(hw, TXGBE_PBTXSIZE(i), txpktsize);
3448 wr32(hw, TXGBE_PBTXDMATH(i), txpbthresh);
3450 /* Clear unused TCs, if any, to zero buffer size*/
3451 for (; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3452 wr32(hw, TXGBE_PBTXSIZE(i), 0);
3453 wr32(hw, TXGBE_PBTXDMATH(i), 0);
3457 /*Calculates traffic class credits*/
3458 txgbe_dcb_calculate_tc_credits_cee(hw, dcb_config, max_frame,
3459 TXGBE_DCB_TX_CONFIG);
3460 txgbe_dcb_calculate_tc_credits_cee(hw, dcb_config, max_frame,
3461 TXGBE_DCB_RX_CONFIG);
3463 if (config_dcb_rx) {
3464 /* Unpack CEE standard containers */
3465 txgbe_dcb_unpack_refill_cee(dcb_config,
3466 TXGBE_DCB_RX_CONFIG, refill);
3467 txgbe_dcb_unpack_max_cee(dcb_config, max);
3468 txgbe_dcb_unpack_bwgid_cee(dcb_config,
3469 TXGBE_DCB_RX_CONFIG, bwgid);
3470 txgbe_dcb_unpack_tsa_cee(dcb_config,
3471 TXGBE_DCB_RX_CONFIG, tsa);
3472 /* Configure PG(ETS) RX */
3473 txgbe_dcb_hw_arbite_rx_config(hw, refill, max, bwgid, tsa, map);
3476 if (config_dcb_tx) {
3477 /* Unpack CEE standard containers */
3478 txgbe_dcb_unpack_refill_cee(dcb_config,
3479 TXGBE_DCB_TX_CONFIG, refill);
3480 txgbe_dcb_unpack_max_cee(dcb_config, max);
3481 txgbe_dcb_unpack_bwgid_cee(dcb_config,
3482 TXGBE_DCB_TX_CONFIG, bwgid);
3483 txgbe_dcb_unpack_tsa_cee(dcb_config,
3484 TXGBE_DCB_TX_CONFIG, tsa);
3485 /* Configure PG(ETS) TX */
3486 txgbe_dcb_hw_arbite_tx_config(hw, refill, max, bwgid, tsa, map);
3489 /* Configure queue statistics registers */
3490 txgbe_dcb_config_tc_stats_raptor(hw, dcb_config);
3492 /* Check if the PFC is supported */
3493 if (dev->data->dev_conf.dcb_capability_en & ETH_DCB_PFC_SUPPORT) {
3494 pbsize = (uint16_t)(rx_buffer_size / nb_tcs);
3495 for (i = 0; i < nb_tcs; i++) {
3496 /* If the TC count is 8,
3497 * and the default high_water is 48,
3498 * the low_water is 16 as default.
3500 hw->fc.high_water[i] = (pbsize * 3) / 4;
3501 hw->fc.low_water[i] = pbsize / 4;
3502 /* Enable pfc for this TC */
3503 tc = &dcb_config->tc_config[i];
3504 tc->pfc = txgbe_dcb_pfc_enabled;
3506 txgbe_dcb_unpack_pfc_cee(dcb_config, map, &pfc_en);
3507 if (dcb_config->num_tcs.pfc_tcs == ETH_4_TCS)
3509 ret = txgbe_dcb_config_pfc(hw, pfc_en, map);
3515 void txgbe_configure_pb(struct rte_eth_dev *dev)
3517 struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
3518 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
3521 int tc = dev_conf->rx_adv_conf.dcb_rx_conf.nb_tcs;
3523 /* Reserve 256KB(/512KB) rx buffer for fdir */
3526 hw->mac.setup_pba(hw, tc, hdrm, PBA_STRATEGY_EQUAL);
3529 void txgbe_configure_port(struct rte_eth_dev *dev)
3531 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
3533 uint16_t tpids[8] = {RTE_ETHER_TYPE_VLAN, RTE_ETHER_TYPE_QINQ,
3538 PMD_INIT_FUNC_TRACE();
3540 /* default outer vlan tpid */
3541 wr32(hw, TXGBE_EXTAG,
3542 TXGBE_EXTAG_ETAG(RTE_ETHER_TYPE_ETAG) |
3543 TXGBE_EXTAG_VLAN(RTE_ETHER_TYPE_QINQ));
3545 /* default inner vlan tpid */
3546 wr32m(hw, TXGBE_VLANCTL,
3547 TXGBE_VLANCTL_TPID_MASK,
3548 TXGBE_VLANCTL_TPID(RTE_ETHER_TYPE_VLAN));
3549 wr32m(hw, TXGBE_DMATXCTRL,
3550 TXGBE_DMATXCTRL_TPID_MASK,
3551 TXGBE_DMATXCTRL_TPID(RTE_ETHER_TYPE_VLAN));
3553 /* default vlan tpid filters */
3554 for (i = 0; i < 8; i++) {
3555 wr32m(hw, TXGBE_TAGTPID(i / 2),
3556 (i % 2 ? TXGBE_TAGTPID_MSB_MASK
3557 : TXGBE_TAGTPID_LSB_MASK),
3558 (i % 2 ? TXGBE_TAGTPID_MSB(tpids[i])
3559 : TXGBE_TAGTPID_LSB(tpids[i])));
3562 /* default vxlan port */
3563 wr32(hw, TXGBE_VXLANPORT, 4789);
3567 * txgbe_configure_dcb - Configure DCB Hardware
3568 * @dev: pointer to rte_eth_dev
3570 void txgbe_configure_dcb(struct rte_eth_dev *dev)
3572 struct txgbe_dcb_config *dcb_cfg = TXGBE_DEV_DCB_CONFIG(dev);
3573 struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
3575 PMD_INIT_FUNC_TRACE();
3577 /* check support mq_mode for DCB */
3578 if (dev_conf->rxmode.mq_mode != ETH_MQ_RX_VMDQ_DCB &&
3579 dev_conf->rxmode.mq_mode != ETH_MQ_RX_DCB &&
3580 dev_conf->rxmode.mq_mode != ETH_MQ_RX_DCB_RSS)
3583 if (dev->data->nb_rx_queues > ETH_DCB_NUM_QUEUES)
3586 /** Configure DCB hardware **/
3587 txgbe_dcb_hw_configure(dev, dcb_cfg);
3591 * VMDq only support for 10 GbE NIC.
3594 txgbe_vmdq_rx_hw_configure(struct rte_eth_dev *dev)
3596 struct rte_eth_vmdq_rx_conf *cfg;
3597 struct txgbe_hw *hw;
3598 enum rte_eth_nb_pools num_pools;
3599 uint32_t mrqc, vt_ctl, vlanctrl;
3603 PMD_INIT_FUNC_TRACE();
3604 hw = TXGBE_DEV_HW(dev);
3605 cfg = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
3606 num_pools = cfg->nb_queue_pools;
3608 txgbe_rss_disable(dev);
3611 mrqc = TXGBE_PORTCTL_NUMVT_64;
3612 wr32m(hw, TXGBE_PORTCTL, TXGBE_PORTCTL_NUMVT_MASK, mrqc);
3614 /* turn on virtualisation and set the default pool */
3615 vt_ctl = TXGBE_POOLCTL_RPLEN;
3616 if (cfg->enable_default_pool)
3617 vt_ctl |= TXGBE_POOLCTL_DEFPL(cfg->default_pool);
3619 vt_ctl |= TXGBE_POOLCTL_DEFDSA;
3621 wr32(hw, TXGBE_POOLCTL, vt_ctl);
3623 for (i = 0; i < (int)num_pools; i++) {
3624 vmolr = txgbe_convert_vm_rx_mask_to_val(cfg->rx_mode, vmolr);
3625 wr32(hw, TXGBE_POOLETHCTL(i), vmolr);
3628 /* enable vlan filtering and allow all vlan tags through */
3629 vlanctrl = rd32(hw, TXGBE_VLANCTL);
3630 vlanctrl |= TXGBE_VLANCTL_VFE; /* enable vlan filters */
3631 wr32(hw, TXGBE_VLANCTL, vlanctrl);
3633 /* enable all vlan filters */
3634 for (i = 0; i < NUM_VFTA_REGISTERS; i++)
3635 wr32(hw, TXGBE_VLANTBL(i), UINT32_MAX);
3637 /* pool enabling for receive - 64 */
3638 wr32(hw, TXGBE_POOLRXENA(0), UINT32_MAX);
3639 if (num_pools == ETH_64_POOLS)
3640 wr32(hw, TXGBE_POOLRXENA(1), UINT32_MAX);
3643 * allow pools to read specific mac addresses
3644 * In this case, all pools should be able to read from mac addr 0
3646 wr32(hw, TXGBE_ETHADDRIDX, 0);
3647 wr32(hw, TXGBE_ETHADDRASSL, 0xFFFFFFFF);
3648 wr32(hw, TXGBE_ETHADDRASSH, 0xFFFFFFFF);
3650 /* set up filters for vlan tags as configured */
3651 for (i = 0; i < cfg->nb_pool_maps; i++) {
3652 /* set vlan id in VF register and set the valid bit */
3653 wr32(hw, TXGBE_PSRVLANIDX, i);
3654 wr32(hw, TXGBE_PSRVLAN, (TXGBE_PSRVLAN_EA |
3655 TXGBE_PSRVLAN_VID(cfg->pool_map[i].vlan_id)));
3657 * Put the allowed pools in VFB reg. As we only have 16 or 64
3658 * pools, we only need to use the first half of the register
3661 if (((cfg->pool_map[i].pools >> 32) & UINT32_MAX) == 0)
3662 wr32(hw, TXGBE_PSRVLANPLM(0),
3663 (cfg->pool_map[i].pools & UINT32_MAX));
3665 wr32(hw, TXGBE_PSRVLANPLM(1),
3666 ((cfg->pool_map[i].pools >> 32) & UINT32_MAX));
3669 /* Tx General Switch Control Enables VMDQ loopback */
3670 if (cfg->enable_loop_back) {
3671 wr32(hw, TXGBE_PSRCTL, TXGBE_PSRCTL_LBENA);
3672 for (i = 0; i < 64; i++)
3673 wr32m(hw, TXGBE_POOLETHCTL(i),
3674 TXGBE_POOLETHCTL_LLB, TXGBE_POOLETHCTL_LLB);
3681 * txgbe_vmdq_tx_hw_configure - Configure general VMDq TX parameters
3682 * @hw: pointer to hardware structure
3685 txgbe_vmdq_tx_hw_configure(struct txgbe_hw *hw)
3690 PMD_INIT_FUNC_TRACE();
3691 /*PF VF Transmit Enable*/
3692 wr32(hw, TXGBE_POOLTXENA(0), UINT32_MAX);
3693 wr32(hw, TXGBE_POOLTXENA(1), UINT32_MAX);
3695 /* Disable the Tx desc arbiter */
3696 reg = rd32(hw, TXGBE_ARBTXCTL);
3697 reg |= TXGBE_ARBTXCTL_DIA;
3698 wr32(hw, TXGBE_ARBTXCTL, reg);
3700 wr32m(hw, TXGBE_PORTCTL, TXGBE_PORTCTL_NUMVT_MASK,
3701 TXGBE_PORTCTL_NUMVT_64);
3703 /* Disable drop for all queues */
3704 for (q = 0; q < 128; q++) {
3705 u32 val = 1 << (q % 32);
3706 wr32m(hw, TXGBE_QPRXDROP(q / 32), val, val);
3709 /* Enable the Tx desc arbiter */
3710 reg = rd32(hw, TXGBE_ARBTXCTL);
3711 reg &= ~TXGBE_ARBTXCTL_DIA;
3712 wr32(hw, TXGBE_ARBTXCTL, reg);
3717 static int __rte_cold
3718 txgbe_alloc_rx_queue_mbufs(struct txgbe_rx_queue *rxq)
3720 struct txgbe_rx_entry *rxe = rxq->sw_ring;
3724 /* Initialize software ring entries */
3725 for (i = 0; i < rxq->nb_rx_desc; i++) {
3726 volatile struct txgbe_rx_desc *rxd;
3727 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mb_pool);
3730 PMD_INIT_LOG(ERR, "RX mbuf alloc failed queue_id=%u",
3731 (unsigned int)rxq->queue_id);
3735 mbuf->data_off = RTE_PKTMBUF_HEADROOM;
3736 mbuf->port = rxq->port_id;
3739 rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf));
3740 rxd = &rxq->rx_ring[i];
3741 TXGBE_RXD_HDRADDR(rxd, 0);
3742 TXGBE_RXD_PKTADDR(rxd, dma_addr);
3750 txgbe_config_vf_rss(struct rte_eth_dev *dev)
3752 struct txgbe_hw *hw;
3755 txgbe_rss_configure(dev);
3757 hw = TXGBE_DEV_HW(dev);
3760 mrqc = rd32(hw, TXGBE_PORTCTL);
3761 mrqc &= ~(TXGBE_PORTCTL_NUMTC_MASK | TXGBE_PORTCTL_NUMVT_MASK);
3762 switch (RTE_ETH_DEV_SRIOV(dev).active) {
3764 mrqc |= TXGBE_PORTCTL_NUMVT_64;
3768 mrqc |= TXGBE_PORTCTL_NUMVT_32;
3772 PMD_INIT_LOG(ERR, "Invalid pool number in IOV mode with VMDQ RSS");
3776 wr32(hw, TXGBE_PORTCTL, mrqc);
3782 txgbe_config_vf_default(struct rte_eth_dev *dev)
3784 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
3787 mrqc = rd32(hw, TXGBE_PORTCTL);
3788 mrqc &= ~(TXGBE_PORTCTL_NUMTC_MASK | TXGBE_PORTCTL_NUMVT_MASK);
3789 switch (RTE_ETH_DEV_SRIOV(dev).active) {
3791 mrqc |= TXGBE_PORTCTL_NUMVT_64;
3795 mrqc |= TXGBE_PORTCTL_NUMVT_32;
3799 mrqc |= TXGBE_PORTCTL_NUMVT_16;
3803 "invalid pool number in IOV mode");
3807 wr32(hw, TXGBE_PORTCTL, mrqc);
3813 txgbe_dev_mq_rx_configure(struct rte_eth_dev *dev)
3815 if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
3817 * SRIOV inactive scheme
3818 * any DCB/RSS w/o VMDq multi-queue setting
3820 switch (dev->data->dev_conf.rxmode.mq_mode) {
3822 case ETH_MQ_RX_DCB_RSS:
3823 case ETH_MQ_RX_VMDQ_RSS:
3824 txgbe_rss_configure(dev);
3827 case ETH_MQ_RX_VMDQ_DCB:
3828 txgbe_vmdq_dcb_configure(dev);
3831 case ETH_MQ_RX_VMDQ_ONLY:
3832 txgbe_vmdq_rx_hw_configure(dev);
3835 case ETH_MQ_RX_NONE:
3837 /* if mq_mode is none, disable rss mode.*/
3838 txgbe_rss_disable(dev);
3842 /* SRIOV active scheme
3843 * Support RSS together with SRIOV.
3845 switch (dev->data->dev_conf.rxmode.mq_mode) {
3847 case ETH_MQ_RX_VMDQ_RSS:
3848 txgbe_config_vf_rss(dev);
3850 case ETH_MQ_RX_VMDQ_DCB:
3852 /* In SRIOV, the configuration is the same as VMDq case */
3853 txgbe_vmdq_dcb_configure(dev);
3855 /* DCB/RSS together with SRIOV is not supported */
3856 case ETH_MQ_RX_VMDQ_DCB_RSS:
3857 case ETH_MQ_RX_DCB_RSS:
3859 "Could not support DCB/RSS with VMDq & SRIOV");
3862 txgbe_config_vf_default(dev);
3871 txgbe_dev_mq_tx_configure(struct rte_eth_dev *dev)
3873 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
3877 /* disable arbiter */
3878 rttdcs = rd32(hw, TXGBE_ARBTXCTL);
3879 rttdcs |= TXGBE_ARBTXCTL_DIA;
3880 wr32(hw, TXGBE_ARBTXCTL, rttdcs);
3882 if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
3884 * SRIOV inactive scheme
3885 * any DCB w/o VMDq multi-queue setting
3887 if (dev->data->dev_conf.txmode.mq_mode == ETH_MQ_TX_VMDQ_ONLY)
3888 txgbe_vmdq_tx_hw_configure(hw);
3890 wr32m(hw, TXGBE_PORTCTL, TXGBE_PORTCTL_NUMVT_MASK, 0);
3892 switch (RTE_ETH_DEV_SRIOV(dev).active) {
3894 * SRIOV active scheme
3895 * FIXME if support DCB together with VMDq & SRIOV
3898 mtqc = TXGBE_PORTCTL_NUMVT_64;
3901 mtqc = TXGBE_PORTCTL_NUMVT_32;
3904 mtqc = TXGBE_PORTCTL_NUMVT_16;
3908 PMD_INIT_LOG(ERR, "invalid pool number in IOV mode");
3910 wr32m(hw, TXGBE_PORTCTL, TXGBE_PORTCTL_NUMVT_MASK, mtqc);
3913 /* re-enable arbiter */
3914 rttdcs &= ~TXGBE_ARBTXCTL_DIA;
3915 wr32(hw, TXGBE_ARBTXCTL, rttdcs);
3921 * txgbe_get_rscctl_maxdesc
3923 * @pool Memory pool of the Rx queue
3925 static inline uint32_t
3926 txgbe_get_rscctl_maxdesc(struct rte_mempool *pool)
3928 struct rte_pktmbuf_pool_private *mp_priv = rte_mempool_get_priv(pool);
3931 RTE_IPV4_MAX_PKT_LEN /
3932 (mp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM);
3935 return TXGBE_RXCFG_RSCMAX_16;
3936 else if (maxdesc >= 8)
3937 return TXGBE_RXCFG_RSCMAX_8;
3938 else if (maxdesc >= 4)
3939 return TXGBE_RXCFG_RSCMAX_4;
3941 return TXGBE_RXCFG_RSCMAX_1;
3945 * txgbe_set_rsc - configure RSC related port HW registers
3947 * Configures the port's RSC related registers.
3951 * Returns 0 in case of success or a non-zero error code
3954 txgbe_set_rsc(struct rte_eth_dev *dev)
3956 struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode;
3957 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
3958 struct rte_eth_dev_info dev_info = { 0 };
3959 bool rsc_capable = false;
3965 dev->dev_ops->dev_infos_get(dev, &dev_info);
3966 if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TCP_LRO)
3969 if (!rsc_capable && (rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO)) {
3970 PMD_INIT_LOG(CRIT, "LRO is requested on HW that doesn't "
3975 /* RSC global configuration */
3977 if ((rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC) &&
3978 (rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO)) {
3979 PMD_INIT_LOG(CRIT, "LRO can't be enabled when HW CRC "
3984 rfctl = rd32(hw, TXGBE_PSRCTL);
3985 if (rsc_capable && (rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO))
3986 rfctl &= ~TXGBE_PSRCTL_RSCDIA;
3988 rfctl |= TXGBE_PSRCTL_RSCDIA;
3989 wr32(hw, TXGBE_PSRCTL, rfctl);
3991 /* If LRO hasn't been requested - we are done here. */
3992 if (!(rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO))
3995 /* Set PSRCTL.RSCACK bit */
3996 rdrxctl = rd32(hw, TXGBE_PSRCTL);
3997 rdrxctl |= TXGBE_PSRCTL_RSCACK;
3998 wr32(hw, TXGBE_PSRCTL, rdrxctl);
4000 /* Per-queue RSC configuration */
4001 for (i = 0; i < dev->data->nb_rx_queues; i++) {
4002 struct txgbe_rx_queue *rxq = dev->data->rx_queues[i];
4004 rd32(hw, TXGBE_RXCFG(rxq->reg_idx));
4006 rd32(hw, TXGBE_POOLRSS(rxq->reg_idx));
4008 rd32(hw, TXGBE_ITR(rxq->reg_idx));
4011 * txgbe PMD doesn't support header-split at the moment.
4013 srrctl &= ~TXGBE_RXCFG_HDRLEN_MASK;
4014 srrctl |= TXGBE_RXCFG_HDRLEN(128);
4017 * TODO: Consider setting the Receive Descriptor Minimum
4018 * Threshold Size for an RSC case. This is not an obviously
4019 * beneficiary option but the one worth considering...
4022 srrctl |= TXGBE_RXCFG_RSCENA;
4023 srrctl &= ~TXGBE_RXCFG_RSCMAX_MASK;
4024 srrctl |= txgbe_get_rscctl_maxdesc(rxq->mb_pool);
4025 psrtype |= TXGBE_POOLRSS_L4HDR;
4028 * RSC: Set ITR interval corresponding to 2K ints/s.
4030 * Full-sized RSC aggregations for a 10Gb/s link will
4031 * arrive at about 20K aggregation/s rate.
4033 * 2K inst/s rate will make only 10% of the
4034 * aggregations to be closed due to the interrupt timer
4035 * expiration for a streaming at wire-speed case.
4037 * For a sparse streaming case this setting will yield
4038 * at most 500us latency for a single RSC aggregation.
4040 eitr &= ~TXGBE_ITR_IVAL_MASK;
4041 eitr |= TXGBE_ITR_IVAL_10G(TXGBE_QUEUE_ITR_INTERVAL_DEFAULT);
4042 eitr |= TXGBE_ITR_WRDSA;
4044 wr32(hw, TXGBE_RXCFG(rxq->reg_idx), srrctl);
4045 wr32(hw, TXGBE_POOLRSS(rxq->reg_idx), psrtype);
4046 wr32(hw, TXGBE_ITR(rxq->reg_idx), eitr);
4049 * RSC requires the mapping of the queue to the
4052 txgbe_set_ivar_map(hw, 0, rxq->reg_idx, i);
4057 PMD_INIT_LOG(DEBUG, "enabling LRO mode");
4063 txgbe_set_rx_function(struct rte_eth_dev *dev)
4065 struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev);
4068 * Initialize the appropriate LRO callback.
4070 * If all queues satisfy the bulk allocation preconditions
4071 * (adapter->rx_bulk_alloc_allowed is TRUE) then we may use
4072 * bulk allocation. Otherwise use a single allocation version.
4074 if (dev->data->lro) {
4075 if (adapter->rx_bulk_alloc_allowed) {
4076 PMD_INIT_LOG(DEBUG, "LRO is requested. Using a bulk "
4077 "allocation version");
4078 dev->rx_pkt_burst = txgbe_recv_pkts_lro_bulk_alloc;
4080 PMD_INIT_LOG(DEBUG, "LRO is requested. Using a single "
4081 "allocation version");
4082 dev->rx_pkt_burst = txgbe_recv_pkts_lro_single_alloc;
4084 } else if (dev->data->scattered_rx) {
4086 * Set the non-LRO scattered callback: there are bulk and
4087 * single allocation versions.
4089 if (adapter->rx_bulk_alloc_allowed) {
4090 PMD_INIT_LOG(DEBUG, "Using a Scattered with bulk "
4091 "allocation callback (port=%d).",
4092 dev->data->port_id);
4093 dev->rx_pkt_burst = txgbe_recv_pkts_lro_bulk_alloc;
4095 PMD_INIT_LOG(DEBUG, "Using Regular (non-vector, "
4096 "single allocation) "
4097 "Scattered Rx callback "
4099 dev->data->port_id);
4101 dev->rx_pkt_burst = txgbe_recv_pkts_lro_single_alloc;
4104 * Below we set "simple" callbacks according to port/queues parameters.
4105 * If parameters allow we are going to choose between the following
4108 * - Single buffer allocation (the simplest one)
4110 } else if (adapter->rx_bulk_alloc_allowed) {
4111 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are "
4112 "satisfied. Rx Burst Bulk Alloc function "
4113 "will be used on port=%d.",
4114 dev->data->port_id);
4116 dev->rx_pkt_burst = txgbe_recv_pkts_bulk_alloc;
4118 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are not "
4119 "satisfied, or Scattered Rx is requested "
4121 dev->data->port_id);
4123 dev->rx_pkt_burst = txgbe_recv_pkts;
4128 * Initializes Receive Unit.
4131 txgbe_dev_rx_init(struct rte_eth_dev *dev)
4133 struct txgbe_hw *hw;
4134 struct txgbe_rx_queue *rxq;
4143 struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode;
4146 PMD_INIT_FUNC_TRACE();
4147 hw = TXGBE_DEV_HW(dev);
4150 * Make sure receives are disabled while setting
4151 * up the RX context (registers, descriptor rings, etc.).
4153 wr32m(hw, TXGBE_MACRXCFG, TXGBE_MACRXCFG_ENA, 0);
4154 wr32m(hw, TXGBE_PBRXCTL, TXGBE_PBRXCTL_ENA, 0);
4156 /* Enable receipt of broadcasted frames */
4157 fctrl = rd32(hw, TXGBE_PSRCTL);
4158 fctrl |= TXGBE_PSRCTL_BCA;
4159 wr32(hw, TXGBE_PSRCTL, fctrl);
4162 * Configure CRC stripping, if any.
4164 hlreg0 = rd32(hw, TXGBE_SECRXCTL);
4165 if (rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC)
4166 hlreg0 &= ~TXGBE_SECRXCTL_CRCSTRIP;
4168 hlreg0 |= TXGBE_SECRXCTL_CRCSTRIP;
4169 wr32(hw, TXGBE_SECRXCTL, hlreg0);
4172 * Configure jumbo frame support, if any.
4174 if (rx_conf->offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) {
4175 wr32m(hw, TXGBE_FRMSZ, TXGBE_FRMSZ_MAX_MASK,
4176 TXGBE_FRMSZ_MAX(rx_conf->max_rx_pkt_len));
4178 wr32m(hw, TXGBE_FRMSZ, TXGBE_FRMSZ_MAX_MASK,
4179 TXGBE_FRMSZ_MAX(TXGBE_FRAME_SIZE_DFT));
4183 * If loopback mode is configured, set LPBK bit.
4185 hlreg0 = rd32(hw, TXGBE_PSRCTL);
4186 if (hw->mac.type == txgbe_mac_raptor &&
4187 dev->data->dev_conf.lpbk_mode)
4188 hlreg0 |= TXGBE_PSRCTL_LBENA;
4190 hlreg0 &= ~TXGBE_PSRCTL_LBENA;
4192 wr32(hw, TXGBE_PSRCTL, hlreg0);
4195 * Assume no header split and no VLAN strip support
4196 * on any Rx queue first .
4198 rx_conf->offloads &= ~DEV_RX_OFFLOAD_VLAN_STRIP;
4200 /* Setup RX queues */
4201 for (i = 0; i < dev->data->nb_rx_queues; i++) {
4202 rxq = dev->data->rx_queues[i];
4205 * Reset crc_len in case it was changed after queue setup by a
4206 * call to configure.
4208 if (rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC)
4209 rxq->crc_len = RTE_ETHER_CRC_LEN;
4213 /* Setup the Base and Length of the Rx Descriptor Rings */
4214 bus_addr = rxq->rx_ring_phys_addr;
4215 wr32(hw, TXGBE_RXBAL(rxq->reg_idx),
4216 (uint32_t)(bus_addr & BIT_MASK32));
4217 wr32(hw, TXGBE_RXBAH(rxq->reg_idx),
4218 (uint32_t)(bus_addr >> 32));
4219 wr32(hw, TXGBE_RXRP(rxq->reg_idx), 0);
4220 wr32(hw, TXGBE_RXWP(rxq->reg_idx), 0);
4222 srrctl = TXGBE_RXCFG_RNGLEN(rxq->nb_rx_desc);
4224 /* Set if packets are dropped when no descriptors available */
4226 srrctl |= TXGBE_RXCFG_DROP;
4229 * Configure the RX buffer size in the PKTLEN field of
4230 * the RXCFG register of the queue.
4231 * The value is in 1 KB resolution. Valid values can be from
4234 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) -
4235 RTE_PKTMBUF_HEADROOM);
4236 buf_size = ROUND_UP(buf_size, 0x1 << 10);
4237 srrctl |= TXGBE_RXCFG_PKTLEN(buf_size);
4239 wr32(hw, TXGBE_RXCFG(rxq->reg_idx), srrctl);
4241 /* It adds dual VLAN length for supporting dual VLAN */
4242 if (dev->data->dev_conf.rxmode.max_rx_pkt_len +
4243 2 * TXGBE_VLAN_TAG_SIZE > buf_size)
4244 dev->data->scattered_rx = 1;
4245 if (rxq->offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
4246 rx_conf->offloads |= DEV_RX_OFFLOAD_VLAN_STRIP;
4249 if (rx_conf->offloads & DEV_RX_OFFLOAD_SCATTER)
4250 dev->data->scattered_rx = 1;
4253 * Device configured with multiple RX queues.
4255 txgbe_dev_mq_rx_configure(dev);
4258 * Setup the Checksum Register.
4259 * Disable Full-Packet Checksum which is mutually exclusive with RSS.
4260 * Enable IP/L4 checksum computation by hardware if requested to do so.
4262 rxcsum = rd32(hw, TXGBE_PSRCTL);
4263 rxcsum |= TXGBE_PSRCTL_PCSD;
4264 if (rx_conf->offloads & DEV_RX_OFFLOAD_CHECKSUM)
4265 rxcsum |= TXGBE_PSRCTL_L4CSUM;
4267 rxcsum &= ~TXGBE_PSRCTL_L4CSUM;
4269 wr32(hw, TXGBE_PSRCTL, rxcsum);
4271 if (hw->mac.type == txgbe_mac_raptor) {
4272 rdrxctl = rd32(hw, TXGBE_SECRXCTL);
4273 if (rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC)
4274 rdrxctl &= ~TXGBE_SECRXCTL_CRCSTRIP;
4276 rdrxctl |= TXGBE_SECRXCTL_CRCSTRIP;
4277 wr32(hw, TXGBE_SECRXCTL, rdrxctl);
4280 rc = txgbe_set_rsc(dev);
4284 txgbe_set_rx_function(dev);
4290 * Initializes Transmit Unit.
4293 txgbe_dev_tx_init(struct rte_eth_dev *dev)
4295 struct txgbe_hw *hw;
4296 struct txgbe_tx_queue *txq;
4300 PMD_INIT_FUNC_TRACE();
4301 hw = TXGBE_DEV_HW(dev);
4303 /* Setup the Base and Length of the Tx Descriptor Rings */
4304 for (i = 0; i < dev->data->nb_tx_queues; i++) {
4305 txq = dev->data->tx_queues[i];
4307 bus_addr = txq->tx_ring_phys_addr;
4308 wr32(hw, TXGBE_TXBAL(txq->reg_idx),
4309 (uint32_t)(bus_addr & BIT_MASK32));
4310 wr32(hw, TXGBE_TXBAH(txq->reg_idx),
4311 (uint32_t)(bus_addr >> 32));
4312 wr32m(hw, TXGBE_TXCFG(txq->reg_idx), TXGBE_TXCFG_BUFLEN_MASK,
4313 TXGBE_TXCFG_BUFLEN(txq->nb_tx_desc));
4314 /* Setup the HW Tx Head and TX Tail descriptor pointers */
4315 wr32(hw, TXGBE_TXRP(txq->reg_idx), 0);
4316 wr32(hw, TXGBE_TXWP(txq->reg_idx), 0);
4319 /* Device configured with multiple TX queues. */
4320 txgbe_dev_mq_tx_configure(dev);
4324 * Set up link loopback mode Tx->Rx.
4326 static inline void __rte_cold
4327 txgbe_setup_loopback_link_raptor(struct txgbe_hw *hw)
4329 PMD_INIT_FUNC_TRACE();
4331 wr32m(hw, TXGBE_MACRXCFG, TXGBE_MACRXCFG_LB, TXGBE_MACRXCFG_LB);
4337 * Start Transmit and Receive Units.
4340 txgbe_dev_rxtx_start(struct rte_eth_dev *dev)
4342 struct txgbe_hw *hw;
4343 struct txgbe_tx_queue *txq;
4344 struct txgbe_rx_queue *rxq;
4350 PMD_INIT_FUNC_TRACE();
4351 hw = TXGBE_DEV_HW(dev);
4353 for (i = 0; i < dev->data->nb_tx_queues; i++) {
4354 txq = dev->data->tx_queues[i];
4355 /* Setup Transmit Threshold Registers */
4356 wr32m(hw, TXGBE_TXCFG(txq->reg_idx),
4357 TXGBE_TXCFG_HTHRESH_MASK |
4358 TXGBE_TXCFG_WTHRESH_MASK,
4359 TXGBE_TXCFG_HTHRESH(txq->hthresh) |
4360 TXGBE_TXCFG_WTHRESH(txq->wthresh));
4363 dmatxctl = rd32(hw, TXGBE_DMATXCTRL);
4364 dmatxctl |= TXGBE_DMATXCTRL_ENA;
4365 wr32(hw, TXGBE_DMATXCTRL, dmatxctl);
4367 for (i = 0; i < dev->data->nb_tx_queues; i++) {
4368 txq = dev->data->tx_queues[i];
4369 if (!txq->tx_deferred_start) {
4370 ret = txgbe_dev_tx_queue_start(dev, i);
4376 for (i = 0; i < dev->data->nb_rx_queues; i++) {
4377 rxq = dev->data->rx_queues[i];
4378 if (!rxq->rx_deferred_start) {
4379 ret = txgbe_dev_rx_queue_start(dev, i);
4385 /* Enable Receive engine */
4386 rxctrl = rd32(hw, TXGBE_PBRXCTL);
4387 rxctrl |= TXGBE_PBRXCTL_ENA;
4388 hw->mac.enable_rx_dma(hw, rxctrl);
4390 /* If loopback mode is enabled, set up the link accordingly */
4391 if (hw->mac.type == txgbe_mac_raptor &&
4392 dev->data->dev_conf.lpbk_mode)
4393 txgbe_setup_loopback_link_raptor(hw);
4399 txgbe_dev_save_rx_queue(struct txgbe_hw *hw, uint16_t rx_queue_id)
4401 u32 *reg = &hw->q_rx_regs[rx_queue_id * 8];
4402 *(reg++) = rd32(hw, TXGBE_RXBAL(rx_queue_id));
4403 *(reg++) = rd32(hw, TXGBE_RXBAH(rx_queue_id));
4404 *(reg++) = rd32(hw, TXGBE_RXCFG(rx_queue_id));
4408 txgbe_dev_store_rx_queue(struct txgbe_hw *hw, uint16_t rx_queue_id)
4410 u32 *reg = &hw->q_rx_regs[rx_queue_id * 8];
4411 wr32(hw, TXGBE_RXBAL(rx_queue_id), *(reg++));
4412 wr32(hw, TXGBE_RXBAH(rx_queue_id), *(reg++));
4413 wr32(hw, TXGBE_RXCFG(rx_queue_id), *(reg++) & ~TXGBE_RXCFG_ENA);
4417 txgbe_dev_save_tx_queue(struct txgbe_hw *hw, uint16_t tx_queue_id)
4419 u32 *reg = &hw->q_tx_regs[tx_queue_id * 8];
4420 *(reg++) = rd32(hw, TXGBE_TXBAL(tx_queue_id));
4421 *(reg++) = rd32(hw, TXGBE_TXBAH(tx_queue_id));
4422 *(reg++) = rd32(hw, TXGBE_TXCFG(tx_queue_id));
4426 txgbe_dev_store_tx_queue(struct txgbe_hw *hw, uint16_t tx_queue_id)
4428 u32 *reg = &hw->q_tx_regs[tx_queue_id * 8];
4429 wr32(hw, TXGBE_TXBAL(tx_queue_id), *(reg++));
4430 wr32(hw, TXGBE_TXBAH(tx_queue_id), *(reg++));
4431 wr32(hw, TXGBE_TXCFG(tx_queue_id), *(reg++) & ~TXGBE_TXCFG_ENA);
4435 * Start Receive Units for specified queue.
4438 txgbe_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
4440 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
4441 struct txgbe_rx_queue *rxq;
4445 PMD_INIT_FUNC_TRACE();
4447 rxq = dev->data->rx_queues[rx_queue_id];
4449 /* Allocate buffers for descriptor rings */
4450 if (txgbe_alloc_rx_queue_mbufs(rxq) != 0) {
4451 PMD_INIT_LOG(ERR, "Could not alloc mbuf for queue:%d",
4455 rxdctl = rd32(hw, TXGBE_RXCFG(rxq->reg_idx));
4456 rxdctl |= TXGBE_RXCFG_ENA;
4457 wr32(hw, TXGBE_RXCFG(rxq->reg_idx), rxdctl);
4459 /* Wait until RX Enable ready */
4460 poll_ms = RTE_TXGBE_REGISTER_POLL_WAIT_10_MS;
4463 rxdctl = rd32(hw, TXGBE_RXCFG(rxq->reg_idx));
4464 } while (--poll_ms && !(rxdctl & TXGBE_RXCFG_ENA));
4466 PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", rx_queue_id);
4468 wr32(hw, TXGBE_RXRP(rxq->reg_idx), 0);
4469 wr32(hw, TXGBE_RXWP(rxq->reg_idx), rxq->nb_rx_desc - 1);
4470 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
4476 * Stop Receive Units for specified queue.
4479 txgbe_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
4481 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
4482 struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev);
4483 struct txgbe_rx_queue *rxq;
4487 PMD_INIT_FUNC_TRACE();
4489 rxq = dev->data->rx_queues[rx_queue_id];
4491 txgbe_dev_save_rx_queue(hw, rxq->reg_idx);
4492 wr32m(hw, TXGBE_RXCFG(rxq->reg_idx), TXGBE_RXCFG_ENA, 0);
4494 /* Wait until RX Enable bit clear */
4495 poll_ms = RTE_TXGBE_REGISTER_POLL_WAIT_10_MS;
4498 rxdctl = rd32(hw, TXGBE_RXCFG(rxq->reg_idx));
4499 } while (--poll_ms && (rxdctl & TXGBE_RXCFG_ENA));
4501 PMD_INIT_LOG(ERR, "Could not disable Rx Queue %d", rx_queue_id);
4503 rte_delay_us(RTE_TXGBE_WAIT_100_US);
4504 txgbe_dev_store_rx_queue(hw, rxq->reg_idx);
4506 txgbe_rx_queue_release_mbufs(rxq);
4507 txgbe_reset_rx_queue(adapter, rxq);
4508 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
4514 * Start Transmit Units for specified queue.
4517 txgbe_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
4519 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
4520 struct txgbe_tx_queue *txq;
4524 PMD_INIT_FUNC_TRACE();
4526 txq = dev->data->tx_queues[tx_queue_id];
4527 wr32m(hw, TXGBE_TXCFG(txq->reg_idx), TXGBE_TXCFG_ENA, TXGBE_TXCFG_ENA);
4529 /* Wait until TX Enable ready */
4530 poll_ms = RTE_TXGBE_REGISTER_POLL_WAIT_10_MS;
4533 txdctl = rd32(hw, TXGBE_TXCFG(txq->reg_idx));
4534 } while (--poll_ms && !(txdctl & TXGBE_TXCFG_ENA));
4536 PMD_INIT_LOG(ERR, "Could not enable "
4537 "Tx Queue %d", tx_queue_id);
4540 wr32(hw, TXGBE_TXWP(txq->reg_idx), txq->tx_tail);
4541 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
4547 * Stop Transmit Units for specified queue.
4550 txgbe_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
4552 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
4553 struct txgbe_tx_queue *txq;
4555 uint32_t txtdh, txtdt;
4558 PMD_INIT_FUNC_TRACE();
4560 txq = dev->data->tx_queues[tx_queue_id];
4562 /* Wait until TX queue is empty */
4563 poll_ms = RTE_TXGBE_REGISTER_POLL_WAIT_10_MS;
4565 rte_delay_us(RTE_TXGBE_WAIT_100_US);
4566 txtdh = rd32(hw, TXGBE_TXRP(txq->reg_idx));
4567 txtdt = rd32(hw, TXGBE_TXWP(txq->reg_idx));
4568 } while (--poll_ms && (txtdh != txtdt));
4571 "Tx Queue %d is not empty when stopping.",
4574 txgbe_dev_save_tx_queue(hw, txq->reg_idx);
4575 wr32m(hw, TXGBE_TXCFG(txq->reg_idx), TXGBE_TXCFG_ENA, 0);
4577 /* Wait until TX Enable bit clear */
4578 poll_ms = RTE_TXGBE_REGISTER_POLL_WAIT_10_MS;
4581 txdctl = rd32(hw, TXGBE_TXCFG(txq->reg_idx));
4582 } while (--poll_ms && (txdctl & TXGBE_TXCFG_ENA));
4584 PMD_INIT_LOG(ERR, "Could not disable Tx Queue %d",
4587 rte_delay_us(RTE_TXGBE_WAIT_100_US);
4588 txgbe_dev_store_tx_queue(hw, txq->reg_idx);
4590 if (txq->ops != NULL) {
4591 txq->ops->release_mbufs(txq);
4592 txq->ops->reset(txq);
4594 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
4600 txgbe_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
4601 struct rte_eth_rxq_info *qinfo)
4603 struct txgbe_rx_queue *rxq;
4605 rxq = dev->data->rx_queues[queue_id];
4607 qinfo->mp = rxq->mb_pool;
4608 qinfo->scattered_rx = dev->data->scattered_rx;
4609 qinfo->nb_desc = rxq->nb_rx_desc;
4611 qinfo->conf.rx_free_thresh = rxq->rx_free_thresh;
4612 qinfo->conf.rx_drop_en = rxq->drop_en;
4613 qinfo->conf.rx_deferred_start = rxq->rx_deferred_start;
4614 qinfo->conf.offloads = rxq->offloads;
4618 txgbe_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
4619 struct rte_eth_txq_info *qinfo)
4621 struct txgbe_tx_queue *txq;
4623 txq = dev->data->tx_queues[queue_id];
4625 qinfo->nb_desc = txq->nb_tx_desc;
4627 qinfo->conf.tx_thresh.pthresh = txq->pthresh;
4628 qinfo->conf.tx_thresh.hthresh = txq->hthresh;
4629 qinfo->conf.tx_thresh.wthresh = txq->wthresh;
4631 qinfo->conf.tx_free_thresh = txq->tx_free_thresh;
4632 qinfo->conf.offloads = txq->offloads;
4633 qinfo->conf.tx_deferred_start = txq->tx_deferred_start;