1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2015-2020
16 #include <rte_byteorder.h>
17 #include <rte_common.h>
18 #include <rte_cycles.h>
20 #include <rte_debug.h>
21 #include <rte_ethdev.h>
22 #include <ethdev_driver.h>
23 #include <rte_security_driver.h>
24 #include <rte_memzone.h>
25 #include <rte_atomic.h>
26 #include <rte_mempool.h>
27 #include <rte_malloc.h>
29 #include <rte_ether.h>
30 #include <rte_prefetch.h>
34 #include <rte_string_fns.h>
35 #include <rte_errno.h>
39 #include "txgbe_logs.h"
40 #include "base/txgbe.h"
41 #include "txgbe_ethdev.h"
42 #include "txgbe_rxtx.h"
44 #ifdef RTE_LIBRTE_IEEE1588
45 #define TXGBE_TX_IEEE1588_TMST PKT_TX_IEEE1588_TMST
47 #define TXGBE_TX_IEEE1588_TMST 0
50 /* Bit Mask to indicate what bits required for building TX context */
51 static const u64 TXGBE_TX_OFFLOAD_MASK = (PKT_TX_IP_CKSUM |
60 PKT_TX_OUTER_IP_CKSUM |
61 #ifdef RTE_LIB_SECURITY
64 TXGBE_TX_IEEE1588_TMST);
66 #define TXGBE_TX_OFFLOAD_NOTSUP_MASK \
67 (PKT_TX_OFFLOAD_MASK ^ TXGBE_TX_OFFLOAD_MASK)
70 * Prefetch a cache line into all cache levels.
72 #define rte_txgbe_prefetch(p) rte_prefetch0(p)
75 txgbe_is_vf(struct rte_eth_dev *dev)
77 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
79 switch (hw->mac.type) {
80 case txgbe_mac_raptor_vf:
87 /*********************************************************************
91 **********************************************************************/
94 * Check for descriptors with their DD bit set and free mbufs.
95 * Return the total number of buffers freed.
97 static __rte_always_inline int
98 txgbe_tx_free_bufs(struct txgbe_tx_queue *txq)
100 struct txgbe_tx_entry *txep;
103 struct rte_mbuf *m, *free[RTE_TXGBE_TX_MAX_FREE_BUF_SZ];
105 /* check DD bit on threshold descriptor */
106 status = txq->tx_ring[txq->tx_next_dd].dw3;
107 if (!(status & rte_cpu_to_le_32(TXGBE_TXD_DD))) {
108 if (txq->nb_tx_free >> 1 < txq->tx_free_thresh)
109 txgbe_set32_masked(txq->tdc_reg_addr,
110 TXGBE_TXCFG_FLUSH, TXGBE_TXCFG_FLUSH);
115 * first buffer to free from S/W ring is at index
116 * tx_next_dd - (tx_free_thresh-1)
118 txep = &txq->sw_ring[txq->tx_next_dd - (txq->tx_free_thresh - 1)];
119 for (i = 0; i < txq->tx_free_thresh; ++i, ++txep) {
120 /* free buffers one at a time */
121 m = rte_pktmbuf_prefree_seg(txep->mbuf);
124 if (unlikely(m == NULL))
127 if (nb_free >= RTE_TXGBE_TX_MAX_FREE_BUF_SZ ||
128 (nb_free > 0 && m->pool != free[0]->pool)) {
129 rte_mempool_put_bulk(free[0]->pool,
130 (void **)free, nb_free);
138 rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free);
140 /* buffers were freed, update counters */
141 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_free_thresh);
142 txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_free_thresh);
143 if (txq->tx_next_dd >= txq->nb_tx_desc)
144 txq->tx_next_dd = (uint16_t)(txq->tx_free_thresh - 1);
146 return txq->tx_free_thresh;
149 /* Populate 4 descriptors with data from 4 mbufs */
151 tx4(volatile struct txgbe_tx_desc *txdp, struct rte_mbuf **pkts)
153 uint64_t buf_dma_addr;
157 for (i = 0; i < 4; ++i, ++txdp, ++pkts) {
158 buf_dma_addr = rte_mbuf_data_iova(*pkts);
159 pkt_len = (*pkts)->data_len;
161 /* write data to descriptor */
162 txdp->qw0 = rte_cpu_to_le_64(buf_dma_addr);
163 txdp->dw2 = cpu_to_le32(TXGBE_TXD_FLAGS |
164 TXGBE_TXD_DATLEN(pkt_len));
165 txdp->dw3 = cpu_to_le32(TXGBE_TXD_PAYLEN(pkt_len));
167 rte_prefetch0(&(*pkts)->pool);
171 /* Populate 1 descriptor with data from 1 mbuf */
173 tx1(volatile struct txgbe_tx_desc *txdp, struct rte_mbuf **pkts)
175 uint64_t buf_dma_addr;
178 buf_dma_addr = rte_mbuf_data_iova(*pkts);
179 pkt_len = (*pkts)->data_len;
181 /* write data to descriptor */
182 txdp->qw0 = cpu_to_le64(buf_dma_addr);
183 txdp->dw2 = cpu_to_le32(TXGBE_TXD_FLAGS |
184 TXGBE_TXD_DATLEN(pkt_len));
185 txdp->dw3 = cpu_to_le32(TXGBE_TXD_PAYLEN(pkt_len));
187 rte_prefetch0(&(*pkts)->pool);
191 * Fill H/W descriptor ring with mbuf data.
192 * Copy mbuf pointers to the S/W ring.
195 txgbe_tx_fill_hw_ring(struct txgbe_tx_queue *txq, struct rte_mbuf **pkts,
198 volatile struct txgbe_tx_desc *txdp = &txq->tx_ring[txq->tx_tail];
199 struct txgbe_tx_entry *txep = &txq->sw_ring[txq->tx_tail];
200 const int N_PER_LOOP = 4;
201 const int N_PER_LOOP_MASK = N_PER_LOOP - 1;
202 int mainpart, leftover;
206 * Process most of the packets in chunks of N pkts. Any
207 * leftover packets will get processed one at a time.
209 mainpart = (nb_pkts & ((uint32_t)~N_PER_LOOP_MASK));
210 leftover = (nb_pkts & ((uint32_t)N_PER_LOOP_MASK));
211 for (i = 0; i < mainpart; i += N_PER_LOOP) {
212 /* Copy N mbuf pointers to the S/W ring */
213 for (j = 0; j < N_PER_LOOP; ++j)
214 (txep + i + j)->mbuf = *(pkts + i + j);
215 tx4(txdp + i, pkts + i);
218 if (unlikely(leftover > 0)) {
219 for (i = 0; i < leftover; ++i) {
220 (txep + mainpart + i)->mbuf = *(pkts + mainpart + i);
221 tx1(txdp + mainpart + i, pkts + mainpart + i);
226 static inline uint16_t
227 tx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
230 struct txgbe_tx_queue *txq = (struct txgbe_tx_queue *)tx_queue;
234 * Begin scanning the H/W ring for done descriptors when the
235 * number of available descriptors drops below tx_free_thresh. For
236 * each done descriptor, free the associated buffer.
238 if (txq->nb_tx_free < txq->tx_free_thresh)
239 txgbe_tx_free_bufs(txq);
241 /* Only use descriptors that are available */
242 nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts);
243 if (unlikely(nb_pkts == 0))
246 /* Use exactly nb_pkts descriptors */
247 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts);
250 * At this point, we know there are enough descriptors in the
251 * ring to transmit all the packets. This assumes that each
252 * mbuf contains a single segment, and that no new offloads
253 * are expected, which would require a new context descriptor.
257 * See if we're going to wrap-around. If so, handle the top
258 * of the descriptor ring first, then do the bottom. If not,
259 * the processing looks just like the "bottom" part anyway...
261 if ((txq->tx_tail + nb_pkts) > txq->nb_tx_desc) {
262 n = (uint16_t)(txq->nb_tx_desc - txq->tx_tail);
263 txgbe_tx_fill_hw_ring(txq, tx_pkts, n);
267 /* Fill H/W descriptor ring with mbuf data */
268 txgbe_tx_fill_hw_ring(txq, tx_pkts + n, (uint16_t)(nb_pkts - n));
269 txq->tx_tail = (uint16_t)(txq->tx_tail + (nb_pkts - n));
272 * Check for wrap-around. This would only happen if we used
273 * up to the last descriptor in the ring, no more, no less.
275 if (txq->tx_tail >= txq->nb_tx_desc)
278 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u",
279 (uint16_t)txq->port_id, (uint16_t)txq->queue_id,
280 (uint16_t)txq->tx_tail, (uint16_t)nb_pkts);
282 /* update tail pointer */
284 txgbe_set32_relaxed(txq->tdt_reg_addr, txq->tx_tail);
290 txgbe_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts,
295 /* Try to transmit at least chunks of TX_MAX_BURST pkts */
296 if (likely(nb_pkts <= RTE_PMD_TXGBE_TX_MAX_BURST))
297 return tx_xmit_pkts(tx_queue, tx_pkts, nb_pkts);
299 /* transmit more than the max burst, in chunks of TX_MAX_BURST */
304 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_TXGBE_TX_MAX_BURST);
305 ret = tx_xmit_pkts(tx_queue, &tx_pkts[nb_tx], n);
306 nb_tx = (uint16_t)(nb_tx + ret);
307 nb_pkts = (uint16_t)(nb_pkts - ret);
316 txgbe_set_xmit_ctx(struct txgbe_tx_queue *txq,
317 volatile struct txgbe_tx_ctx_desc *ctx_txd,
318 uint64_t ol_flags, union txgbe_tx_offload tx_offload,
319 __rte_unused uint64_t *mdata)
321 union txgbe_tx_offload tx_offload_mask;
322 uint32_t type_tucmd_mlhl;
323 uint32_t mss_l4len_idx;
325 uint32_t vlan_macip_lens;
326 uint32_t tunnel_seed;
328 ctx_idx = txq->ctx_curr;
329 tx_offload_mask.data[0] = 0;
330 tx_offload_mask.data[1] = 0;
332 /* Specify which HW CTX to upload. */
333 mss_l4len_idx = TXGBE_TXD_IDX(ctx_idx);
334 type_tucmd_mlhl = TXGBE_TXD_CTXT;
336 tx_offload_mask.ptid |= ~0;
337 type_tucmd_mlhl |= TXGBE_TXD_PTID(tx_offload.ptid);
339 /* check if TCP segmentation required for this packet */
340 if (ol_flags & PKT_TX_TCP_SEG) {
341 tx_offload_mask.l2_len |= ~0;
342 tx_offload_mask.l3_len |= ~0;
343 tx_offload_mask.l4_len |= ~0;
344 tx_offload_mask.tso_segsz |= ~0;
345 mss_l4len_idx |= TXGBE_TXD_MSS(tx_offload.tso_segsz);
346 mss_l4len_idx |= TXGBE_TXD_L4LEN(tx_offload.l4_len);
347 } else { /* no TSO, check if hardware checksum is needed */
348 if (ol_flags & PKT_TX_IP_CKSUM) {
349 tx_offload_mask.l2_len |= ~0;
350 tx_offload_mask.l3_len |= ~0;
353 switch (ol_flags & PKT_TX_L4_MASK) {
354 case PKT_TX_UDP_CKSUM:
356 TXGBE_TXD_L4LEN(sizeof(struct rte_udp_hdr));
357 tx_offload_mask.l2_len |= ~0;
358 tx_offload_mask.l3_len |= ~0;
360 case PKT_TX_TCP_CKSUM:
362 TXGBE_TXD_L4LEN(sizeof(struct rte_tcp_hdr));
363 tx_offload_mask.l2_len |= ~0;
364 tx_offload_mask.l3_len |= ~0;
366 case PKT_TX_SCTP_CKSUM:
368 TXGBE_TXD_L4LEN(sizeof(struct rte_sctp_hdr));
369 tx_offload_mask.l2_len |= ~0;
370 tx_offload_mask.l3_len |= ~0;
377 vlan_macip_lens = TXGBE_TXD_IPLEN(tx_offload.l3_len >> 1);
379 if (ol_flags & PKT_TX_TUNNEL_MASK) {
380 tx_offload_mask.outer_tun_len |= ~0;
381 tx_offload_mask.outer_l2_len |= ~0;
382 tx_offload_mask.outer_l3_len |= ~0;
383 tx_offload_mask.l2_len |= ~0;
384 tunnel_seed = TXGBE_TXD_ETUNLEN(tx_offload.outer_tun_len >> 1);
385 tunnel_seed |= TXGBE_TXD_EIPLEN(tx_offload.outer_l3_len >> 2);
387 switch (ol_flags & PKT_TX_TUNNEL_MASK) {
388 case PKT_TX_TUNNEL_IPIP:
389 /* for non UDP / GRE tunneling, set to 0b */
391 case PKT_TX_TUNNEL_VXLAN:
392 case PKT_TX_TUNNEL_GENEVE:
393 tunnel_seed |= TXGBE_TXD_ETYPE_UDP;
395 case PKT_TX_TUNNEL_GRE:
396 tunnel_seed |= TXGBE_TXD_ETYPE_GRE;
399 PMD_TX_LOG(ERR, "Tunnel type not supported");
402 vlan_macip_lens |= TXGBE_TXD_MACLEN(tx_offload.outer_l2_len);
405 vlan_macip_lens |= TXGBE_TXD_MACLEN(tx_offload.l2_len);
408 if (ol_flags & PKT_TX_VLAN_PKT) {
409 tx_offload_mask.vlan_tci |= ~0;
410 vlan_macip_lens |= TXGBE_TXD_VLAN(tx_offload.vlan_tci);
413 #ifdef RTE_LIB_SECURITY
414 if (ol_flags & PKT_TX_SEC_OFFLOAD) {
415 union txgbe_crypto_tx_desc_md *md =
416 (union txgbe_crypto_tx_desc_md *)mdata;
417 tunnel_seed |= TXGBE_TXD_IPSEC_SAIDX(md->sa_idx);
418 type_tucmd_mlhl |= md->enc ?
419 (TXGBE_TXD_IPSEC_ESP | TXGBE_TXD_IPSEC_ESPENC) : 0;
420 type_tucmd_mlhl |= TXGBE_TXD_IPSEC_ESPLEN(md->pad_len);
421 tx_offload_mask.sa_idx |= ~0;
422 tx_offload_mask.sec_pad_len |= ~0;
426 txq->ctx_cache[ctx_idx].flags = ol_flags;
427 txq->ctx_cache[ctx_idx].tx_offload.data[0] =
428 tx_offload_mask.data[0] & tx_offload.data[0];
429 txq->ctx_cache[ctx_idx].tx_offload.data[1] =
430 tx_offload_mask.data[1] & tx_offload.data[1];
431 txq->ctx_cache[ctx_idx].tx_offload_mask = tx_offload_mask;
433 ctx_txd->dw0 = rte_cpu_to_le_32(vlan_macip_lens);
434 ctx_txd->dw1 = rte_cpu_to_le_32(tunnel_seed);
435 ctx_txd->dw2 = rte_cpu_to_le_32(type_tucmd_mlhl);
436 ctx_txd->dw3 = rte_cpu_to_le_32(mss_l4len_idx);
440 * Check which hardware context can be used. Use the existing match
441 * or create a new context descriptor.
443 static inline uint32_t
444 what_ctx_update(struct txgbe_tx_queue *txq, uint64_t flags,
445 union txgbe_tx_offload tx_offload)
447 /* If match with the current used context */
448 if (likely(txq->ctx_cache[txq->ctx_curr].flags == flags &&
449 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] ==
450 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0]
451 & tx_offload.data[0])) &&
452 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] ==
453 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1]
454 & tx_offload.data[1]))))
455 return txq->ctx_curr;
457 /* What if match with the next context */
459 if (likely(txq->ctx_cache[txq->ctx_curr].flags == flags &&
460 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] ==
461 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0]
462 & tx_offload.data[0])) &&
463 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] ==
464 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1]
465 & tx_offload.data[1]))))
466 return txq->ctx_curr;
468 /* Mismatch, use the previous context */
469 return TXGBE_CTX_NUM;
472 static inline uint32_t
473 tx_desc_cksum_flags_to_olinfo(uint64_t ol_flags)
477 if ((ol_flags & PKT_TX_L4_MASK) != PKT_TX_L4_NO_CKSUM) {
479 tmp |= TXGBE_TXD_L4CS;
481 if (ol_flags & PKT_TX_IP_CKSUM) {
483 tmp |= TXGBE_TXD_IPCS;
485 if (ol_flags & PKT_TX_OUTER_IP_CKSUM) {
487 tmp |= TXGBE_TXD_EIPCS;
489 if (ol_flags & PKT_TX_TCP_SEG) {
491 /* implies IPv4 cksum */
492 if (ol_flags & PKT_TX_IPV4)
493 tmp |= TXGBE_TXD_IPCS;
494 tmp |= TXGBE_TXD_L4CS;
496 if (ol_flags & PKT_TX_VLAN_PKT)
502 static inline uint32_t
503 tx_desc_ol_flags_to_cmdtype(uint64_t ol_flags)
505 uint32_t cmdtype = 0;
507 if (ol_flags & PKT_TX_VLAN_PKT)
508 cmdtype |= TXGBE_TXD_VLE;
509 if (ol_flags & PKT_TX_TCP_SEG)
510 cmdtype |= TXGBE_TXD_TSE;
511 if (ol_flags & PKT_TX_MACSEC)
512 cmdtype |= TXGBE_TXD_LINKSEC;
516 static inline uint8_t
517 tx_desc_ol_flags_to_ptid(uint64_t oflags, uint32_t ptype)
522 return txgbe_encode_ptype(ptype);
524 /* Only support flags in TXGBE_TX_OFFLOAD_MASK */
525 tun = !!(oflags & PKT_TX_TUNNEL_MASK);
528 ptype = RTE_PTYPE_L2_ETHER;
529 if (oflags & PKT_TX_VLAN)
530 ptype |= RTE_PTYPE_L2_ETHER_VLAN;
533 if (oflags & (PKT_TX_OUTER_IPV4 | PKT_TX_OUTER_IP_CKSUM))
534 ptype |= RTE_PTYPE_L3_IPV4;
535 else if (oflags & (PKT_TX_OUTER_IPV6))
536 ptype |= RTE_PTYPE_L3_IPV6;
538 if (oflags & (PKT_TX_IPV4 | PKT_TX_IP_CKSUM))
539 ptype |= (tun ? RTE_PTYPE_INNER_L3_IPV4 : RTE_PTYPE_L3_IPV4);
540 else if (oflags & (PKT_TX_IPV6))
541 ptype |= (tun ? RTE_PTYPE_INNER_L3_IPV6 : RTE_PTYPE_L3_IPV6);
544 switch (oflags & (PKT_TX_L4_MASK)) {
545 case PKT_TX_TCP_CKSUM:
546 ptype |= (tun ? RTE_PTYPE_INNER_L4_TCP : RTE_PTYPE_L4_TCP);
548 case PKT_TX_UDP_CKSUM:
549 ptype |= (tun ? RTE_PTYPE_INNER_L4_UDP : RTE_PTYPE_L4_UDP);
551 case PKT_TX_SCTP_CKSUM:
552 ptype |= (tun ? RTE_PTYPE_INNER_L4_SCTP : RTE_PTYPE_L4_SCTP);
556 if (oflags & PKT_TX_TCP_SEG)
557 ptype |= (tun ? RTE_PTYPE_INNER_L4_TCP : RTE_PTYPE_L4_TCP);
560 switch (oflags & PKT_TX_TUNNEL_MASK) {
561 case PKT_TX_TUNNEL_VXLAN:
562 ptype |= RTE_PTYPE_L2_ETHER |
564 RTE_PTYPE_TUNNEL_VXLAN;
565 ptype |= RTE_PTYPE_INNER_L2_ETHER;
567 case PKT_TX_TUNNEL_GRE:
568 ptype |= RTE_PTYPE_L2_ETHER |
570 RTE_PTYPE_TUNNEL_GRE;
571 ptype |= RTE_PTYPE_INNER_L2_ETHER;
573 case PKT_TX_TUNNEL_GENEVE:
574 ptype |= RTE_PTYPE_L2_ETHER |
576 RTE_PTYPE_TUNNEL_GENEVE;
577 ptype |= RTE_PTYPE_INNER_L2_ETHER;
579 case PKT_TX_TUNNEL_VXLAN_GPE:
580 ptype |= RTE_PTYPE_L2_ETHER |
582 RTE_PTYPE_TUNNEL_VXLAN_GPE;
583 ptype |= RTE_PTYPE_INNER_L2_ETHER;
585 case PKT_TX_TUNNEL_IPIP:
586 case PKT_TX_TUNNEL_IP:
587 ptype |= RTE_PTYPE_L2_ETHER |
593 return txgbe_encode_ptype(ptype);
596 #ifndef DEFAULT_TX_FREE_THRESH
597 #define DEFAULT_TX_FREE_THRESH 32
600 /* Reset transmit descriptors after they have been used */
602 txgbe_xmit_cleanup(struct txgbe_tx_queue *txq)
604 struct txgbe_tx_entry *sw_ring = txq->sw_ring;
605 volatile struct txgbe_tx_desc *txr = txq->tx_ring;
606 uint16_t last_desc_cleaned = txq->last_desc_cleaned;
607 uint16_t nb_tx_desc = txq->nb_tx_desc;
608 uint16_t desc_to_clean_to;
609 uint16_t nb_tx_to_clean;
612 /* Determine the last descriptor needing to be cleaned */
613 desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->tx_free_thresh);
614 if (desc_to_clean_to >= nb_tx_desc)
615 desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc);
617 /* Check to make sure the last descriptor to clean is done */
618 desc_to_clean_to = sw_ring[desc_to_clean_to].last_id;
619 status = txr[desc_to_clean_to].dw3;
620 if (!(status & rte_cpu_to_le_32(TXGBE_TXD_DD))) {
621 PMD_TX_FREE_LOG(DEBUG,
622 "TX descriptor %4u is not done"
623 "(port=%d queue=%d)",
625 txq->port_id, txq->queue_id);
626 if (txq->nb_tx_free >> 1 < txq->tx_free_thresh)
627 txgbe_set32_masked(txq->tdc_reg_addr,
628 TXGBE_TXCFG_FLUSH, TXGBE_TXCFG_FLUSH);
629 /* Failed to clean any descriptors, better luck next time */
633 /* Figure out how many descriptors will be cleaned */
634 if (last_desc_cleaned > desc_to_clean_to)
635 nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) +
638 nb_tx_to_clean = (uint16_t)(desc_to_clean_to -
641 PMD_TX_FREE_LOG(DEBUG,
642 "Cleaning %4u TX descriptors: %4u to %4u "
643 "(port=%d queue=%d)",
644 nb_tx_to_clean, last_desc_cleaned, desc_to_clean_to,
645 txq->port_id, txq->queue_id);
648 * The last descriptor to clean is done, so that means all the
649 * descriptors from the last descriptor that was cleaned
650 * up to the last descriptor with the RS bit set
651 * are done. Only reset the threshold descriptor.
653 txr[desc_to_clean_to].dw3 = 0;
655 /* Update the txq to reflect the last descriptor that was cleaned */
656 txq->last_desc_cleaned = desc_to_clean_to;
657 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + nb_tx_to_clean);
663 static inline uint8_t
664 txgbe_get_tun_len(struct rte_mbuf *mbuf)
666 struct txgbe_genevehdr genevehdr;
667 const struct txgbe_genevehdr *gh;
670 switch (mbuf->ol_flags & PKT_TX_TUNNEL_MASK) {
671 case PKT_TX_TUNNEL_IPIP:
674 case PKT_TX_TUNNEL_VXLAN:
675 case PKT_TX_TUNNEL_VXLAN_GPE:
676 tun_len = sizeof(struct txgbe_udphdr)
677 + sizeof(struct txgbe_vxlanhdr);
679 case PKT_TX_TUNNEL_GRE:
680 tun_len = sizeof(struct txgbe_nvgrehdr);
682 case PKT_TX_TUNNEL_GENEVE:
683 gh = rte_pktmbuf_read(mbuf,
684 mbuf->outer_l2_len + mbuf->outer_l3_len,
685 sizeof(genevehdr), &genevehdr);
686 tun_len = sizeof(struct txgbe_udphdr)
687 + sizeof(struct txgbe_genevehdr)
688 + (gh->opt_len << 2);
698 txgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
701 struct txgbe_tx_queue *txq;
702 struct txgbe_tx_entry *sw_ring;
703 struct txgbe_tx_entry *txe, *txn;
704 volatile struct txgbe_tx_desc *txr;
705 volatile struct txgbe_tx_desc *txd;
706 struct rte_mbuf *tx_pkt;
707 struct rte_mbuf *m_seg;
708 uint64_t buf_dma_addr;
709 uint32_t olinfo_status;
710 uint32_t cmd_type_len;
721 union txgbe_tx_offload tx_offload;
722 #ifdef RTE_LIB_SECURITY
726 tx_offload.data[0] = 0;
727 tx_offload.data[1] = 0;
729 sw_ring = txq->sw_ring;
731 tx_id = txq->tx_tail;
732 txe = &sw_ring[tx_id];
734 /* Determine if the descriptor ring needs to be cleaned. */
735 if (txq->nb_tx_free < txq->tx_free_thresh)
736 txgbe_xmit_cleanup(txq);
738 rte_prefetch0(&txe->mbuf->pool);
741 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
744 pkt_len = tx_pkt->pkt_len;
747 * Determine how many (if any) context descriptors
748 * are needed for offload functionality.
750 ol_flags = tx_pkt->ol_flags;
751 #ifdef RTE_LIB_SECURITY
752 use_ipsec = txq->using_ipsec && (ol_flags & PKT_TX_SEC_OFFLOAD);
755 /* If hardware offload required */
756 tx_ol_req = ol_flags & TXGBE_TX_OFFLOAD_MASK;
758 tx_offload.ptid = tx_desc_ol_flags_to_ptid(tx_ol_req,
759 tx_pkt->packet_type);
760 tx_offload.l2_len = tx_pkt->l2_len;
761 tx_offload.l3_len = tx_pkt->l3_len;
762 tx_offload.l4_len = tx_pkt->l4_len;
763 tx_offload.vlan_tci = tx_pkt->vlan_tci;
764 tx_offload.tso_segsz = tx_pkt->tso_segsz;
765 tx_offload.outer_l2_len = tx_pkt->outer_l2_len;
766 tx_offload.outer_l3_len = tx_pkt->outer_l3_len;
767 tx_offload.outer_tun_len = txgbe_get_tun_len(tx_pkt);
769 #ifdef RTE_LIB_SECURITY
771 union txgbe_crypto_tx_desc_md *ipsec_mdata =
772 (union txgbe_crypto_tx_desc_md *)
773 rte_security_dynfield(tx_pkt);
774 tx_offload.sa_idx = ipsec_mdata->sa_idx;
775 tx_offload.sec_pad_len = ipsec_mdata->pad_len;
779 /* If new context need be built or reuse the exist ctx*/
780 ctx = what_ctx_update(txq, tx_ol_req, tx_offload);
781 /* Only allocate context descriptor if required */
782 new_ctx = (ctx == TXGBE_CTX_NUM);
787 * Keep track of how many descriptors are used this loop
788 * This will always be the number of segments + the number of
789 * Context descriptors required to transmit the packet
791 nb_used = (uint16_t)(tx_pkt->nb_segs + new_ctx);
794 * The number of descriptors that must be allocated for a
795 * packet is the number of segments of that packet, plus 1
796 * Context Descriptor for the hardware offload, if any.
797 * Determine the last TX descriptor to allocate in the TX ring
798 * for the packet, starting from the current position (tx_id)
801 tx_last = (uint16_t)(tx_id + nb_used - 1);
804 if (tx_last >= txq->nb_tx_desc)
805 tx_last = (uint16_t)(tx_last - txq->nb_tx_desc);
807 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u pktlen=%u"
808 " tx_first=%u tx_last=%u",
809 (uint16_t)txq->port_id,
810 (uint16_t)txq->queue_id,
816 * Make sure there are enough TX descriptors available to
817 * transmit the entire packet.
818 * nb_used better be less than or equal to txq->tx_free_thresh
820 if (nb_used > txq->nb_tx_free) {
821 PMD_TX_FREE_LOG(DEBUG,
822 "Not enough free TX descriptors "
823 "nb_used=%4u nb_free=%4u "
824 "(port=%d queue=%d)",
825 nb_used, txq->nb_tx_free,
826 txq->port_id, txq->queue_id);
828 if (txgbe_xmit_cleanup(txq) != 0) {
829 /* Could not clean any descriptors */
835 /* nb_used better be <= txq->tx_free_thresh */
836 if (unlikely(nb_used > txq->tx_free_thresh)) {
837 PMD_TX_FREE_LOG(DEBUG,
838 "The number of descriptors needed to "
839 "transmit the packet exceeds the "
840 "RS bit threshold. This will impact "
842 "nb_used=%4u nb_free=%4u "
843 "tx_free_thresh=%4u. "
844 "(port=%d queue=%d)",
845 nb_used, txq->nb_tx_free,
847 txq->port_id, txq->queue_id);
849 * Loop here until there are enough TX
850 * descriptors or until the ring cannot be
853 while (nb_used > txq->nb_tx_free) {
854 if (txgbe_xmit_cleanup(txq) != 0) {
856 * Could not clean any
868 * By now there are enough free TX descriptors to transmit
873 * Set common flags of all TX Data Descriptors.
875 * The following bits must be set in all Data Descriptors:
876 * - TXGBE_TXD_DTYP_DATA
877 * - TXGBE_TXD_DCMD_DEXT
879 * The following bits must be set in the first Data Descriptor
880 * and are ignored in the other ones:
881 * - TXGBE_TXD_DCMD_IFCS
882 * - TXGBE_TXD_MAC_1588
883 * - TXGBE_TXD_DCMD_VLE
885 * The following bits must only be set in the last Data
887 * - TXGBE_TXD_CMD_EOP
889 * The following bits can be set in any Data Descriptor, but
890 * are only set in the last Data Descriptor:
893 cmd_type_len = TXGBE_TXD_FCS;
895 #ifdef RTE_LIBRTE_IEEE1588
896 if (ol_flags & PKT_TX_IEEE1588_TMST)
897 cmd_type_len |= TXGBE_TXD_1588;
902 if (ol_flags & PKT_TX_TCP_SEG) {
903 /* when TSO is on, paylen in descriptor is the
904 * not the packet len but the tcp payload len
906 pkt_len -= (tx_offload.l2_len +
907 tx_offload.l3_len + tx_offload.l4_len);
909 (tx_pkt->ol_flags & PKT_TX_TUNNEL_MASK)
910 ? tx_offload.outer_l2_len +
911 tx_offload.outer_l3_len : 0;
915 * Setup the TX Advanced Context Descriptor if required
918 volatile struct txgbe_tx_ctx_desc *ctx_txd;
920 ctx_txd = (volatile struct txgbe_tx_ctx_desc *)
923 txn = &sw_ring[txe->next_id];
924 rte_prefetch0(&txn->mbuf->pool);
926 if (txe->mbuf != NULL) {
927 rte_pktmbuf_free_seg(txe->mbuf);
931 txgbe_set_xmit_ctx(txq, ctx_txd, tx_ol_req,
933 rte_security_dynfield(tx_pkt));
935 txe->last_id = tx_last;
936 tx_id = txe->next_id;
941 * Setup the TX Advanced Data Descriptor,
942 * This path will go through
943 * whatever new/reuse the context descriptor
945 cmd_type_len |= tx_desc_ol_flags_to_cmdtype(ol_flags);
947 tx_desc_cksum_flags_to_olinfo(ol_flags);
948 olinfo_status |= TXGBE_TXD_IDX(ctx);
951 olinfo_status |= TXGBE_TXD_PAYLEN(pkt_len);
952 #ifdef RTE_LIB_SECURITY
954 olinfo_status |= TXGBE_TXD_IPSEC;
960 txn = &sw_ring[txe->next_id];
961 rte_prefetch0(&txn->mbuf->pool);
963 if (txe->mbuf != NULL)
964 rte_pktmbuf_free_seg(txe->mbuf);
968 * Set up Transmit Data Descriptor.
970 slen = m_seg->data_len;
971 buf_dma_addr = rte_mbuf_data_iova(m_seg);
972 txd->qw0 = rte_cpu_to_le_64(buf_dma_addr);
973 txd->dw2 = rte_cpu_to_le_32(cmd_type_len | slen);
974 txd->dw3 = rte_cpu_to_le_32(olinfo_status);
975 txe->last_id = tx_last;
976 tx_id = txe->next_id;
979 } while (m_seg != NULL);
982 * The last packet data descriptor needs End Of Packet (EOP)
984 cmd_type_len |= TXGBE_TXD_EOP;
985 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_used);
987 txd->dw2 |= rte_cpu_to_le_32(cmd_type_len);
995 * Set the Transmit Descriptor Tail (TDT)
997 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u",
998 (uint16_t)txq->port_id, (uint16_t)txq->queue_id,
999 (uint16_t)tx_id, (uint16_t)nb_tx);
1000 txgbe_set32_relaxed(txq->tdt_reg_addr, tx_id);
1001 txq->tx_tail = tx_id;
1006 /*********************************************************************
1010 **********************************************************************/
1012 txgbe_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
1017 struct txgbe_tx_queue *txq = (struct txgbe_tx_queue *)tx_queue;
1019 for (i = 0; i < nb_pkts; i++) {
1021 ol_flags = m->ol_flags;
1024 * Check if packet meets requirements for number of segments
1026 * NOTE: for txgbe it's always (40 - WTHRESH) for both TSO and
1030 if (m->nb_segs > TXGBE_TX_MAX_SEG - txq->wthresh) {
1031 rte_errno = -EINVAL;
1035 if (ol_flags & TXGBE_TX_OFFLOAD_NOTSUP_MASK) {
1036 rte_errno = -ENOTSUP;
1040 #ifdef RTE_LIBRTE_ETHDEV_DEBUG
1041 ret = rte_validate_tx_offload(m);
1047 ret = rte_net_intel_cksum_prepare(m);
1057 /*********************************************************************
1061 **********************************************************************/
1062 /* @note: fix txgbe_dev_supported_ptypes_get() if any change here. */
1063 static inline uint32_t
1064 txgbe_rxd_pkt_info_to_pkt_type(uint32_t pkt_info, uint16_t ptid_mask)
1066 uint16_t ptid = TXGBE_RXD_PTID(pkt_info);
1070 return txgbe_decode_ptype(ptid);
1073 static inline uint64_t
1074 txgbe_rxd_pkt_info_to_pkt_flags(uint32_t pkt_info)
1076 static uint64_t ip_rss_types_map[16] __rte_cache_aligned = {
1077 0, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH,
1078 0, PKT_RX_RSS_HASH, 0, PKT_RX_RSS_HASH,
1079 PKT_RX_RSS_HASH, 0, 0, 0,
1080 0, 0, 0, PKT_RX_FDIR,
1082 #ifdef RTE_LIBRTE_IEEE1588
1083 static uint64_t ip_pkt_etqf_map[8] = {
1084 0, 0, 0, PKT_RX_IEEE1588_PTP,
1087 int etfid = txgbe_etflt_id(TXGBE_RXD_PTID(pkt_info));
1088 if (likely(-1 != etfid))
1089 return ip_pkt_etqf_map[etfid] |
1090 ip_rss_types_map[TXGBE_RXD_RSSTYPE(pkt_info)];
1092 return ip_rss_types_map[TXGBE_RXD_RSSTYPE(pkt_info)];
1094 return ip_rss_types_map[TXGBE_RXD_RSSTYPE(pkt_info)];
1098 static inline uint64_t
1099 rx_desc_status_to_pkt_flags(uint32_t rx_status, uint64_t vlan_flags)
1104 * Check if VLAN present only.
1105 * Do not check whether L3/L4 rx checksum done by NIC or not,
1106 * That can be found from rte_eth_rxmode.offloads flag
1108 pkt_flags = (rx_status & TXGBE_RXD_STAT_VLAN &&
1109 vlan_flags & PKT_RX_VLAN_STRIPPED)
1112 #ifdef RTE_LIBRTE_IEEE1588
1113 if (rx_status & TXGBE_RXD_STAT_1588)
1114 pkt_flags = pkt_flags | PKT_RX_IEEE1588_TMST;
1119 static inline uint64_t
1120 rx_desc_error_to_pkt_flags(uint32_t rx_status)
1122 uint64_t pkt_flags = 0;
1124 /* checksum offload can't be disabled */
1125 if (rx_status & TXGBE_RXD_STAT_IPCS) {
1126 pkt_flags |= (rx_status & TXGBE_RXD_ERR_IPCS
1127 ? PKT_RX_IP_CKSUM_BAD : PKT_RX_IP_CKSUM_GOOD);
1130 if (rx_status & TXGBE_RXD_STAT_L4CS) {
1131 pkt_flags |= (rx_status & TXGBE_RXD_ERR_L4CS
1132 ? PKT_RX_L4_CKSUM_BAD : PKT_RX_L4_CKSUM_GOOD);
1135 if (rx_status & TXGBE_RXD_STAT_EIPCS &&
1136 rx_status & TXGBE_RXD_ERR_EIPCS) {
1137 pkt_flags |= PKT_RX_EIP_CKSUM_BAD;
1140 #ifdef RTE_LIB_SECURITY
1141 if (rx_status & TXGBE_RXD_STAT_SECP) {
1142 pkt_flags |= PKT_RX_SEC_OFFLOAD;
1143 if (rx_status & TXGBE_RXD_ERR_SECERR)
1144 pkt_flags |= PKT_RX_SEC_OFFLOAD_FAILED;
1152 * LOOK_AHEAD defines how many desc statuses to check beyond the
1153 * current descriptor.
1154 * It must be a pound define for optimal performance.
1155 * Do not change the value of LOOK_AHEAD, as the txgbe_rx_scan_hw_ring
1156 * function only works with LOOK_AHEAD=8.
1158 #define LOOK_AHEAD 8
1159 #if (LOOK_AHEAD != 8)
1160 #error "PMD TXGBE: LOOK_AHEAD must be 8\n"
1163 txgbe_rx_scan_hw_ring(struct txgbe_rx_queue *rxq)
1165 volatile struct txgbe_rx_desc *rxdp;
1166 struct txgbe_rx_entry *rxep;
1167 struct rte_mbuf *mb;
1171 uint32_t s[LOOK_AHEAD];
1172 uint32_t pkt_info[LOOK_AHEAD];
1173 int i, j, nb_rx = 0;
1176 /* get references to current descriptor and S/W ring entry */
1177 rxdp = &rxq->rx_ring[rxq->rx_tail];
1178 rxep = &rxq->sw_ring[rxq->rx_tail];
1180 status = rxdp->qw1.lo.status;
1181 /* check to make sure there is at least 1 packet to receive */
1182 if (!(status & rte_cpu_to_le_32(TXGBE_RXD_STAT_DD)))
1186 * Scan LOOK_AHEAD descriptors at a time to determine which descriptors
1187 * reference packets that are ready to be received.
1189 for (i = 0; i < RTE_PMD_TXGBE_RX_MAX_BURST;
1190 i += LOOK_AHEAD, rxdp += LOOK_AHEAD, rxep += LOOK_AHEAD) {
1191 /* Read desc statuses backwards to avoid race condition */
1192 for (j = 0; j < LOOK_AHEAD; j++)
1193 s[j] = rte_le_to_cpu_32(rxdp[j].qw1.lo.status);
1195 rte_atomic_thread_fence(__ATOMIC_ACQUIRE);
1197 /* Compute how many status bits were set */
1198 for (nb_dd = 0; nb_dd < LOOK_AHEAD &&
1199 (s[nb_dd] & TXGBE_RXD_STAT_DD); nb_dd++)
1202 for (j = 0; j < nb_dd; j++)
1203 pkt_info[j] = rte_le_to_cpu_32(rxdp[j].qw0.dw0);
1207 /* Translate descriptor info to mbuf format */
1208 for (j = 0; j < nb_dd; ++j) {
1210 pkt_len = rte_le_to_cpu_16(rxdp[j].qw1.hi.len) -
1212 mb->data_len = pkt_len;
1213 mb->pkt_len = pkt_len;
1214 mb->vlan_tci = rte_le_to_cpu_16(rxdp[j].qw1.hi.tag);
1216 /* convert descriptor fields to rte mbuf flags */
1217 pkt_flags = rx_desc_status_to_pkt_flags(s[j],
1219 pkt_flags |= rx_desc_error_to_pkt_flags(s[j]);
1221 txgbe_rxd_pkt_info_to_pkt_flags(pkt_info[j]);
1222 mb->ol_flags = pkt_flags;
1224 txgbe_rxd_pkt_info_to_pkt_type(pkt_info[j],
1225 rxq->pkt_type_mask);
1227 if (likely(pkt_flags & PKT_RX_RSS_HASH))
1229 rte_le_to_cpu_32(rxdp[j].qw0.dw1);
1230 else if (pkt_flags & PKT_RX_FDIR) {
1231 mb->hash.fdir.hash =
1232 rte_le_to_cpu_16(rxdp[j].qw0.hi.csum) &
1233 TXGBE_ATR_HASH_MASK;
1235 rte_le_to_cpu_16(rxdp[j].qw0.hi.ipid);
1239 /* Move mbuf pointers from the S/W ring to the stage */
1240 for (j = 0; j < LOOK_AHEAD; ++j)
1241 rxq->rx_stage[i + j] = rxep[j].mbuf;
1243 /* stop if all requested packets could not be received */
1244 if (nb_dd != LOOK_AHEAD)
1248 /* clear software ring entries so we can cleanup correctly */
1249 for (i = 0; i < nb_rx; ++i)
1250 rxq->sw_ring[rxq->rx_tail + i].mbuf = NULL;
1256 txgbe_rx_alloc_bufs(struct txgbe_rx_queue *rxq, bool reset_mbuf)
1258 volatile struct txgbe_rx_desc *rxdp;
1259 struct txgbe_rx_entry *rxep;
1260 struct rte_mbuf *mb;
1265 /* allocate buffers in bulk directly into the S/W ring */
1266 alloc_idx = rxq->rx_free_trigger - (rxq->rx_free_thresh - 1);
1267 rxep = &rxq->sw_ring[alloc_idx];
1268 diag = rte_mempool_get_bulk(rxq->mb_pool, (void *)rxep,
1269 rxq->rx_free_thresh);
1270 if (unlikely(diag != 0))
1273 rxdp = &rxq->rx_ring[alloc_idx];
1274 for (i = 0; i < rxq->rx_free_thresh; ++i) {
1275 /* populate the static rte mbuf fields */
1278 mb->port = rxq->port_id;
1280 rte_mbuf_refcnt_set(mb, 1);
1281 mb->data_off = RTE_PKTMBUF_HEADROOM;
1283 /* populate the descriptors */
1284 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mb));
1285 TXGBE_RXD_HDRADDR(&rxdp[i], 0);
1286 TXGBE_RXD_PKTADDR(&rxdp[i], dma_addr);
1289 /* update state of internal queue structure */
1290 rxq->rx_free_trigger = rxq->rx_free_trigger + rxq->rx_free_thresh;
1291 if (rxq->rx_free_trigger >= rxq->nb_rx_desc)
1292 rxq->rx_free_trigger = rxq->rx_free_thresh - 1;
1298 static inline uint16_t
1299 txgbe_rx_fill_from_stage(struct txgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
1302 struct rte_mbuf **stage = &rxq->rx_stage[rxq->rx_next_avail];
1305 /* how many packets are ready to return? */
1306 nb_pkts = (uint16_t)RTE_MIN(nb_pkts, rxq->rx_nb_avail);
1308 /* copy mbuf pointers to the application's packet list */
1309 for (i = 0; i < nb_pkts; ++i)
1310 rx_pkts[i] = stage[i];
1312 /* update internal queue state */
1313 rxq->rx_nb_avail = (uint16_t)(rxq->rx_nb_avail - nb_pkts);
1314 rxq->rx_next_avail = (uint16_t)(rxq->rx_next_avail + nb_pkts);
1319 static inline uint16_t
1320 txgbe_rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
1323 struct txgbe_rx_queue *rxq = (struct txgbe_rx_queue *)rx_queue;
1324 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id];
1327 /* Any previously recv'd pkts will be returned from the Rx stage */
1328 if (rxq->rx_nb_avail)
1329 return txgbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
1331 /* Scan the H/W ring for packets to receive */
1332 nb_rx = (uint16_t)txgbe_rx_scan_hw_ring(rxq);
1334 /* update internal queue state */
1335 rxq->rx_next_avail = 0;
1336 rxq->rx_nb_avail = nb_rx;
1337 rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_rx);
1339 /* if required, allocate new buffers to replenish descriptors */
1340 if (rxq->rx_tail > rxq->rx_free_trigger) {
1341 uint16_t cur_free_trigger = rxq->rx_free_trigger;
1343 if (txgbe_rx_alloc_bufs(rxq, true) != 0) {
1346 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u "
1347 "queue_id=%u", (uint16_t)rxq->port_id,
1348 (uint16_t)rxq->queue_id);
1350 dev->data->rx_mbuf_alloc_failed +=
1351 rxq->rx_free_thresh;
1354 * Need to rewind any previous receives if we cannot
1355 * allocate new buffers to replenish the old ones.
1357 rxq->rx_nb_avail = 0;
1358 rxq->rx_tail = (uint16_t)(rxq->rx_tail - nb_rx);
1359 for (i = 0, j = rxq->rx_tail; i < nb_rx; ++i, ++j)
1360 rxq->sw_ring[j].mbuf = rxq->rx_stage[i];
1365 /* update tail pointer */
1367 txgbe_set32_relaxed(rxq->rdt_reg_addr, cur_free_trigger);
1370 if (rxq->rx_tail >= rxq->nb_rx_desc)
1373 /* received any packets this loop? */
1374 if (rxq->rx_nb_avail)
1375 return txgbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
1380 /* split requests into chunks of size RTE_PMD_TXGBE_RX_MAX_BURST */
1382 txgbe_recv_pkts_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
1387 if (unlikely(nb_pkts == 0))
1390 if (likely(nb_pkts <= RTE_PMD_TXGBE_RX_MAX_BURST))
1391 return txgbe_rx_recv_pkts(rx_queue, rx_pkts, nb_pkts);
1393 /* request is relatively large, chunk it up */
1398 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_TXGBE_RX_MAX_BURST);
1399 ret = txgbe_rx_recv_pkts(rx_queue, &rx_pkts[nb_rx], n);
1400 nb_rx = (uint16_t)(nb_rx + ret);
1401 nb_pkts = (uint16_t)(nb_pkts - ret);
1410 txgbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
1413 struct txgbe_rx_queue *rxq;
1414 volatile struct txgbe_rx_desc *rx_ring;
1415 volatile struct txgbe_rx_desc *rxdp;
1416 struct txgbe_rx_entry *sw_ring;
1417 struct txgbe_rx_entry *rxe;
1418 struct rte_mbuf *rxm;
1419 struct rte_mbuf *nmb;
1420 struct txgbe_rx_desc rxd;
1433 rx_id = rxq->rx_tail;
1434 rx_ring = rxq->rx_ring;
1435 sw_ring = rxq->sw_ring;
1436 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id];
1437 while (nb_rx < nb_pkts) {
1439 * The order of operations here is important as the DD status
1440 * bit must not be read after any other descriptor fields.
1441 * rx_ring and rxdp are pointing to volatile data so the order
1442 * of accesses cannot be reordered by the compiler. If they were
1443 * not volatile, they could be reordered which could lead to
1444 * using invalid descriptor fields when read from rxd.
1446 rxdp = &rx_ring[rx_id];
1447 staterr = rxdp->qw1.lo.status;
1448 if (!(staterr & rte_cpu_to_le_32(TXGBE_RXD_STAT_DD)))
1455 * If the TXGBE_RXD_STAT_EOP flag is not set, the RX packet
1456 * is likely to be invalid and to be dropped by the various
1457 * validation checks performed by the network stack.
1459 * Allocate a new mbuf to replenish the RX ring descriptor.
1460 * If the allocation fails:
1461 * - arrange for that RX descriptor to be the first one
1462 * being parsed the next time the receive function is
1463 * invoked [on the same queue].
1465 * - Stop parsing the RX ring and return immediately.
1467 * This policy do not drop the packet received in the RX
1468 * descriptor for which the allocation of a new mbuf failed.
1469 * Thus, it allows that packet to be later retrieved if
1470 * mbuf have been freed in the mean time.
1471 * As a side effect, holding RX descriptors instead of
1472 * systematically giving them back to the NIC may lead to
1473 * RX ring exhaustion situations.
1474 * However, the NIC can gracefully prevent such situations
1475 * to happen by sending specific "back-pressure" flow control
1476 * frames to its peer(s).
1478 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u "
1479 "ext_err_stat=0x%08x pkt_len=%u",
1480 (uint16_t)rxq->port_id, (uint16_t)rxq->queue_id,
1481 (uint16_t)rx_id, (uint32_t)staterr,
1482 (uint16_t)rte_le_to_cpu_16(rxd.qw1.hi.len));
1484 nmb = rte_mbuf_raw_alloc(rxq->mb_pool);
1486 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u "
1487 "queue_id=%u", (uint16_t)rxq->port_id,
1488 (uint16_t)rxq->queue_id);
1489 dev->data->rx_mbuf_alloc_failed++;
1494 rxe = &sw_ring[rx_id];
1496 if (rx_id == rxq->nb_rx_desc)
1499 /* Prefetch next mbuf while processing current one. */
1500 rte_txgbe_prefetch(sw_ring[rx_id].mbuf);
1503 * When next RX descriptor is on a cache-line boundary,
1504 * prefetch the next 4 RX descriptors and the next 8 pointers
1507 if ((rx_id & 0x3) == 0) {
1508 rte_txgbe_prefetch(&rx_ring[rx_id]);
1509 rte_txgbe_prefetch(&sw_ring[rx_id]);
1514 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
1515 TXGBE_RXD_HDRADDR(rxdp, 0);
1516 TXGBE_RXD_PKTADDR(rxdp, dma_addr);
1519 * Initialize the returned mbuf.
1520 * 1) setup generic mbuf fields:
1521 * - number of segments,
1524 * - RX port identifier.
1525 * 2) integrate hardware offload data, if any:
1526 * - RSS flag & hash,
1527 * - IP checksum flag,
1528 * - VLAN TCI, if any,
1531 pkt_len = (uint16_t)(rte_le_to_cpu_16(rxd.qw1.hi.len) -
1533 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1534 rte_packet_prefetch((char *)rxm->buf_addr + rxm->data_off);
1537 rxm->pkt_len = pkt_len;
1538 rxm->data_len = pkt_len;
1539 rxm->port = rxq->port_id;
1541 pkt_info = rte_le_to_cpu_32(rxd.qw0.dw0);
1542 /* Only valid if PKT_RX_VLAN set in pkt_flags */
1543 rxm->vlan_tci = rte_le_to_cpu_16(rxd.qw1.hi.tag);
1545 pkt_flags = rx_desc_status_to_pkt_flags(staterr,
1547 pkt_flags |= rx_desc_error_to_pkt_flags(staterr);
1548 pkt_flags |= txgbe_rxd_pkt_info_to_pkt_flags(pkt_info);
1549 rxm->ol_flags = pkt_flags;
1550 rxm->packet_type = txgbe_rxd_pkt_info_to_pkt_type(pkt_info,
1551 rxq->pkt_type_mask);
1553 if (likely(pkt_flags & PKT_RX_RSS_HASH)) {
1554 rxm->hash.rss = rte_le_to_cpu_32(rxd.qw0.dw1);
1555 } else if (pkt_flags & PKT_RX_FDIR) {
1556 rxm->hash.fdir.hash =
1557 rte_le_to_cpu_16(rxd.qw0.hi.csum) &
1558 TXGBE_ATR_HASH_MASK;
1559 rxm->hash.fdir.id = rte_le_to_cpu_16(rxd.qw0.hi.ipid);
1562 * Store the mbuf address into the next entry of the array
1563 * of returned packets.
1565 rx_pkts[nb_rx++] = rxm;
1567 rxq->rx_tail = rx_id;
1570 * If the number of free RX descriptors is greater than the RX free
1571 * threshold of the queue, advance the Receive Descriptor Tail (RDT)
1573 * Update the RDT with the value of the last processed RX descriptor
1574 * minus 1, to guarantee that the RDT register is never equal to the
1575 * RDH register, which creates a "full" ring situation from the
1576 * hardware point of view...
1578 nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold);
1579 if (nb_hold > rxq->rx_free_thresh) {
1580 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u "
1581 "nb_hold=%u nb_rx=%u",
1582 (uint16_t)rxq->port_id, (uint16_t)rxq->queue_id,
1583 (uint16_t)rx_id, (uint16_t)nb_hold,
1585 rx_id = (uint16_t)((rx_id == 0) ?
1586 (rxq->nb_rx_desc - 1) : (rx_id - 1));
1587 txgbe_set32(rxq->rdt_reg_addr, rx_id);
1590 rxq->nb_rx_hold = nb_hold;
1595 * txgbe_fill_cluster_head_buf - fill the first mbuf of the returned packet
1597 * Fill the following info in the HEAD buffer of the Rx cluster:
1598 * - RX port identifier
1599 * - hardware offload data, if any:
1601 * - IP checksum flag
1602 * - VLAN TCI, if any
1604 * @head HEAD of the packet cluster
1605 * @desc HW descriptor to get data from
1606 * @rxq Pointer to the Rx queue
1609 txgbe_fill_cluster_head_buf(struct rte_mbuf *head, struct txgbe_rx_desc *desc,
1610 struct txgbe_rx_queue *rxq, uint32_t staterr)
1615 head->port = rxq->port_id;
1617 /* The vlan_tci field is only valid when PKT_RX_VLAN is
1618 * set in the pkt_flags field.
1620 head->vlan_tci = rte_le_to_cpu_16(desc->qw1.hi.tag);
1621 pkt_info = rte_le_to_cpu_32(desc->qw0.dw0);
1622 pkt_flags = rx_desc_status_to_pkt_flags(staterr, rxq->vlan_flags);
1623 pkt_flags |= rx_desc_error_to_pkt_flags(staterr);
1624 pkt_flags |= txgbe_rxd_pkt_info_to_pkt_flags(pkt_info);
1625 head->ol_flags = pkt_flags;
1626 head->packet_type = txgbe_rxd_pkt_info_to_pkt_type(pkt_info,
1627 rxq->pkt_type_mask);
1629 if (likely(pkt_flags & PKT_RX_RSS_HASH)) {
1630 head->hash.rss = rte_le_to_cpu_32(desc->qw0.dw1);
1631 } else if (pkt_flags & PKT_RX_FDIR) {
1632 head->hash.fdir.hash = rte_le_to_cpu_16(desc->qw0.hi.csum)
1633 & TXGBE_ATR_HASH_MASK;
1634 head->hash.fdir.id = rte_le_to_cpu_16(desc->qw0.hi.ipid);
1639 * txgbe_recv_pkts_lro - receive handler for and LRO case.
1641 * @rx_queue Rx queue handle
1642 * @rx_pkts table of received packets
1643 * @nb_pkts size of rx_pkts table
1644 * @bulk_alloc if TRUE bulk allocation is used for a HW ring refilling
1646 * Handles the Rx HW ring completions when RSC feature is configured. Uses an
1647 * additional ring of txgbe_rsc_entry's that will hold the relevant RSC info.
1649 * We use the same logic as in Linux and in FreeBSD txgbe drivers:
1650 * 1) When non-EOP RSC completion arrives:
1651 * a) Update the HEAD of the current RSC aggregation cluster with the new
1652 * segment's data length.
1653 * b) Set the "next" pointer of the current segment to point to the segment
1654 * at the NEXTP index.
1655 * c) Pass the HEAD of RSC aggregation cluster on to the next NEXTP entry
1656 * in the sw_rsc_ring.
1657 * 2) When EOP arrives we just update the cluster's total length and offload
1658 * flags and deliver the cluster up to the upper layers. In our case - put it
1659 * in the rx_pkts table.
1661 * Returns the number of received packets/clusters (according to the "bulk
1662 * receive" interface).
1664 static inline uint16_t
1665 txgbe_recv_pkts_lro(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts,
1668 struct txgbe_rx_queue *rxq = rx_queue;
1669 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id];
1670 volatile struct txgbe_rx_desc *rx_ring = rxq->rx_ring;
1671 struct txgbe_rx_entry *sw_ring = rxq->sw_ring;
1672 struct txgbe_scattered_rx_entry *sw_sc_ring = rxq->sw_sc_ring;
1673 uint16_t rx_id = rxq->rx_tail;
1675 uint16_t nb_hold = rxq->nb_rx_hold;
1676 uint16_t prev_id = rxq->rx_tail;
1678 while (nb_rx < nb_pkts) {
1680 struct txgbe_rx_entry *rxe;
1681 struct txgbe_scattered_rx_entry *sc_entry;
1682 struct txgbe_scattered_rx_entry *next_sc_entry = NULL;
1683 struct txgbe_rx_entry *next_rxe = NULL;
1684 struct rte_mbuf *first_seg;
1685 struct rte_mbuf *rxm;
1686 struct rte_mbuf *nmb = NULL;
1687 struct txgbe_rx_desc rxd;
1690 volatile struct txgbe_rx_desc *rxdp;
1695 * The code in this whole file uses the volatile pointer to
1696 * ensure the read ordering of the status and the rest of the
1697 * descriptor fields (on the compiler level only!!!). This is so
1698 * UGLY - why not to just use the compiler barrier instead? DPDK
1699 * even has the rte_compiler_barrier() for that.
1701 * But most importantly this is just wrong because this doesn't
1702 * ensure memory ordering in a general case at all. For
1703 * instance, DPDK is supposed to work on Power CPUs where
1704 * compiler barrier may just not be enough!
1706 * I tried to write only this function properly to have a
1707 * starting point (as a part of an LRO/RSC series) but the
1708 * compiler cursed at me when I tried to cast away the
1709 * "volatile" from rx_ring (yes, it's volatile too!!!). So, I'm
1710 * keeping it the way it is for now.
1712 * The code in this file is broken in so many other places and
1713 * will just not work on a big endian CPU anyway therefore the
1714 * lines below will have to be revisited together with the rest
1718 * - Get rid of "volatile" and let the compiler do its job.
1719 * - Use the proper memory barrier (rte_rmb()) to ensure the
1720 * memory ordering below.
1722 rxdp = &rx_ring[rx_id];
1723 staterr = rte_le_to_cpu_32(rxdp->qw1.lo.status);
1725 if (!(staterr & TXGBE_RXD_STAT_DD))
1730 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u "
1731 "staterr=0x%x data_len=%u",
1732 rxq->port_id, rxq->queue_id, rx_id, staterr,
1733 rte_le_to_cpu_16(rxd.qw1.hi.len));
1736 nmb = rte_mbuf_raw_alloc(rxq->mb_pool);
1738 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed "
1739 "port_id=%u queue_id=%u",
1740 rxq->port_id, rxq->queue_id);
1742 dev->data->rx_mbuf_alloc_failed++;
1745 } else if (nb_hold > rxq->rx_free_thresh) {
1746 uint16_t next_rdt = rxq->rx_free_trigger;
1748 if (!txgbe_rx_alloc_bufs(rxq, false)) {
1750 txgbe_set32_relaxed(rxq->rdt_reg_addr,
1752 nb_hold -= rxq->rx_free_thresh;
1754 PMD_RX_LOG(DEBUG, "RX bulk alloc failed "
1755 "port_id=%u queue_id=%u",
1756 rxq->port_id, rxq->queue_id);
1758 dev->data->rx_mbuf_alloc_failed++;
1764 rxe = &sw_ring[rx_id];
1765 eop = staterr & TXGBE_RXD_STAT_EOP;
1767 next_id = rx_id + 1;
1768 if (next_id == rxq->nb_rx_desc)
1771 /* Prefetch next mbuf while processing current one. */
1772 rte_txgbe_prefetch(sw_ring[next_id].mbuf);
1775 * When next RX descriptor is on a cache-line boundary,
1776 * prefetch the next 4 RX descriptors and the next 4 pointers
1779 if ((next_id & 0x3) == 0) {
1780 rte_txgbe_prefetch(&rx_ring[next_id]);
1781 rte_txgbe_prefetch(&sw_ring[next_id]);
1788 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
1790 * Update RX descriptor with the physical address of the
1791 * new data buffer of the new allocated mbuf.
1795 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1796 TXGBE_RXD_HDRADDR(rxdp, 0);
1797 TXGBE_RXD_PKTADDR(rxdp, dma);
1803 * Set data length & data buffer address of mbuf.
1805 data_len = rte_le_to_cpu_16(rxd.qw1.hi.len);
1806 rxm->data_len = data_len;
1811 * Get next descriptor index:
1812 * - For RSC it's in the NEXTP field.
1813 * - For a scattered packet - it's just a following
1816 if (TXGBE_RXD_RSCCNT(rxd.qw0.dw0))
1817 nextp_id = TXGBE_RXD_NEXTP(staterr);
1821 next_sc_entry = &sw_sc_ring[nextp_id];
1822 next_rxe = &sw_ring[nextp_id];
1823 rte_txgbe_prefetch(next_rxe);
1826 sc_entry = &sw_sc_ring[rx_id];
1827 first_seg = sc_entry->fbuf;
1828 sc_entry->fbuf = NULL;
1831 * If this is the first buffer of the received packet,
1832 * set the pointer to the first mbuf of the packet and
1833 * initialize its context.
1834 * Otherwise, update the total length and the number of segments
1835 * of the current scattered packet, and update the pointer to
1836 * the last mbuf of the current packet.
1838 if (first_seg == NULL) {
1840 first_seg->pkt_len = data_len;
1841 first_seg->nb_segs = 1;
1843 first_seg->pkt_len += data_len;
1844 first_seg->nb_segs++;
1851 * If this is not the last buffer of the received packet, update
1852 * the pointer to the first mbuf at the NEXTP entry in the
1853 * sw_sc_ring and continue to parse the RX ring.
1855 if (!eop && next_rxe) {
1856 rxm->next = next_rxe->mbuf;
1857 next_sc_entry->fbuf = first_seg;
1861 /* Initialize the first mbuf of the returned packet */
1862 txgbe_fill_cluster_head_buf(first_seg, &rxd, rxq, staterr);
1865 * Deal with the case, when HW CRC srip is disabled.
1866 * That can't happen when LRO is enabled, but still could
1867 * happen for scattered RX mode.
1869 first_seg->pkt_len -= rxq->crc_len;
1870 if (unlikely(rxm->data_len <= rxq->crc_len)) {
1871 struct rte_mbuf *lp;
1873 for (lp = first_seg; lp->next != rxm; lp = lp->next)
1876 first_seg->nb_segs--;
1877 lp->data_len -= rxq->crc_len - rxm->data_len;
1879 rte_pktmbuf_free_seg(rxm);
1881 rxm->data_len -= rxq->crc_len;
1884 /* Prefetch data of first segment, if configured to do so. */
1885 rte_packet_prefetch((char *)first_seg->buf_addr +
1886 first_seg->data_off);
1889 * Store the mbuf address into the next entry of the array
1890 * of returned packets.
1892 rx_pkts[nb_rx++] = first_seg;
1896 * Record index of the next RX descriptor to probe.
1898 rxq->rx_tail = rx_id;
1901 * If the number of free RX descriptors is greater than the RX free
1902 * threshold of the queue, advance the Receive Descriptor Tail (RDT)
1904 * Update the RDT with the value of the last processed RX descriptor
1905 * minus 1, to guarantee that the RDT register is never equal to the
1906 * RDH register, which creates a "full" ring situation from the
1907 * hardware point of view...
1909 if (!bulk_alloc && nb_hold > rxq->rx_free_thresh) {
1910 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u "
1911 "nb_hold=%u nb_rx=%u",
1912 rxq->port_id, rxq->queue_id, rx_id, nb_hold, nb_rx);
1915 txgbe_set32_relaxed(rxq->rdt_reg_addr, prev_id);
1919 rxq->nb_rx_hold = nb_hold;
1924 txgbe_recv_pkts_lro_single_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
1927 return txgbe_recv_pkts_lro(rx_queue, rx_pkts, nb_pkts, false);
1931 txgbe_recv_pkts_lro_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
1934 return txgbe_recv_pkts_lro(rx_queue, rx_pkts, nb_pkts, true);
1938 txgbe_get_rx_queue_offloads(struct rte_eth_dev *dev __rte_unused)
1940 return DEV_RX_OFFLOAD_VLAN_STRIP;
1944 txgbe_get_rx_port_offloads(struct rte_eth_dev *dev)
1947 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
1948 struct rte_eth_dev_sriov *sriov = &RTE_ETH_DEV_SRIOV(dev);
1950 offloads = DEV_RX_OFFLOAD_IPV4_CKSUM |
1951 DEV_RX_OFFLOAD_UDP_CKSUM |
1952 DEV_RX_OFFLOAD_TCP_CKSUM |
1953 DEV_RX_OFFLOAD_KEEP_CRC |
1954 DEV_RX_OFFLOAD_JUMBO_FRAME |
1955 DEV_RX_OFFLOAD_VLAN_FILTER |
1956 DEV_RX_OFFLOAD_RSS_HASH |
1957 DEV_RX_OFFLOAD_SCATTER;
1959 if (!txgbe_is_vf(dev))
1960 offloads |= (DEV_RX_OFFLOAD_VLAN_FILTER |
1961 DEV_RX_OFFLOAD_QINQ_STRIP |
1962 DEV_RX_OFFLOAD_VLAN_EXTEND);
1965 * RSC is only supported by PF devices in a non-SR-IOV
1968 if (hw->mac.type == txgbe_mac_raptor && !sriov->active)
1969 offloads |= DEV_RX_OFFLOAD_TCP_LRO;
1971 if (hw->mac.type == txgbe_mac_raptor)
1972 offloads |= DEV_RX_OFFLOAD_MACSEC_STRIP;
1974 offloads |= DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM;
1976 #ifdef RTE_LIB_SECURITY
1977 if (dev->security_ctx)
1978 offloads |= DEV_RX_OFFLOAD_SECURITY;
1984 static void __rte_cold
1985 txgbe_tx_queue_release_mbufs(struct txgbe_tx_queue *txq)
1989 if (txq->sw_ring != NULL) {
1990 for (i = 0; i < txq->nb_tx_desc; i++) {
1991 if (txq->sw_ring[i].mbuf != NULL) {
1992 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
1993 txq->sw_ring[i].mbuf = NULL;
2000 txgbe_tx_done_cleanup_full(struct txgbe_tx_queue *txq, uint32_t free_cnt)
2002 struct txgbe_tx_entry *swr_ring = txq->sw_ring;
2003 uint16_t i, tx_last, tx_id;
2004 uint16_t nb_tx_free_last;
2005 uint16_t nb_tx_to_clean;
2008 /* Start free mbuf from the next of tx_tail */
2009 tx_last = txq->tx_tail;
2010 tx_id = swr_ring[tx_last].next_id;
2012 if (txq->nb_tx_free == 0 && txgbe_xmit_cleanup(txq))
2015 nb_tx_to_clean = txq->nb_tx_free;
2016 nb_tx_free_last = txq->nb_tx_free;
2018 free_cnt = txq->nb_tx_desc;
2020 /* Loop through swr_ring to count the amount of
2021 * freeable mubfs and packets.
2023 for (pkt_cnt = 0; pkt_cnt < free_cnt; ) {
2024 for (i = 0; i < nb_tx_to_clean &&
2025 pkt_cnt < free_cnt &&
2026 tx_id != tx_last; i++) {
2027 if (swr_ring[tx_id].mbuf != NULL) {
2028 rte_pktmbuf_free_seg(swr_ring[tx_id].mbuf);
2029 swr_ring[tx_id].mbuf = NULL;
2032 * last segment in the packet,
2033 * increment packet count
2035 pkt_cnt += (swr_ring[tx_id].last_id == tx_id);
2038 tx_id = swr_ring[tx_id].next_id;
2041 if (pkt_cnt < free_cnt) {
2042 if (txgbe_xmit_cleanup(txq))
2045 nb_tx_to_clean = txq->nb_tx_free - nb_tx_free_last;
2046 nb_tx_free_last = txq->nb_tx_free;
2050 return (int)pkt_cnt;
2054 txgbe_tx_done_cleanup_simple(struct txgbe_tx_queue *txq,
2059 if (free_cnt == 0 || free_cnt > txq->nb_tx_desc)
2060 free_cnt = txq->nb_tx_desc;
2062 cnt = free_cnt - free_cnt % txq->tx_free_thresh;
2064 for (i = 0; i < cnt; i += n) {
2065 if (txq->nb_tx_desc - txq->nb_tx_free < txq->tx_free_thresh)
2068 n = txgbe_tx_free_bufs(txq);
2078 txgbe_dev_tx_done_cleanup(void *tx_queue, uint32_t free_cnt)
2080 struct txgbe_tx_queue *txq = (struct txgbe_tx_queue *)tx_queue;
2081 if (txq->offloads == 0 &&
2082 #ifdef RTE_LIB_SECURITY
2083 !(txq->using_ipsec) &&
2085 txq->tx_free_thresh >= RTE_PMD_TXGBE_TX_MAX_BURST)
2086 return txgbe_tx_done_cleanup_simple(txq, free_cnt);
2088 return txgbe_tx_done_cleanup_full(txq, free_cnt);
2091 static void __rte_cold
2092 txgbe_tx_free_swring(struct txgbe_tx_queue *txq)
2095 txq->sw_ring != NULL)
2096 rte_free(txq->sw_ring);
2099 static void __rte_cold
2100 txgbe_tx_queue_release(struct txgbe_tx_queue *txq)
2102 if (txq != NULL && txq->ops != NULL) {
2103 txq->ops->release_mbufs(txq);
2104 txq->ops->free_swring(txq);
2110 txgbe_dev_tx_queue_release(void *txq)
2112 txgbe_tx_queue_release(txq);
2115 /* (Re)set dynamic txgbe_tx_queue fields to defaults */
2116 static void __rte_cold
2117 txgbe_reset_tx_queue(struct txgbe_tx_queue *txq)
2119 static const struct txgbe_tx_desc zeroed_desc = {0};
2120 struct txgbe_tx_entry *txe = txq->sw_ring;
2123 /* Zero out HW ring memory */
2124 for (i = 0; i < txq->nb_tx_desc; i++)
2125 txq->tx_ring[i] = zeroed_desc;
2127 /* Initialize SW ring entries */
2128 prev = (uint16_t)(txq->nb_tx_desc - 1);
2129 for (i = 0; i < txq->nb_tx_desc; i++) {
2130 volatile struct txgbe_tx_desc *txd = &txq->tx_ring[i];
2132 txd->dw3 = rte_cpu_to_le_32(TXGBE_TXD_DD);
2135 txe[prev].next_id = i;
2139 txq->tx_next_dd = (uint16_t)(txq->tx_free_thresh - 1);
2143 * Always allow 1 descriptor to be un-allocated to avoid
2144 * a H/W race condition
2146 txq->last_desc_cleaned = (uint16_t)(txq->nb_tx_desc - 1);
2147 txq->nb_tx_free = (uint16_t)(txq->nb_tx_desc - 1);
2149 memset((void *)&txq->ctx_cache, 0,
2150 TXGBE_CTX_NUM * sizeof(struct txgbe_ctx_info));
2153 static const struct txgbe_txq_ops def_txq_ops = {
2154 .release_mbufs = txgbe_tx_queue_release_mbufs,
2155 .free_swring = txgbe_tx_free_swring,
2156 .reset = txgbe_reset_tx_queue,
2159 /* Takes an ethdev and a queue and sets up the tx function to be used based on
2160 * the queue parameters. Used in tx_queue_setup by primary process and then
2161 * in dev_init by secondary process when attaching to an existing ethdev.
2164 txgbe_set_tx_function(struct rte_eth_dev *dev, struct txgbe_tx_queue *txq)
2166 /* Use a simple Tx queue (no offloads, no multi segs) if possible */
2167 if (txq->offloads == 0 &&
2168 #ifdef RTE_LIB_SECURITY
2169 !(txq->using_ipsec) &&
2171 txq->tx_free_thresh >= RTE_PMD_TXGBE_TX_MAX_BURST) {
2172 PMD_INIT_LOG(DEBUG, "Using simple tx code path");
2173 dev->tx_pkt_burst = txgbe_xmit_pkts_simple;
2174 dev->tx_pkt_prepare = NULL;
2176 PMD_INIT_LOG(DEBUG, "Using full-featured tx code path");
2178 " - offloads = 0x%" PRIx64,
2181 " - tx_free_thresh = %lu [RTE_PMD_TXGBE_TX_MAX_BURST=%lu]",
2182 (unsigned long)txq->tx_free_thresh,
2183 (unsigned long)RTE_PMD_TXGBE_TX_MAX_BURST);
2184 dev->tx_pkt_burst = txgbe_xmit_pkts;
2185 dev->tx_pkt_prepare = txgbe_prep_pkts;
2190 txgbe_get_tx_queue_offloads(struct rte_eth_dev *dev)
2198 txgbe_get_tx_port_offloads(struct rte_eth_dev *dev)
2200 uint64_t tx_offload_capa;
2203 DEV_TX_OFFLOAD_VLAN_INSERT |
2204 DEV_TX_OFFLOAD_IPV4_CKSUM |
2205 DEV_TX_OFFLOAD_UDP_CKSUM |
2206 DEV_TX_OFFLOAD_TCP_CKSUM |
2207 DEV_TX_OFFLOAD_SCTP_CKSUM |
2208 DEV_TX_OFFLOAD_TCP_TSO |
2209 DEV_TX_OFFLOAD_UDP_TSO |
2210 DEV_TX_OFFLOAD_UDP_TNL_TSO |
2211 DEV_TX_OFFLOAD_IP_TNL_TSO |
2212 DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
2213 DEV_TX_OFFLOAD_GRE_TNL_TSO |
2214 DEV_TX_OFFLOAD_IPIP_TNL_TSO |
2215 DEV_TX_OFFLOAD_GENEVE_TNL_TSO |
2216 DEV_TX_OFFLOAD_MULTI_SEGS;
2218 if (!txgbe_is_vf(dev))
2219 tx_offload_capa |= DEV_TX_OFFLOAD_QINQ_INSERT;
2221 tx_offload_capa |= DEV_TX_OFFLOAD_MACSEC_INSERT;
2223 tx_offload_capa |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM;
2225 #ifdef RTE_LIB_SECURITY
2226 if (dev->security_ctx)
2227 tx_offload_capa |= DEV_TX_OFFLOAD_SECURITY;
2229 return tx_offload_capa;
2233 txgbe_dev_tx_queue_setup(struct rte_eth_dev *dev,
2236 unsigned int socket_id,
2237 const struct rte_eth_txconf *tx_conf)
2239 const struct rte_memzone *tz;
2240 struct txgbe_tx_queue *txq;
2241 struct txgbe_hw *hw;
2242 uint16_t tx_free_thresh;
2245 PMD_INIT_FUNC_TRACE();
2246 hw = TXGBE_DEV_HW(dev);
2248 offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads;
2251 * Validate number of transmit descriptors.
2252 * It must not exceed hardware maximum, and must be multiple
2255 if (nb_desc % TXGBE_TXD_ALIGN != 0 ||
2256 nb_desc > TXGBE_RING_DESC_MAX ||
2257 nb_desc < TXGBE_RING_DESC_MIN) {
2262 * The TX descriptor ring will be cleaned after txq->tx_free_thresh
2263 * descriptors are used or if the number of descriptors required
2264 * to transmit a packet is greater than the number of free TX
2266 * One descriptor in the TX ring is used as a sentinel to avoid a
2267 * H/W race condition, hence the maximum threshold constraints.
2268 * When set to zero use default values.
2270 tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ?
2271 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH);
2272 if (tx_free_thresh >= (nb_desc - 3)) {
2273 PMD_INIT_LOG(ERR, "tx_free_thresh must be less than the number of "
2274 "TX descriptors minus 3. (tx_free_thresh=%u "
2275 "port=%d queue=%d)",
2276 (unsigned int)tx_free_thresh,
2277 (int)dev->data->port_id, (int)queue_idx);
2281 if ((nb_desc % tx_free_thresh) != 0) {
2282 PMD_INIT_LOG(ERR, "tx_free_thresh must be a divisor of the "
2283 "number of TX descriptors. (tx_free_thresh=%u "
2284 "port=%d queue=%d)", (unsigned int)tx_free_thresh,
2285 (int)dev->data->port_id, (int)queue_idx);
2289 /* Free memory prior to re-allocation if needed... */
2290 if (dev->data->tx_queues[queue_idx] != NULL) {
2291 txgbe_tx_queue_release(dev->data->tx_queues[queue_idx]);
2292 dev->data->tx_queues[queue_idx] = NULL;
2295 /* First allocate the tx queue data structure */
2296 txq = rte_zmalloc_socket("ethdev TX queue",
2297 sizeof(struct txgbe_tx_queue),
2298 RTE_CACHE_LINE_SIZE, socket_id);
2303 * Allocate TX ring hardware descriptors. A memzone large enough to
2304 * handle the maximum ring size is allocated in order to allow for
2305 * resizing in later calls to the queue setup function.
2307 tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx,
2308 sizeof(struct txgbe_tx_desc) * TXGBE_RING_DESC_MAX,
2309 TXGBE_ALIGN, socket_id);
2311 txgbe_tx_queue_release(txq);
2315 txq->nb_tx_desc = nb_desc;
2316 txq->tx_free_thresh = tx_free_thresh;
2317 txq->pthresh = tx_conf->tx_thresh.pthresh;
2318 txq->hthresh = tx_conf->tx_thresh.hthresh;
2319 txq->wthresh = tx_conf->tx_thresh.wthresh;
2320 txq->queue_id = queue_idx;
2321 txq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ?
2322 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx);
2323 txq->port_id = dev->data->port_id;
2324 txq->offloads = offloads;
2325 txq->ops = &def_txq_ops;
2326 txq->tx_deferred_start = tx_conf->tx_deferred_start;
2327 #ifdef RTE_LIB_SECURITY
2328 txq->using_ipsec = !!(dev->data->dev_conf.txmode.offloads &
2329 DEV_TX_OFFLOAD_SECURITY);
2332 /* Modification to set tail pointer for virtual function
2333 * if vf is detected.
2335 if (hw->mac.type == txgbe_mac_raptor_vf) {
2336 txq->tdt_reg_addr = TXGBE_REG_ADDR(hw, TXGBE_TXWP(queue_idx));
2337 txq->tdc_reg_addr = TXGBE_REG_ADDR(hw, TXGBE_TXCFG(queue_idx));
2339 txq->tdt_reg_addr = TXGBE_REG_ADDR(hw,
2340 TXGBE_TXWP(txq->reg_idx));
2341 txq->tdc_reg_addr = TXGBE_REG_ADDR(hw,
2342 TXGBE_TXCFG(txq->reg_idx));
2345 txq->tx_ring_phys_addr = TMZ_PADDR(tz);
2346 txq->tx_ring = (struct txgbe_tx_desc *)TMZ_VADDR(tz);
2348 /* Allocate software ring */
2349 txq->sw_ring = rte_zmalloc_socket("txq->sw_ring",
2350 sizeof(struct txgbe_tx_entry) * nb_desc,
2351 RTE_CACHE_LINE_SIZE, socket_id);
2352 if (txq->sw_ring == NULL) {
2353 txgbe_tx_queue_release(txq);
2356 PMD_INIT_LOG(DEBUG, "sw_ring=%p hw_ring=%p dma_addr=0x%" PRIx64,
2357 txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr);
2359 /* set up scalar TX function as appropriate */
2360 txgbe_set_tx_function(dev, txq);
2362 txq->ops->reset(txq);
2364 dev->data->tx_queues[queue_idx] = txq;
2370 * txgbe_free_sc_cluster - free the not-yet-completed scattered cluster
2372 * The "next" pointer of the last segment of (not-yet-completed) RSC clusters
2373 * in the sw_rsc_ring is not set to NULL but rather points to the next
2374 * mbuf of this RSC aggregation (that has not been completed yet and still
2375 * resides on the HW ring). So, instead of calling for rte_pktmbuf_free() we
2376 * will just free first "nb_segs" segments of the cluster explicitly by calling
2377 * an rte_pktmbuf_free_seg().
2379 * @m scattered cluster head
2381 static void __rte_cold
2382 txgbe_free_sc_cluster(struct rte_mbuf *m)
2384 uint16_t i, nb_segs = m->nb_segs;
2385 struct rte_mbuf *next_seg;
2387 for (i = 0; i < nb_segs; i++) {
2389 rte_pktmbuf_free_seg(m);
2394 static void __rte_cold
2395 txgbe_rx_queue_release_mbufs(struct txgbe_rx_queue *rxq)
2399 if (rxq->sw_ring != NULL) {
2400 for (i = 0; i < rxq->nb_rx_desc; i++) {
2401 if (rxq->sw_ring[i].mbuf != NULL) {
2402 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
2403 rxq->sw_ring[i].mbuf = NULL;
2406 if (rxq->rx_nb_avail) {
2407 for (i = 0; i < rxq->rx_nb_avail; ++i) {
2408 struct rte_mbuf *mb;
2410 mb = rxq->rx_stage[rxq->rx_next_avail + i];
2411 rte_pktmbuf_free_seg(mb);
2413 rxq->rx_nb_avail = 0;
2417 if (rxq->sw_sc_ring)
2418 for (i = 0; i < rxq->nb_rx_desc; i++)
2419 if (rxq->sw_sc_ring[i].fbuf) {
2420 txgbe_free_sc_cluster(rxq->sw_sc_ring[i].fbuf);
2421 rxq->sw_sc_ring[i].fbuf = NULL;
2425 static void __rte_cold
2426 txgbe_rx_queue_release(struct txgbe_rx_queue *rxq)
2429 txgbe_rx_queue_release_mbufs(rxq);
2430 rte_free(rxq->sw_ring);
2431 rte_free(rxq->sw_sc_ring);
2437 txgbe_dev_rx_queue_release(void *rxq)
2439 txgbe_rx_queue_release(rxq);
2443 * Check if Rx Burst Bulk Alloc function can be used.
2445 * 0: the preconditions are satisfied and the bulk allocation function
2447 * -EINVAL: the preconditions are NOT satisfied and the default Rx burst
2448 * function must be used.
2450 static inline int __rte_cold
2451 check_rx_burst_bulk_alloc_preconditions(struct txgbe_rx_queue *rxq)
2456 * Make sure the following pre-conditions are satisfied:
2457 * rxq->rx_free_thresh >= RTE_PMD_TXGBE_RX_MAX_BURST
2458 * rxq->rx_free_thresh < rxq->nb_rx_desc
2459 * (rxq->nb_rx_desc % rxq->rx_free_thresh) == 0
2460 * Scattered packets are not supported. This should be checked
2461 * outside of this function.
2463 if (!(rxq->rx_free_thresh >= RTE_PMD_TXGBE_RX_MAX_BURST)) {
2464 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
2465 "rxq->rx_free_thresh=%d, "
2466 "RTE_PMD_TXGBE_RX_MAX_BURST=%d",
2467 rxq->rx_free_thresh, RTE_PMD_TXGBE_RX_MAX_BURST);
2469 } else if (!(rxq->rx_free_thresh < rxq->nb_rx_desc)) {
2470 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
2471 "rxq->rx_free_thresh=%d, "
2472 "rxq->nb_rx_desc=%d",
2473 rxq->rx_free_thresh, rxq->nb_rx_desc);
2475 } else if (!((rxq->nb_rx_desc % rxq->rx_free_thresh) == 0)) {
2476 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
2477 "rxq->nb_rx_desc=%d, "
2478 "rxq->rx_free_thresh=%d",
2479 rxq->nb_rx_desc, rxq->rx_free_thresh);
2486 /* Reset dynamic txgbe_rx_queue fields back to defaults */
2487 static void __rte_cold
2488 txgbe_reset_rx_queue(struct txgbe_adapter *adapter, struct txgbe_rx_queue *rxq)
2490 static const struct txgbe_rx_desc zeroed_desc = {
2491 {{0}, {0} }, {{0}, {0} } };
2493 uint16_t len = rxq->nb_rx_desc;
2496 * By default, the Rx queue setup function allocates enough memory for
2497 * TXGBE_RING_DESC_MAX. The Rx Burst bulk allocation function requires
2498 * extra memory at the end of the descriptor ring to be zero'd out.
2500 if (adapter->rx_bulk_alloc_allowed)
2501 /* zero out extra memory */
2502 len += RTE_PMD_TXGBE_RX_MAX_BURST;
2505 * Zero out HW ring memory. Zero out extra memory at the end of
2506 * the H/W ring so look-ahead logic in Rx Burst bulk alloc function
2507 * reads extra memory as zeros.
2509 for (i = 0; i < len; i++)
2510 rxq->rx_ring[i] = zeroed_desc;
2513 * initialize extra software ring entries. Space for these extra
2514 * entries is always allocated
2516 memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf));
2517 for (i = rxq->nb_rx_desc; i < len; ++i)
2518 rxq->sw_ring[i].mbuf = &rxq->fake_mbuf;
2520 rxq->rx_nb_avail = 0;
2521 rxq->rx_next_avail = 0;
2522 rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1);
2524 rxq->nb_rx_hold = 0;
2525 rxq->pkt_first_seg = NULL;
2526 rxq->pkt_last_seg = NULL;
2530 txgbe_dev_rx_queue_setup(struct rte_eth_dev *dev,
2533 unsigned int socket_id,
2534 const struct rte_eth_rxconf *rx_conf,
2535 struct rte_mempool *mp)
2537 const struct rte_memzone *rz;
2538 struct txgbe_rx_queue *rxq;
2539 struct txgbe_hw *hw;
2541 struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev);
2544 PMD_INIT_FUNC_TRACE();
2545 hw = TXGBE_DEV_HW(dev);
2547 offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads;
2550 * Validate number of receive descriptors.
2551 * It must not exceed hardware maximum, and must be multiple
2554 if (nb_desc % TXGBE_RXD_ALIGN != 0 ||
2555 nb_desc > TXGBE_RING_DESC_MAX ||
2556 nb_desc < TXGBE_RING_DESC_MIN) {
2560 /* Free memory prior to re-allocation if needed... */
2561 if (dev->data->rx_queues[queue_idx] != NULL) {
2562 txgbe_rx_queue_release(dev->data->rx_queues[queue_idx]);
2563 dev->data->rx_queues[queue_idx] = NULL;
2566 /* First allocate the rx queue data structure */
2567 rxq = rte_zmalloc_socket("ethdev RX queue",
2568 sizeof(struct txgbe_rx_queue),
2569 RTE_CACHE_LINE_SIZE, socket_id);
2573 rxq->nb_rx_desc = nb_desc;
2574 rxq->rx_free_thresh = rx_conf->rx_free_thresh;
2575 rxq->queue_id = queue_idx;
2576 rxq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ?
2577 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx);
2578 rxq->port_id = dev->data->port_id;
2579 if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC)
2580 rxq->crc_len = RTE_ETHER_CRC_LEN;
2583 rxq->drop_en = rx_conf->rx_drop_en;
2584 rxq->rx_deferred_start = rx_conf->rx_deferred_start;
2585 rxq->offloads = offloads;
2588 * The packet type in RX descriptor is different for different NICs.
2589 * So set different masks for different NICs.
2591 rxq->pkt_type_mask = TXGBE_PTID_MASK;
2594 * Allocate RX ring hardware descriptors. A memzone large enough to
2595 * handle the maximum ring size is allocated in order to allow for
2596 * resizing in later calls to the queue setup function.
2598 rz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx,
2599 RX_RING_SZ, TXGBE_ALIGN, socket_id);
2601 txgbe_rx_queue_release(rxq);
2606 * Zero init all the descriptors in the ring.
2608 memset(rz->addr, 0, RX_RING_SZ);
2611 * Modified to setup VFRDT for Virtual Function
2613 if (hw->mac.type == txgbe_mac_raptor_vf) {
2615 TXGBE_REG_ADDR(hw, TXGBE_RXWP(queue_idx));
2617 TXGBE_REG_ADDR(hw, TXGBE_RXRP(queue_idx));
2620 TXGBE_REG_ADDR(hw, TXGBE_RXWP(rxq->reg_idx));
2622 TXGBE_REG_ADDR(hw, TXGBE_RXRP(rxq->reg_idx));
2625 rxq->rx_ring_phys_addr = TMZ_PADDR(rz);
2626 rxq->rx_ring = (struct txgbe_rx_desc *)TMZ_VADDR(rz);
2629 * Certain constraints must be met in order to use the bulk buffer
2630 * allocation Rx burst function. If any of Rx queues doesn't meet them
2631 * the feature should be disabled for the whole port.
2633 if (check_rx_burst_bulk_alloc_preconditions(rxq)) {
2634 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Rx Bulk Alloc "
2635 "preconditions - canceling the feature for "
2636 "the whole port[%d]",
2637 rxq->queue_id, rxq->port_id);
2638 adapter->rx_bulk_alloc_allowed = false;
2642 * Allocate software ring. Allow for space at the end of the
2643 * S/W ring to make sure look-ahead logic in bulk alloc Rx burst
2644 * function does not access an invalid memory region.
2647 if (adapter->rx_bulk_alloc_allowed)
2648 len += RTE_PMD_TXGBE_RX_MAX_BURST;
2650 rxq->sw_ring = rte_zmalloc_socket("rxq->sw_ring",
2651 sizeof(struct txgbe_rx_entry) * len,
2652 RTE_CACHE_LINE_SIZE, socket_id);
2653 if (!rxq->sw_ring) {
2654 txgbe_rx_queue_release(rxq);
2659 * Always allocate even if it's not going to be needed in order to
2660 * simplify the code.
2662 * This ring is used in LRO and Scattered Rx cases and Scattered Rx may
2663 * be requested in txgbe_dev_rx_init(), which is called later from
2667 rte_zmalloc_socket("rxq->sw_sc_ring",
2668 sizeof(struct txgbe_scattered_rx_entry) * len,
2669 RTE_CACHE_LINE_SIZE, socket_id);
2670 if (!rxq->sw_sc_ring) {
2671 txgbe_rx_queue_release(rxq);
2675 PMD_INIT_LOG(DEBUG, "sw_ring=%p sw_sc_ring=%p hw_ring=%p "
2676 "dma_addr=0x%" PRIx64,
2677 rxq->sw_ring, rxq->sw_sc_ring, rxq->rx_ring,
2678 rxq->rx_ring_phys_addr);
2680 dev->data->rx_queues[queue_idx] = rxq;
2682 txgbe_reset_rx_queue(adapter, rxq);
2688 txgbe_dev_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id)
2690 #define TXGBE_RXQ_SCAN_INTERVAL 4
2691 volatile struct txgbe_rx_desc *rxdp;
2692 struct txgbe_rx_queue *rxq;
2695 rxq = dev->data->rx_queues[rx_queue_id];
2696 rxdp = &rxq->rx_ring[rxq->rx_tail];
2698 while ((desc < rxq->nb_rx_desc) &&
2699 (rxdp->qw1.lo.status &
2700 rte_cpu_to_le_32(TXGBE_RXD_STAT_DD))) {
2701 desc += TXGBE_RXQ_SCAN_INTERVAL;
2702 rxdp += TXGBE_RXQ_SCAN_INTERVAL;
2703 if (rxq->rx_tail + desc >= rxq->nb_rx_desc)
2704 rxdp = &(rxq->rx_ring[rxq->rx_tail +
2705 desc - rxq->nb_rx_desc]);
2712 txgbe_dev_rx_descriptor_status(void *rx_queue, uint16_t offset)
2714 struct txgbe_rx_queue *rxq = rx_queue;
2715 volatile uint32_t *status;
2716 uint32_t nb_hold, desc;
2718 if (unlikely(offset >= rxq->nb_rx_desc))
2721 nb_hold = rxq->nb_rx_hold;
2722 if (offset >= rxq->nb_rx_desc - nb_hold)
2723 return RTE_ETH_RX_DESC_UNAVAIL;
2725 desc = rxq->rx_tail + offset;
2726 if (desc >= rxq->nb_rx_desc)
2727 desc -= rxq->nb_rx_desc;
2729 status = &rxq->rx_ring[desc].qw1.lo.status;
2730 if (*status & rte_cpu_to_le_32(TXGBE_RXD_STAT_DD))
2731 return RTE_ETH_RX_DESC_DONE;
2733 return RTE_ETH_RX_DESC_AVAIL;
2737 txgbe_dev_tx_descriptor_status(void *tx_queue, uint16_t offset)
2739 struct txgbe_tx_queue *txq = tx_queue;
2740 volatile uint32_t *status;
2743 if (unlikely(offset >= txq->nb_tx_desc))
2746 desc = txq->tx_tail + offset;
2747 if (desc >= txq->nb_tx_desc) {
2748 desc -= txq->nb_tx_desc;
2749 if (desc >= txq->nb_tx_desc)
2750 desc -= txq->nb_tx_desc;
2753 status = &txq->tx_ring[desc].dw3;
2754 if (*status & rte_cpu_to_le_32(TXGBE_TXD_DD))
2755 return RTE_ETH_TX_DESC_DONE;
2757 return RTE_ETH_TX_DESC_FULL;
2761 txgbe_dev_clear_queues(struct rte_eth_dev *dev)
2764 struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev);
2766 PMD_INIT_FUNC_TRACE();
2768 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2769 struct txgbe_tx_queue *txq = dev->data->tx_queues[i];
2772 txq->ops->release_mbufs(txq);
2773 txq->ops->reset(txq);
2777 for (i = 0; i < dev->data->nb_rx_queues; i++) {
2778 struct txgbe_rx_queue *rxq = dev->data->rx_queues[i];
2781 txgbe_rx_queue_release_mbufs(rxq);
2782 txgbe_reset_rx_queue(adapter, rxq);
2788 txgbe_dev_free_queues(struct rte_eth_dev *dev)
2792 PMD_INIT_FUNC_TRACE();
2794 for (i = 0; i < dev->data->nb_rx_queues; i++) {
2795 txgbe_dev_rx_queue_release(dev->data->rx_queues[i]);
2796 dev->data->rx_queues[i] = NULL;
2798 dev->data->nb_rx_queues = 0;
2800 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2801 txgbe_dev_tx_queue_release(dev->data->tx_queues[i]);
2802 dev->data->tx_queues[i] = NULL;
2804 dev->data->nb_tx_queues = 0;
2808 * Receive Side Scaling (RSS)
2811 * The source and destination IP addresses of the IP header and the source
2812 * and destination ports of TCP/UDP headers, if any, of received packets are
2813 * hashed against a configurable random key to compute a 32-bit RSS hash result.
2814 * The seven (7) LSBs of the 32-bit hash result are used as an index into a
2815 * 128-entry redirection table (RETA). Each entry of the RETA provides a 3-bit
2816 * RSS output index which is used as the RX queue index where to store the
2818 * The following output is supplied in the RX write-back descriptor:
2819 * - 32-bit result of the Microsoft RSS hash function,
2820 * - 4-bit RSS type field.
2824 * Used as the default key.
2826 static uint8_t rss_intel_key[40] = {
2827 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
2828 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
2829 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
2830 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
2831 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
2835 txgbe_rss_disable(struct rte_eth_dev *dev)
2837 struct txgbe_hw *hw;
2839 hw = TXGBE_DEV_HW(dev);
2841 wr32m(hw, TXGBE_RACTL, TXGBE_RACTL_RSSENA, 0);
2845 txgbe_dev_rss_hash_update(struct rte_eth_dev *dev,
2846 struct rte_eth_rss_conf *rss_conf)
2848 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
2855 if (!txgbe_rss_update_sp(hw->mac.type)) {
2856 PMD_DRV_LOG(ERR, "RSS hash update is not supported on this "
2861 hash_key = rss_conf->rss_key;
2863 /* Fill in RSS hash key */
2864 for (i = 0; i < 10; i++) {
2865 rss_key = LS32(hash_key[(i * 4) + 0], 0, 0xFF);
2866 rss_key |= LS32(hash_key[(i * 4) + 1], 8, 0xFF);
2867 rss_key |= LS32(hash_key[(i * 4) + 2], 16, 0xFF);
2868 rss_key |= LS32(hash_key[(i * 4) + 3], 24, 0xFF);
2869 wr32a(hw, TXGBE_REG_RSSKEY, i, rss_key);
2873 /* Set configured hashing protocols */
2874 rss_hf = rss_conf->rss_hf & TXGBE_RSS_OFFLOAD_ALL;
2875 mrqc = rd32(hw, TXGBE_RACTL);
2876 mrqc &= ~TXGBE_RACTL_RSSMASK;
2877 if (rss_hf & ETH_RSS_IPV4)
2878 mrqc |= TXGBE_RACTL_RSSIPV4;
2879 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
2880 mrqc |= TXGBE_RACTL_RSSIPV4TCP;
2881 if (rss_hf & ETH_RSS_IPV6 ||
2882 rss_hf & ETH_RSS_IPV6_EX)
2883 mrqc |= TXGBE_RACTL_RSSIPV6;
2884 if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP ||
2885 rss_hf & ETH_RSS_IPV6_TCP_EX)
2886 mrqc |= TXGBE_RACTL_RSSIPV6TCP;
2887 if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP)
2888 mrqc |= TXGBE_RACTL_RSSIPV4UDP;
2889 if (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP ||
2890 rss_hf & ETH_RSS_IPV6_UDP_EX)
2891 mrqc |= TXGBE_RACTL_RSSIPV6UDP;
2894 mrqc |= TXGBE_RACTL_RSSENA;
2896 mrqc &= ~TXGBE_RACTL_RSSENA;
2898 wr32(hw, TXGBE_RACTL, mrqc);
2904 txgbe_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
2905 struct rte_eth_rss_conf *rss_conf)
2907 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
2914 hash_key = rss_conf->rss_key;
2916 /* Return RSS hash key */
2917 for (i = 0; i < 10; i++) {
2918 rss_key = rd32a(hw, TXGBE_REG_RSSKEY, i);
2919 hash_key[(i * 4) + 0] = RS32(rss_key, 0, 0xFF);
2920 hash_key[(i * 4) + 1] = RS32(rss_key, 8, 0xFF);
2921 hash_key[(i * 4) + 2] = RS32(rss_key, 16, 0xFF);
2922 hash_key[(i * 4) + 3] = RS32(rss_key, 24, 0xFF);
2927 mrqc = rd32(hw, TXGBE_RACTL);
2928 if (mrqc & TXGBE_RACTL_RSSIPV4)
2929 rss_hf |= ETH_RSS_IPV4;
2930 if (mrqc & TXGBE_RACTL_RSSIPV4TCP)
2931 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP;
2932 if (mrqc & TXGBE_RACTL_RSSIPV6)
2933 rss_hf |= ETH_RSS_IPV6 |
2935 if (mrqc & TXGBE_RACTL_RSSIPV6TCP)
2936 rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP |
2937 ETH_RSS_IPV6_TCP_EX;
2938 if (mrqc & TXGBE_RACTL_RSSIPV4UDP)
2939 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP;
2940 if (mrqc & TXGBE_RACTL_RSSIPV6UDP)
2941 rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP |
2942 ETH_RSS_IPV6_UDP_EX;
2943 if (!(mrqc & TXGBE_RACTL_RSSENA))
2946 rss_hf &= TXGBE_RSS_OFFLOAD_ALL;
2948 rss_conf->rss_hf = rss_hf;
2953 txgbe_rss_configure(struct rte_eth_dev *dev)
2955 struct rte_eth_rss_conf rss_conf;
2956 struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev);
2957 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
2962 PMD_INIT_FUNC_TRACE();
2965 * Fill in redirection table
2966 * The byte-swap is needed because NIC registers are in
2967 * little-endian order.
2969 if (adapter->rss_reta_updated == 0) {
2971 for (i = 0, j = 0; i < ETH_RSS_RETA_SIZE_128; i++, j++) {
2972 if (j == dev->data->nb_rx_queues)
2974 reta = (reta >> 8) | LS32(j, 24, 0xFF);
2976 wr32a(hw, TXGBE_REG_RSSTBL, i >> 2, reta);
2980 * Configure the RSS key and the RSS protocols used to compute
2981 * the RSS hash of input packets.
2983 rss_conf = dev->data->dev_conf.rx_adv_conf.rss_conf;
2984 if (rss_conf.rss_key == NULL)
2985 rss_conf.rss_key = rss_intel_key; /* Default hash key */
2986 txgbe_dev_rss_hash_update(dev, &rss_conf);
2989 #define NUM_VFTA_REGISTERS 128
2990 #define NIC_RX_BUFFER_SIZE 0x200
2993 txgbe_vmdq_dcb_configure(struct rte_eth_dev *dev)
2995 struct rte_eth_vmdq_dcb_conf *cfg;
2996 struct txgbe_hw *hw;
2997 enum rte_eth_nb_pools num_pools;
2998 uint32_t mrqc, vt_ctl, queue_mapping, vlanctrl;
3000 uint8_t nb_tcs; /* number of traffic classes */
3003 PMD_INIT_FUNC_TRACE();
3004 hw = TXGBE_DEV_HW(dev);
3005 cfg = &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf;
3006 num_pools = cfg->nb_queue_pools;
3007 /* Check we have a valid number of pools */
3008 if (num_pools != ETH_16_POOLS && num_pools != ETH_32_POOLS) {
3009 txgbe_rss_disable(dev);
3012 /* 16 pools -> 8 traffic classes, 32 pools -> 4 traffic classes */
3013 nb_tcs = (uint8_t)(ETH_VMDQ_DCB_NUM_QUEUES / (int)num_pools);
3016 * split rx buffer up into sections, each for 1 traffic class
3018 pbsize = (uint16_t)(NIC_RX_BUFFER_SIZE / nb_tcs);
3019 for (i = 0; i < nb_tcs; i++) {
3020 uint32_t rxpbsize = rd32(hw, TXGBE_PBRXSIZE(i));
3022 rxpbsize &= (~(0x3FF << 10));
3023 /* clear 10 bits. */
3024 rxpbsize |= (pbsize << 10); /* set value */
3025 wr32(hw, TXGBE_PBRXSIZE(i), rxpbsize);
3027 /* zero alloc all unused TCs */
3028 for (i = nb_tcs; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3029 uint32_t rxpbsize = rd32(hw, TXGBE_PBRXSIZE(i));
3031 rxpbsize &= (~(0x3FF << 10));
3032 /* clear 10 bits. */
3033 wr32(hw, TXGBE_PBRXSIZE(i), rxpbsize);
3036 if (num_pools == ETH_16_POOLS) {
3037 mrqc = TXGBE_PORTCTL_NUMTC_8;
3038 mrqc |= TXGBE_PORTCTL_NUMVT_16;
3040 mrqc = TXGBE_PORTCTL_NUMTC_4;
3041 mrqc |= TXGBE_PORTCTL_NUMVT_32;
3043 wr32m(hw, TXGBE_PORTCTL,
3044 TXGBE_PORTCTL_NUMTC_MASK | TXGBE_PORTCTL_NUMVT_MASK, mrqc);
3046 vt_ctl = TXGBE_POOLCTL_RPLEN;
3047 if (cfg->enable_default_pool)
3048 vt_ctl |= TXGBE_POOLCTL_DEFPL(cfg->default_pool);
3050 vt_ctl |= TXGBE_POOLCTL_DEFDSA;
3052 wr32(hw, TXGBE_POOLCTL, vt_ctl);
3055 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++)
3057 * mapping is done with 3 bits per priority,
3058 * so shift by i*3 each time
3060 queue_mapping |= ((cfg->dcb_tc[i] & 0x07) << (i * 3));
3062 wr32(hw, TXGBE_RPUP2TC, queue_mapping);
3064 wr32(hw, TXGBE_ARBRXCTL, TXGBE_ARBRXCTL_RRM);
3066 /* enable vlan filtering and allow all vlan tags through */
3067 vlanctrl = rd32(hw, TXGBE_VLANCTL);
3068 vlanctrl |= TXGBE_VLANCTL_VFE; /* enable vlan filters */
3069 wr32(hw, TXGBE_VLANCTL, vlanctrl);
3071 /* enable all vlan filters */
3072 for (i = 0; i < NUM_VFTA_REGISTERS; i++)
3073 wr32(hw, TXGBE_VLANTBL(i), 0xFFFFFFFF);
3075 wr32(hw, TXGBE_POOLRXENA(0),
3076 num_pools == ETH_16_POOLS ? 0xFFFF : 0xFFFFFFFF);
3078 wr32(hw, TXGBE_ETHADDRIDX, 0);
3079 wr32(hw, TXGBE_ETHADDRASSL, 0xFFFFFFFF);
3080 wr32(hw, TXGBE_ETHADDRASSH, 0xFFFFFFFF);
3082 /* set up filters for vlan tags as configured */
3083 for (i = 0; i < cfg->nb_pool_maps; i++) {
3084 /* set vlan id in VF register and set the valid bit */
3085 wr32(hw, TXGBE_PSRVLANIDX, i);
3086 wr32(hw, TXGBE_PSRVLAN, (TXGBE_PSRVLAN_EA |
3087 (cfg->pool_map[i].vlan_id & 0xFFF)));
3089 wr32(hw, TXGBE_PSRVLANPLM(0), cfg->pool_map[i].pools);
3094 * txgbe_dcb_config_tx_hw_config - Configure general DCB TX parameters
3095 * @dev: pointer to eth_dev structure
3096 * @dcb_config: pointer to txgbe_dcb_config structure
3099 txgbe_dcb_tx_hw_config(struct rte_eth_dev *dev,
3100 struct txgbe_dcb_config *dcb_config)
3103 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
3105 PMD_INIT_FUNC_TRACE();
3107 /* Disable the Tx desc arbiter */
3108 reg = rd32(hw, TXGBE_ARBTXCTL);
3109 reg |= TXGBE_ARBTXCTL_DIA;
3110 wr32(hw, TXGBE_ARBTXCTL, reg);
3112 /* Enable DCB for Tx with 8 TCs */
3113 reg = rd32(hw, TXGBE_PORTCTL);
3114 reg &= TXGBE_PORTCTL_NUMTC_MASK;
3115 reg |= TXGBE_PORTCTL_DCB;
3116 if (dcb_config->num_tcs.pg_tcs == 8)
3117 reg |= TXGBE_PORTCTL_NUMTC_8;
3119 reg |= TXGBE_PORTCTL_NUMTC_4;
3121 wr32(hw, TXGBE_PORTCTL, reg);
3123 /* Enable the Tx desc arbiter */
3124 reg = rd32(hw, TXGBE_ARBTXCTL);
3125 reg &= ~TXGBE_ARBTXCTL_DIA;
3126 wr32(hw, TXGBE_ARBTXCTL, reg);
3130 * txgbe_vmdq_dcb_hw_tx_config - Configure general VMDQ+DCB TX parameters
3131 * @dev: pointer to rte_eth_dev structure
3132 * @dcb_config: pointer to txgbe_dcb_config structure
3135 txgbe_vmdq_dcb_hw_tx_config(struct rte_eth_dev *dev,
3136 struct txgbe_dcb_config *dcb_config)
3138 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3139 &dev->data->dev_conf.tx_adv_conf.vmdq_dcb_tx_conf;
3140 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
3142 PMD_INIT_FUNC_TRACE();
3143 /*PF VF Transmit Enable*/
3144 wr32(hw, TXGBE_POOLTXENA(0),
3145 vmdq_tx_conf->nb_queue_pools ==
3146 ETH_16_POOLS ? 0xFFFF : 0xFFFFFFFF);
3148 /*Configure general DCB TX parameters*/
3149 txgbe_dcb_tx_hw_config(dev, dcb_config);
3153 txgbe_vmdq_dcb_rx_config(struct rte_eth_dev *dev,
3154 struct txgbe_dcb_config *dcb_config)
3156 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3157 &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf;
3158 struct txgbe_dcb_tc_config *tc;
3161 /* convert rte_eth_conf.rx_adv_conf to struct txgbe_dcb_config */
3162 if (vmdq_rx_conf->nb_queue_pools == ETH_16_POOLS) {
3163 dcb_config->num_tcs.pg_tcs = ETH_8_TCS;
3164 dcb_config->num_tcs.pfc_tcs = ETH_8_TCS;
3166 dcb_config->num_tcs.pg_tcs = ETH_4_TCS;
3167 dcb_config->num_tcs.pfc_tcs = ETH_4_TCS;
3170 /* Initialize User Priority to Traffic Class mapping */
3171 for (j = 0; j < TXGBE_DCB_TC_MAX; j++) {
3172 tc = &dcb_config->tc_config[j];
3173 tc->path[TXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0;
3176 /* User Priority to Traffic Class mapping */
3177 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3178 j = vmdq_rx_conf->dcb_tc[i];
3179 tc = &dcb_config->tc_config[j];
3180 tc->path[TXGBE_DCB_RX_CONFIG].up_to_tc_bitmap |=
3186 txgbe_dcb_vt_tx_config(struct rte_eth_dev *dev,
3187 struct txgbe_dcb_config *dcb_config)
3189 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3190 &dev->data->dev_conf.tx_adv_conf.vmdq_dcb_tx_conf;
3191 struct txgbe_dcb_tc_config *tc;
3194 /* convert rte_eth_conf.rx_adv_conf to struct txgbe_dcb_config */
3195 if (vmdq_tx_conf->nb_queue_pools == ETH_16_POOLS) {
3196 dcb_config->num_tcs.pg_tcs = ETH_8_TCS;
3197 dcb_config->num_tcs.pfc_tcs = ETH_8_TCS;
3199 dcb_config->num_tcs.pg_tcs = ETH_4_TCS;
3200 dcb_config->num_tcs.pfc_tcs = ETH_4_TCS;
3203 /* Initialize User Priority to Traffic Class mapping */
3204 for (j = 0; j < TXGBE_DCB_TC_MAX; j++) {
3205 tc = &dcb_config->tc_config[j];
3206 tc->path[TXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0;
3209 /* User Priority to Traffic Class mapping */
3210 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3211 j = vmdq_tx_conf->dcb_tc[i];
3212 tc = &dcb_config->tc_config[j];
3213 tc->path[TXGBE_DCB_TX_CONFIG].up_to_tc_bitmap |=
3219 txgbe_dcb_rx_config(struct rte_eth_dev *dev,
3220 struct txgbe_dcb_config *dcb_config)
3222 struct rte_eth_dcb_rx_conf *rx_conf =
3223 &dev->data->dev_conf.rx_adv_conf.dcb_rx_conf;
3224 struct txgbe_dcb_tc_config *tc;
3227 dcb_config->num_tcs.pg_tcs = (uint8_t)rx_conf->nb_tcs;
3228 dcb_config->num_tcs.pfc_tcs = (uint8_t)rx_conf->nb_tcs;
3230 /* Initialize User Priority to Traffic Class mapping */
3231 for (j = 0; j < TXGBE_DCB_TC_MAX; j++) {
3232 tc = &dcb_config->tc_config[j];
3233 tc->path[TXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0;
3236 /* User Priority to Traffic Class mapping */
3237 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3238 j = rx_conf->dcb_tc[i];
3239 tc = &dcb_config->tc_config[j];
3240 tc->path[TXGBE_DCB_RX_CONFIG].up_to_tc_bitmap |=
3246 txgbe_dcb_tx_config(struct rte_eth_dev *dev,
3247 struct txgbe_dcb_config *dcb_config)
3249 struct rte_eth_dcb_tx_conf *tx_conf =
3250 &dev->data->dev_conf.tx_adv_conf.dcb_tx_conf;
3251 struct txgbe_dcb_tc_config *tc;
3254 dcb_config->num_tcs.pg_tcs = (uint8_t)tx_conf->nb_tcs;
3255 dcb_config->num_tcs.pfc_tcs = (uint8_t)tx_conf->nb_tcs;
3257 /* Initialize User Priority to Traffic Class mapping */
3258 for (j = 0; j < TXGBE_DCB_TC_MAX; j++) {
3259 tc = &dcb_config->tc_config[j];
3260 tc->path[TXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0;
3263 /* User Priority to Traffic Class mapping */
3264 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3265 j = tx_conf->dcb_tc[i];
3266 tc = &dcb_config->tc_config[j];
3267 tc->path[TXGBE_DCB_TX_CONFIG].up_to_tc_bitmap |=
3273 * txgbe_dcb_rx_hw_config - Configure general DCB RX HW parameters
3274 * @dev: pointer to eth_dev structure
3275 * @dcb_config: pointer to txgbe_dcb_config structure
3278 txgbe_dcb_rx_hw_config(struct rte_eth_dev *dev,
3279 struct txgbe_dcb_config *dcb_config)
3285 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
3287 PMD_INIT_FUNC_TRACE();
3289 * Disable the arbiter before changing parameters
3290 * (always enable recycle mode; WSP)
3292 reg = TXGBE_ARBRXCTL_RRM | TXGBE_ARBRXCTL_WSP | TXGBE_ARBRXCTL_DIA;
3293 wr32(hw, TXGBE_ARBRXCTL, reg);
3295 reg = rd32(hw, TXGBE_PORTCTL);
3296 reg &= ~(TXGBE_PORTCTL_NUMTC_MASK | TXGBE_PORTCTL_NUMVT_MASK);
3297 if (dcb_config->num_tcs.pg_tcs == 4) {
3298 reg |= TXGBE_PORTCTL_NUMTC_4;
3299 if (dcb_config->vt_mode)
3300 reg |= TXGBE_PORTCTL_NUMVT_32;
3302 wr32(hw, TXGBE_POOLCTL, 0);
3305 if (dcb_config->num_tcs.pg_tcs == 8) {
3306 reg |= TXGBE_PORTCTL_NUMTC_8;
3307 if (dcb_config->vt_mode)
3308 reg |= TXGBE_PORTCTL_NUMVT_16;
3310 wr32(hw, TXGBE_POOLCTL, 0);
3313 wr32(hw, TXGBE_PORTCTL, reg);
3315 if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
3316 /* Disable drop for all queues in VMDQ mode*/
3317 for (q = 0; q < TXGBE_MAX_RX_QUEUE_NUM; q++) {
3318 u32 val = 1 << (q % 32);
3319 wr32m(hw, TXGBE_QPRXDROP(q / 32), val, val);
3322 /* Enable drop for all queues in SRIOV mode */
3323 for (q = 0; q < TXGBE_MAX_RX_QUEUE_NUM; q++) {
3324 u32 val = 1 << (q % 32);
3325 wr32m(hw, TXGBE_QPRXDROP(q / 32), val, val);
3329 /* VLNCTL: enable vlan filtering and allow all vlan tags through */
3330 vlanctrl = rd32(hw, TXGBE_VLANCTL);
3331 vlanctrl |= TXGBE_VLANCTL_VFE; /* enable vlan filters */
3332 wr32(hw, TXGBE_VLANCTL, vlanctrl);
3334 /* VLANTBL - enable all vlan filters */
3335 for (i = 0; i < NUM_VFTA_REGISTERS; i++)
3336 wr32(hw, TXGBE_VLANTBL(i), 0xFFFFFFFF);
3339 * Configure Rx packet plane (recycle mode; WSP) and
3342 reg = TXGBE_ARBRXCTL_RRM | TXGBE_ARBRXCTL_WSP;
3343 wr32(hw, TXGBE_ARBRXCTL, reg);
3347 txgbe_dcb_hw_arbite_rx_config(struct txgbe_hw *hw, uint16_t *refill,
3348 uint16_t *max, uint8_t *bwg_id, uint8_t *tsa, uint8_t *map)
3350 txgbe_dcb_config_rx_arbiter_raptor(hw, refill, max, bwg_id,
3355 txgbe_dcb_hw_arbite_tx_config(struct txgbe_hw *hw, uint16_t *refill,
3356 uint16_t *max, uint8_t *bwg_id, uint8_t *tsa, uint8_t *map)
3358 switch (hw->mac.type) {
3359 case txgbe_mac_raptor:
3360 txgbe_dcb_config_tx_desc_arbiter_raptor(hw, refill,
3362 txgbe_dcb_config_tx_data_arbiter_raptor(hw, refill,
3363 max, bwg_id, tsa, map);
3370 #define DCB_RX_CONFIG 1
3371 #define DCB_TX_CONFIG 1
3372 #define DCB_TX_PB 1024
3374 * txgbe_dcb_hw_configure - Enable DCB and configure
3375 * general DCB in VT mode and non-VT mode parameters
3376 * @dev: pointer to rte_eth_dev structure
3377 * @dcb_config: pointer to txgbe_dcb_config structure
3380 txgbe_dcb_hw_configure(struct rte_eth_dev *dev,
3381 struct txgbe_dcb_config *dcb_config)
3384 uint8_t i, pfc_en, nb_tcs;
3385 uint16_t pbsize, rx_buffer_size;
3386 uint8_t config_dcb_rx = 0;
3387 uint8_t config_dcb_tx = 0;
3388 uint8_t tsa[TXGBE_DCB_TC_MAX] = {0};
3389 uint8_t bwgid[TXGBE_DCB_TC_MAX] = {0};
3390 uint16_t refill[TXGBE_DCB_TC_MAX] = {0};
3391 uint16_t max[TXGBE_DCB_TC_MAX] = {0};
3392 uint8_t map[TXGBE_DCB_TC_MAX] = {0};
3393 struct txgbe_dcb_tc_config *tc;
3394 uint32_t max_frame = dev->data->mtu +
3395 RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN;
3396 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
3397 struct txgbe_bw_conf *bw_conf = TXGBE_DEV_BW_CONF(dev);
3399 switch (dev->data->dev_conf.rxmode.mq_mode) {
3400 case ETH_MQ_RX_VMDQ_DCB:
3401 dcb_config->vt_mode = true;
3402 config_dcb_rx = DCB_RX_CONFIG;
3404 * get dcb and VT rx configuration parameters
3407 txgbe_vmdq_dcb_rx_config(dev, dcb_config);
3408 /*Configure general VMDQ and DCB RX parameters*/
3409 txgbe_vmdq_dcb_configure(dev);
3412 case ETH_MQ_RX_DCB_RSS:
3413 dcb_config->vt_mode = false;
3414 config_dcb_rx = DCB_RX_CONFIG;
3415 /* Get dcb TX configuration parameters from rte_eth_conf */
3416 txgbe_dcb_rx_config(dev, dcb_config);
3417 /*Configure general DCB RX parameters*/
3418 txgbe_dcb_rx_hw_config(dev, dcb_config);
3421 PMD_INIT_LOG(ERR, "Incorrect DCB RX mode configuration");
3424 switch (dev->data->dev_conf.txmode.mq_mode) {
3425 case ETH_MQ_TX_VMDQ_DCB:
3426 dcb_config->vt_mode = true;
3427 config_dcb_tx = DCB_TX_CONFIG;
3428 /* get DCB and VT TX configuration parameters
3431 txgbe_dcb_vt_tx_config(dev, dcb_config);
3432 /* Configure general VMDQ and DCB TX parameters */
3433 txgbe_vmdq_dcb_hw_tx_config(dev, dcb_config);
3437 dcb_config->vt_mode = false;
3438 config_dcb_tx = DCB_TX_CONFIG;
3439 /* get DCB TX configuration parameters from rte_eth_conf */
3440 txgbe_dcb_tx_config(dev, dcb_config);
3441 /* Configure general DCB TX parameters */
3442 txgbe_dcb_tx_hw_config(dev, dcb_config);
3445 PMD_INIT_LOG(ERR, "Incorrect DCB TX mode configuration");
3449 nb_tcs = dcb_config->num_tcs.pfc_tcs;
3451 txgbe_dcb_unpack_map_cee(dcb_config, TXGBE_DCB_RX_CONFIG, map);
3452 if (nb_tcs == ETH_4_TCS) {
3453 /* Avoid un-configured priority mapping to TC0 */
3455 uint8_t mask = 0xFF;
3457 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES - 4; i++)
3458 mask = (uint8_t)(mask & (~(1 << map[i])));
3459 for (i = 0; mask && (i < TXGBE_DCB_TC_MAX); i++) {
3460 if ((mask & 0x1) && j < ETH_DCB_NUM_USER_PRIORITIES)
3464 /* Re-configure 4 TCs BW */
3465 for (i = 0; i < nb_tcs; i++) {
3466 tc = &dcb_config->tc_config[i];
3467 if (bw_conf->tc_num != nb_tcs)
3468 tc->path[TXGBE_DCB_TX_CONFIG].bwg_percent =
3469 (uint8_t)(100 / nb_tcs);
3470 tc->path[TXGBE_DCB_RX_CONFIG].bwg_percent =
3471 (uint8_t)(100 / nb_tcs);
3473 for (; i < TXGBE_DCB_TC_MAX; i++) {
3474 tc = &dcb_config->tc_config[i];
3475 tc->path[TXGBE_DCB_TX_CONFIG].bwg_percent = 0;
3476 tc->path[TXGBE_DCB_RX_CONFIG].bwg_percent = 0;
3479 /* Re-configure 8 TCs BW */
3480 for (i = 0; i < nb_tcs; i++) {
3481 tc = &dcb_config->tc_config[i];
3482 if (bw_conf->tc_num != nb_tcs)
3483 tc->path[TXGBE_DCB_TX_CONFIG].bwg_percent =
3484 (uint8_t)(100 / nb_tcs + (i & 1));
3485 tc->path[TXGBE_DCB_RX_CONFIG].bwg_percent =
3486 (uint8_t)(100 / nb_tcs + (i & 1));
3490 rx_buffer_size = NIC_RX_BUFFER_SIZE;
3492 if (config_dcb_rx) {
3493 /* Set RX buffer size */
3494 pbsize = (uint16_t)(rx_buffer_size / nb_tcs);
3495 uint32_t rxpbsize = pbsize << 10;
3497 for (i = 0; i < nb_tcs; i++)
3498 wr32(hw, TXGBE_PBRXSIZE(i), rxpbsize);
3500 /* zero alloc all unused TCs */
3501 for (; i < ETH_DCB_NUM_USER_PRIORITIES; i++)
3502 wr32(hw, TXGBE_PBRXSIZE(i), 0);
3504 if (config_dcb_tx) {
3505 /* Only support an equally distributed
3506 * Tx packet buffer strategy.
3508 uint32_t txpktsize = TXGBE_PBTXSIZE_MAX / nb_tcs;
3509 uint32_t txpbthresh = (txpktsize / DCB_TX_PB) -
3510 TXGBE_TXPKT_SIZE_MAX;
3512 for (i = 0; i < nb_tcs; i++) {
3513 wr32(hw, TXGBE_PBTXSIZE(i), txpktsize);
3514 wr32(hw, TXGBE_PBTXDMATH(i), txpbthresh);
3516 /* Clear unused TCs, if any, to zero buffer size*/
3517 for (; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3518 wr32(hw, TXGBE_PBTXSIZE(i), 0);
3519 wr32(hw, TXGBE_PBTXDMATH(i), 0);
3523 /*Calculates traffic class credits*/
3524 txgbe_dcb_calculate_tc_credits_cee(hw, dcb_config, max_frame,
3525 TXGBE_DCB_TX_CONFIG);
3526 txgbe_dcb_calculate_tc_credits_cee(hw, dcb_config, max_frame,
3527 TXGBE_DCB_RX_CONFIG);
3529 if (config_dcb_rx) {
3530 /* Unpack CEE standard containers */
3531 txgbe_dcb_unpack_refill_cee(dcb_config,
3532 TXGBE_DCB_RX_CONFIG, refill);
3533 txgbe_dcb_unpack_max_cee(dcb_config, max);
3534 txgbe_dcb_unpack_bwgid_cee(dcb_config,
3535 TXGBE_DCB_RX_CONFIG, bwgid);
3536 txgbe_dcb_unpack_tsa_cee(dcb_config,
3537 TXGBE_DCB_RX_CONFIG, tsa);
3538 /* Configure PG(ETS) RX */
3539 txgbe_dcb_hw_arbite_rx_config(hw, refill, max, bwgid, tsa, map);
3542 if (config_dcb_tx) {
3543 /* Unpack CEE standard containers */
3544 txgbe_dcb_unpack_refill_cee(dcb_config,
3545 TXGBE_DCB_TX_CONFIG, refill);
3546 txgbe_dcb_unpack_max_cee(dcb_config, max);
3547 txgbe_dcb_unpack_bwgid_cee(dcb_config,
3548 TXGBE_DCB_TX_CONFIG, bwgid);
3549 txgbe_dcb_unpack_tsa_cee(dcb_config,
3550 TXGBE_DCB_TX_CONFIG, tsa);
3551 /* Configure PG(ETS) TX */
3552 txgbe_dcb_hw_arbite_tx_config(hw, refill, max, bwgid, tsa, map);
3555 /* Configure queue statistics registers */
3556 txgbe_dcb_config_tc_stats_raptor(hw, dcb_config);
3558 /* Check if the PFC is supported */
3559 if (dev->data->dev_conf.dcb_capability_en & ETH_DCB_PFC_SUPPORT) {
3560 pbsize = (uint16_t)(rx_buffer_size / nb_tcs);
3561 for (i = 0; i < nb_tcs; i++) {
3562 /* If the TC count is 8,
3563 * and the default high_water is 48,
3564 * the low_water is 16 as default.
3566 hw->fc.high_water[i] = (pbsize * 3) / 4;
3567 hw->fc.low_water[i] = pbsize / 4;
3568 /* Enable pfc for this TC */
3569 tc = &dcb_config->tc_config[i];
3570 tc->pfc = txgbe_dcb_pfc_enabled;
3572 txgbe_dcb_unpack_pfc_cee(dcb_config, map, &pfc_en);
3573 if (dcb_config->num_tcs.pfc_tcs == ETH_4_TCS)
3575 ret = txgbe_dcb_config_pfc(hw, pfc_en, map);
3581 void txgbe_configure_pb(struct rte_eth_dev *dev)
3583 struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
3584 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
3587 int tc = dev_conf->rx_adv_conf.dcb_rx_conf.nb_tcs;
3589 /* Reserve 256KB(/512KB) rx buffer for fdir */
3592 hw->mac.setup_pba(hw, tc, hdrm, PBA_STRATEGY_EQUAL);
3595 void txgbe_configure_port(struct rte_eth_dev *dev)
3597 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
3599 uint16_t tpids[8] = {RTE_ETHER_TYPE_VLAN, RTE_ETHER_TYPE_QINQ,
3604 PMD_INIT_FUNC_TRACE();
3606 /* default outer vlan tpid */
3607 wr32(hw, TXGBE_EXTAG,
3608 TXGBE_EXTAG_ETAG(RTE_ETHER_TYPE_ETAG) |
3609 TXGBE_EXTAG_VLAN(RTE_ETHER_TYPE_QINQ));
3611 /* default inner vlan tpid */
3612 wr32m(hw, TXGBE_VLANCTL,
3613 TXGBE_VLANCTL_TPID_MASK,
3614 TXGBE_VLANCTL_TPID(RTE_ETHER_TYPE_VLAN));
3615 wr32m(hw, TXGBE_DMATXCTRL,
3616 TXGBE_DMATXCTRL_TPID_MASK,
3617 TXGBE_DMATXCTRL_TPID(RTE_ETHER_TYPE_VLAN));
3619 /* default vlan tpid filters */
3620 for (i = 0; i < 8; i++) {
3621 wr32m(hw, TXGBE_TAGTPID(i / 2),
3622 (i % 2 ? TXGBE_TAGTPID_MSB_MASK
3623 : TXGBE_TAGTPID_LSB_MASK),
3624 (i % 2 ? TXGBE_TAGTPID_MSB(tpids[i])
3625 : TXGBE_TAGTPID_LSB(tpids[i])));
3628 /* default vxlan port */
3629 wr32(hw, TXGBE_VXLANPORT, 4789);
3633 * txgbe_configure_dcb - Configure DCB Hardware
3634 * @dev: pointer to rte_eth_dev
3636 void txgbe_configure_dcb(struct rte_eth_dev *dev)
3638 struct txgbe_dcb_config *dcb_cfg = TXGBE_DEV_DCB_CONFIG(dev);
3639 struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
3641 PMD_INIT_FUNC_TRACE();
3643 /* check support mq_mode for DCB */
3644 if (dev_conf->rxmode.mq_mode != ETH_MQ_RX_VMDQ_DCB &&
3645 dev_conf->rxmode.mq_mode != ETH_MQ_RX_DCB &&
3646 dev_conf->rxmode.mq_mode != ETH_MQ_RX_DCB_RSS)
3649 if (dev->data->nb_rx_queues > ETH_DCB_NUM_QUEUES)
3652 /** Configure DCB hardware **/
3653 txgbe_dcb_hw_configure(dev, dcb_cfg);
3657 * VMDq only support for 10 GbE NIC.
3660 txgbe_vmdq_rx_hw_configure(struct rte_eth_dev *dev)
3662 struct rte_eth_vmdq_rx_conf *cfg;
3663 struct txgbe_hw *hw;
3664 enum rte_eth_nb_pools num_pools;
3665 uint32_t mrqc, vt_ctl, vlanctrl;
3669 PMD_INIT_FUNC_TRACE();
3670 hw = TXGBE_DEV_HW(dev);
3671 cfg = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
3672 num_pools = cfg->nb_queue_pools;
3674 txgbe_rss_disable(dev);
3677 mrqc = TXGBE_PORTCTL_NUMVT_64;
3678 wr32m(hw, TXGBE_PORTCTL, TXGBE_PORTCTL_NUMVT_MASK, mrqc);
3680 /* turn on virtualisation and set the default pool */
3681 vt_ctl = TXGBE_POOLCTL_RPLEN;
3682 if (cfg->enable_default_pool)
3683 vt_ctl |= TXGBE_POOLCTL_DEFPL(cfg->default_pool);
3685 vt_ctl |= TXGBE_POOLCTL_DEFDSA;
3687 wr32(hw, TXGBE_POOLCTL, vt_ctl);
3689 for (i = 0; i < (int)num_pools; i++) {
3690 vmolr = txgbe_convert_vm_rx_mask_to_val(cfg->rx_mode, vmolr);
3691 wr32(hw, TXGBE_POOLETHCTL(i), vmolr);
3694 /* enable vlan filtering and allow all vlan tags through */
3695 vlanctrl = rd32(hw, TXGBE_VLANCTL);
3696 vlanctrl |= TXGBE_VLANCTL_VFE; /* enable vlan filters */
3697 wr32(hw, TXGBE_VLANCTL, vlanctrl);
3699 /* enable all vlan filters */
3700 for (i = 0; i < NUM_VFTA_REGISTERS; i++)
3701 wr32(hw, TXGBE_VLANTBL(i), UINT32_MAX);
3703 /* pool enabling for receive - 64 */
3704 wr32(hw, TXGBE_POOLRXENA(0), UINT32_MAX);
3705 if (num_pools == ETH_64_POOLS)
3706 wr32(hw, TXGBE_POOLRXENA(1), UINT32_MAX);
3709 * allow pools to read specific mac addresses
3710 * In this case, all pools should be able to read from mac addr 0
3712 wr32(hw, TXGBE_ETHADDRIDX, 0);
3713 wr32(hw, TXGBE_ETHADDRASSL, 0xFFFFFFFF);
3714 wr32(hw, TXGBE_ETHADDRASSH, 0xFFFFFFFF);
3716 /* set up filters for vlan tags as configured */
3717 for (i = 0; i < cfg->nb_pool_maps; i++) {
3718 /* set vlan id in VF register and set the valid bit */
3719 wr32(hw, TXGBE_PSRVLANIDX, i);
3720 wr32(hw, TXGBE_PSRVLAN, (TXGBE_PSRVLAN_EA |
3721 TXGBE_PSRVLAN_VID(cfg->pool_map[i].vlan_id)));
3723 * Put the allowed pools in VFB reg. As we only have 16 or 64
3724 * pools, we only need to use the first half of the register
3727 if (((cfg->pool_map[i].pools >> 32) & UINT32_MAX) == 0)
3728 wr32(hw, TXGBE_PSRVLANPLM(0),
3729 (cfg->pool_map[i].pools & UINT32_MAX));
3731 wr32(hw, TXGBE_PSRVLANPLM(1),
3732 ((cfg->pool_map[i].pools >> 32) & UINT32_MAX));
3735 /* Tx General Switch Control Enables VMDQ loopback */
3736 if (cfg->enable_loop_back) {
3737 wr32(hw, TXGBE_PSRCTL, TXGBE_PSRCTL_LBENA);
3738 for (i = 0; i < 64; i++)
3739 wr32m(hw, TXGBE_POOLETHCTL(i),
3740 TXGBE_POOLETHCTL_LLB, TXGBE_POOLETHCTL_LLB);
3747 * txgbe_vmdq_tx_hw_configure - Configure general VMDq TX parameters
3748 * @hw: pointer to hardware structure
3751 txgbe_vmdq_tx_hw_configure(struct txgbe_hw *hw)
3756 PMD_INIT_FUNC_TRACE();
3757 /*PF VF Transmit Enable*/
3758 wr32(hw, TXGBE_POOLTXENA(0), UINT32_MAX);
3759 wr32(hw, TXGBE_POOLTXENA(1), UINT32_MAX);
3761 /* Disable the Tx desc arbiter */
3762 reg = rd32(hw, TXGBE_ARBTXCTL);
3763 reg |= TXGBE_ARBTXCTL_DIA;
3764 wr32(hw, TXGBE_ARBTXCTL, reg);
3766 wr32m(hw, TXGBE_PORTCTL, TXGBE_PORTCTL_NUMVT_MASK,
3767 TXGBE_PORTCTL_NUMVT_64);
3769 /* Disable drop for all queues */
3770 for (q = 0; q < 128; q++) {
3771 u32 val = 1 << (q % 32);
3772 wr32m(hw, TXGBE_QPRXDROP(q / 32), val, val);
3775 /* Enable the Tx desc arbiter */
3776 reg = rd32(hw, TXGBE_ARBTXCTL);
3777 reg &= ~TXGBE_ARBTXCTL_DIA;
3778 wr32(hw, TXGBE_ARBTXCTL, reg);
3783 static int __rte_cold
3784 txgbe_alloc_rx_queue_mbufs(struct txgbe_rx_queue *rxq)
3786 struct txgbe_rx_entry *rxe = rxq->sw_ring;
3790 /* Initialize software ring entries */
3791 for (i = 0; i < rxq->nb_rx_desc; i++) {
3792 volatile struct txgbe_rx_desc *rxd;
3793 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mb_pool);
3796 PMD_INIT_LOG(ERR, "RX mbuf alloc failed queue_id=%u",
3797 (unsigned int)rxq->queue_id);
3801 mbuf->data_off = RTE_PKTMBUF_HEADROOM;
3802 mbuf->port = rxq->port_id;
3805 rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf));
3806 rxd = &rxq->rx_ring[i];
3807 TXGBE_RXD_HDRADDR(rxd, 0);
3808 TXGBE_RXD_PKTADDR(rxd, dma_addr);
3816 txgbe_config_vf_rss(struct rte_eth_dev *dev)
3818 struct txgbe_hw *hw;
3821 txgbe_rss_configure(dev);
3823 hw = TXGBE_DEV_HW(dev);
3826 mrqc = rd32(hw, TXGBE_PORTCTL);
3827 mrqc &= ~(TXGBE_PORTCTL_NUMTC_MASK | TXGBE_PORTCTL_NUMVT_MASK);
3828 switch (RTE_ETH_DEV_SRIOV(dev).active) {
3830 mrqc |= TXGBE_PORTCTL_NUMVT_64;
3834 mrqc |= TXGBE_PORTCTL_NUMVT_32;
3838 PMD_INIT_LOG(ERR, "Invalid pool number in IOV mode with VMDQ RSS");
3842 wr32(hw, TXGBE_PORTCTL, mrqc);
3848 txgbe_config_vf_default(struct rte_eth_dev *dev)
3850 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
3853 mrqc = rd32(hw, TXGBE_PORTCTL);
3854 mrqc &= ~(TXGBE_PORTCTL_NUMTC_MASK | TXGBE_PORTCTL_NUMVT_MASK);
3855 switch (RTE_ETH_DEV_SRIOV(dev).active) {
3857 mrqc |= TXGBE_PORTCTL_NUMVT_64;
3861 mrqc |= TXGBE_PORTCTL_NUMVT_32;
3865 mrqc |= TXGBE_PORTCTL_NUMVT_16;
3869 "invalid pool number in IOV mode");
3873 wr32(hw, TXGBE_PORTCTL, mrqc);
3879 txgbe_dev_mq_rx_configure(struct rte_eth_dev *dev)
3881 if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
3883 * SRIOV inactive scheme
3884 * any DCB/RSS w/o VMDq multi-queue setting
3886 switch (dev->data->dev_conf.rxmode.mq_mode) {
3888 case ETH_MQ_RX_DCB_RSS:
3889 case ETH_MQ_RX_VMDQ_RSS:
3890 txgbe_rss_configure(dev);
3893 case ETH_MQ_RX_VMDQ_DCB:
3894 txgbe_vmdq_dcb_configure(dev);
3897 case ETH_MQ_RX_VMDQ_ONLY:
3898 txgbe_vmdq_rx_hw_configure(dev);
3901 case ETH_MQ_RX_NONE:
3903 /* if mq_mode is none, disable rss mode.*/
3904 txgbe_rss_disable(dev);
3908 /* SRIOV active scheme
3909 * Support RSS together with SRIOV.
3911 switch (dev->data->dev_conf.rxmode.mq_mode) {
3913 case ETH_MQ_RX_VMDQ_RSS:
3914 txgbe_config_vf_rss(dev);
3916 case ETH_MQ_RX_VMDQ_DCB:
3918 /* In SRIOV, the configuration is the same as VMDq case */
3919 txgbe_vmdq_dcb_configure(dev);
3921 /* DCB/RSS together with SRIOV is not supported */
3922 case ETH_MQ_RX_VMDQ_DCB_RSS:
3923 case ETH_MQ_RX_DCB_RSS:
3925 "Could not support DCB/RSS with VMDq & SRIOV");
3928 txgbe_config_vf_default(dev);
3937 txgbe_dev_mq_tx_configure(struct rte_eth_dev *dev)
3939 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
3943 /* disable arbiter */
3944 rttdcs = rd32(hw, TXGBE_ARBTXCTL);
3945 rttdcs |= TXGBE_ARBTXCTL_DIA;
3946 wr32(hw, TXGBE_ARBTXCTL, rttdcs);
3948 if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
3950 * SRIOV inactive scheme
3951 * any DCB w/o VMDq multi-queue setting
3953 if (dev->data->dev_conf.txmode.mq_mode == ETH_MQ_TX_VMDQ_ONLY)
3954 txgbe_vmdq_tx_hw_configure(hw);
3956 wr32m(hw, TXGBE_PORTCTL, TXGBE_PORTCTL_NUMVT_MASK, 0);
3958 switch (RTE_ETH_DEV_SRIOV(dev).active) {
3960 * SRIOV active scheme
3961 * FIXME if support DCB together with VMDq & SRIOV
3964 mtqc = TXGBE_PORTCTL_NUMVT_64;
3967 mtqc = TXGBE_PORTCTL_NUMVT_32;
3970 mtqc = TXGBE_PORTCTL_NUMVT_16;
3974 PMD_INIT_LOG(ERR, "invalid pool number in IOV mode");
3976 wr32m(hw, TXGBE_PORTCTL, TXGBE_PORTCTL_NUMVT_MASK, mtqc);
3979 /* re-enable arbiter */
3980 rttdcs &= ~TXGBE_ARBTXCTL_DIA;
3981 wr32(hw, TXGBE_ARBTXCTL, rttdcs);
3987 * txgbe_get_rscctl_maxdesc
3989 * @pool Memory pool of the Rx queue
3991 static inline uint32_t
3992 txgbe_get_rscctl_maxdesc(struct rte_mempool *pool)
3994 struct rte_pktmbuf_pool_private *mp_priv = rte_mempool_get_priv(pool);
3997 RTE_IPV4_MAX_PKT_LEN /
3998 (mp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM);
4001 return TXGBE_RXCFG_RSCMAX_16;
4002 else if (maxdesc >= 8)
4003 return TXGBE_RXCFG_RSCMAX_8;
4004 else if (maxdesc >= 4)
4005 return TXGBE_RXCFG_RSCMAX_4;
4007 return TXGBE_RXCFG_RSCMAX_1;
4011 * txgbe_set_rsc - configure RSC related port HW registers
4013 * Configures the port's RSC related registers.
4017 * Returns 0 in case of success or a non-zero error code
4020 txgbe_set_rsc(struct rte_eth_dev *dev)
4022 struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode;
4023 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
4024 struct rte_eth_dev_info dev_info = { 0 };
4025 bool rsc_capable = false;
4031 dev->dev_ops->dev_infos_get(dev, &dev_info);
4032 if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TCP_LRO)
4035 if (!rsc_capable && (rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO)) {
4036 PMD_INIT_LOG(CRIT, "LRO is requested on HW that doesn't "
4041 /* RSC global configuration */
4043 if ((rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC) &&
4044 (rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO)) {
4045 PMD_INIT_LOG(CRIT, "LRO can't be enabled when HW CRC "
4050 rfctl = rd32(hw, TXGBE_PSRCTL);
4051 if (rsc_capable && (rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO))
4052 rfctl &= ~TXGBE_PSRCTL_RSCDIA;
4054 rfctl |= TXGBE_PSRCTL_RSCDIA;
4055 wr32(hw, TXGBE_PSRCTL, rfctl);
4057 /* If LRO hasn't been requested - we are done here. */
4058 if (!(rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO))
4061 /* Set PSRCTL.RSCACK bit */
4062 rdrxctl = rd32(hw, TXGBE_PSRCTL);
4063 rdrxctl |= TXGBE_PSRCTL_RSCACK;
4064 wr32(hw, TXGBE_PSRCTL, rdrxctl);
4066 /* Per-queue RSC configuration */
4067 for (i = 0; i < dev->data->nb_rx_queues; i++) {
4068 struct txgbe_rx_queue *rxq = dev->data->rx_queues[i];
4070 rd32(hw, TXGBE_RXCFG(rxq->reg_idx));
4072 rd32(hw, TXGBE_POOLRSS(rxq->reg_idx));
4074 rd32(hw, TXGBE_ITR(rxq->reg_idx));
4077 * txgbe PMD doesn't support header-split at the moment.
4079 srrctl &= ~TXGBE_RXCFG_HDRLEN_MASK;
4080 srrctl |= TXGBE_RXCFG_HDRLEN(128);
4083 * TODO: Consider setting the Receive Descriptor Minimum
4084 * Threshold Size for an RSC case. This is not an obviously
4085 * beneficiary option but the one worth considering...
4088 srrctl |= TXGBE_RXCFG_RSCENA;
4089 srrctl &= ~TXGBE_RXCFG_RSCMAX_MASK;
4090 srrctl |= txgbe_get_rscctl_maxdesc(rxq->mb_pool);
4091 psrtype |= TXGBE_POOLRSS_L4HDR;
4094 * RSC: Set ITR interval corresponding to 2K ints/s.
4096 * Full-sized RSC aggregations for a 10Gb/s link will
4097 * arrive at about 20K aggregation/s rate.
4099 * 2K inst/s rate will make only 10% of the
4100 * aggregations to be closed due to the interrupt timer
4101 * expiration for a streaming at wire-speed case.
4103 * For a sparse streaming case this setting will yield
4104 * at most 500us latency for a single RSC aggregation.
4106 eitr &= ~TXGBE_ITR_IVAL_MASK;
4107 eitr |= TXGBE_ITR_IVAL_10G(TXGBE_QUEUE_ITR_INTERVAL_DEFAULT);
4108 eitr |= TXGBE_ITR_WRDSA;
4110 wr32(hw, TXGBE_RXCFG(rxq->reg_idx), srrctl);
4111 wr32(hw, TXGBE_POOLRSS(rxq->reg_idx), psrtype);
4112 wr32(hw, TXGBE_ITR(rxq->reg_idx), eitr);
4115 * RSC requires the mapping of the queue to the
4118 txgbe_set_ivar_map(hw, 0, rxq->reg_idx, i);
4123 PMD_INIT_LOG(DEBUG, "enabling LRO mode");
4129 txgbe_set_rx_function(struct rte_eth_dev *dev)
4132 struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev);
4135 * Initialize the appropriate LRO callback.
4137 * If all queues satisfy the bulk allocation preconditions
4138 * (adapter->rx_bulk_alloc_allowed is TRUE) then we may use
4139 * bulk allocation. Otherwise use a single allocation version.
4141 if (dev->data->lro) {
4142 if (adapter->rx_bulk_alloc_allowed) {
4143 PMD_INIT_LOG(DEBUG, "LRO is requested. Using a bulk "
4144 "allocation version");
4145 dev->rx_pkt_burst = txgbe_recv_pkts_lro_bulk_alloc;
4147 PMD_INIT_LOG(DEBUG, "LRO is requested. Using a single "
4148 "allocation version");
4149 dev->rx_pkt_burst = txgbe_recv_pkts_lro_single_alloc;
4151 } else if (dev->data->scattered_rx) {
4153 * Set the non-LRO scattered callback: there are bulk and
4154 * single allocation versions.
4156 if (adapter->rx_bulk_alloc_allowed) {
4157 PMD_INIT_LOG(DEBUG, "Using a Scattered with bulk "
4158 "allocation callback (port=%d).",
4159 dev->data->port_id);
4160 dev->rx_pkt_burst = txgbe_recv_pkts_lro_bulk_alloc;
4162 PMD_INIT_LOG(DEBUG, "Using Regular (non-vector, "
4163 "single allocation) "
4164 "Scattered Rx callback "
4166 dev->data->port_id);
4168 dev->rx_pkt_burst = txgbe_recv_pkts_lro_single_alloc;
4171 * Below we set "simple" callbacks according to port/queues parameters.
4172 * If parameters allow we are going to choose between the following
4175 * - Single buffer allocation (the simplest one)
4177 } else if (adapter->rx_bulk_alloc_allowed) {
4178 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are "
4179 "satisfied. Rx Burst Bulk Alloc function "
4180 "will be used on port=%d.",
4181 dev->data->port_id);
4183 dev->rx_pkt_burst = txgbe_recv_pkts_bulk_alloc;
4185 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are not "
4186 "satisfied, or Scattered Rx is requested "
4188 dev->data->port_id);
4190 dev->rx_pkt_burst = txgbe_recv_pkts;
4193 #ifdef RTE_LIB_SECURITY
4194 for (i = 0; i < dev->data->nb_rx_queues; i++) {
4195 struct txgbe_rx_queue *rxq = dev->data->rx_queues[i];
4197 rxq->using_ipsec = !!(dev->data->dev_conf.rxmode.offloads &
4198 DEV_RX_OFFLOAD_SECURITY);
4204 * Initializes Receive Unit.
4207 txgbe_dev_rx_init(struct rte_eth_dev *dev)
4209 struct txgbe_hw *hw;
4210 struct txgbe_rx_queue *rxq;
4219 struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode;
4222 PMD_INIT_FUNC_TRACE();
4223 hw = TXGBE_DEV_HW(dev);
4226 * Make sure receives are disabled while setting
4227 * up the RX context (registers, descriptor rings, etc.).
4229 wr32m(hw, TXGBE_MACRXCFG, TXGBE_MACRXCFG_ENA, 0);
4230 wr32m(hw, TXGBE_PBRXCTL, TXGBE_PBRXCTL_ENA, 0);
4232 /* Enable receipt of broadcasted frames */
4233 fctrl = rd32(hw, TXGBE_PSRCTL);
4234 fctrl |= TXGBE_PSRCTL_BCA;
4235 wr32(hw, TXGBE_PSRCTL, fctrl);
4238 * Configure CRC stripping, if any.
4240 hlreg0 = rd32(hw, TXGBE_SECRXCTL);
4241 if (rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC)
4242 hlreg0 &= ~TXGBE_SECRXCTL_CRCSTRIP;
4244 hlreg0 |= TXGBE_SECRXCTL_CRCSTRIP;
4245 wr32(hw, TXGBE_SECRXCTL, hlreg0);
4248 * Configure jumbo frame support, if any.
4250 if (rx_conf->offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) {
4251 wr32m(hw, TXGBE_FRMSZ, TXGBE_FRMSZ_MAX_MASK,
4252 TXGBE_FRMSZ_MAX(rx_conf->max_rx_pkt_len));
4254 wr32m(hw, TXGBE_FRMSZ, TXGBE_FRMSZ_MAX_MASK,
4255 TXGBE_FRMSZ_MAX(TXGBE_FRAME_SIZE_DFT));
4259 * If loopback mode is configured, set LPBK bit.
4261 hlreg0 = rd32(hw, TXGBE_PSRCTL);
4262 if (hw->mac.type == txgbe_mac_raptor &&
4263 dev->data->dev_conf.lpbk_mode)
4264 hlreg0 |= TXGBE_PSRCTL_LBENA;
4266 hlreg0 &= ~TXGBE_PSRCTL_LBENA;
4268 wr32(hw, TXGBE_PSRCTL, hlreg0);
4271 * Assume no header split and no VLAN strip support
4272 * on any Rx queue first .
4274 rx_conf->offloads &= ~DEV_RX_OFFLOAD_VLAN_STRIP;
4276 /* Setup RX queues */
4277 for (i = 0; i < dev->data->nb_rx_queues; i++) {
4278 rxq = dev->data->rx_queues[i];
4281 * Reset crc_len in case it was changed after queue setup by a
4282 * call to configure.
4284 if (rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC)
4285 rxq->crc_len = RTE_ETHER_CRC_LEN;
4289 /* Setup the Base and Length of the Rx Descriptor Rings */
4290 bus_addr = rxq->rx_ring_phys_addr;
4291 wr32(hw, TXGBE_RXBAL(rxq->reg_idx),
4292 (uint32_t)(bus_addr & BIT_MASK32));
4293 wr32(hw, TXGBE_RXBAH(rxq->reg_idx),
4294 (uint32_t)(bus_addr >> 32));
4295 wr32(hw, TXGBE_RXRP(rxq->reg_idx), 0);
4296 wr32(hw, TXGBE_RXWP(rxq->reg_idx), 0);
4298 srrctl = TXGBE_RXCFG_RNGLEN(rxq->nb_rx_desc);
4300 /* Set if packets are dropped when no descriptors available */
4302 srrctl |= TXGBE_RXCFG_DROP;
4305 * Configure the RX buffer size in the PKTLEN field of
4306 * the RXCFG register of the queue.
4307 * The value is in 1 KB resolution. Valid values can be from
4310 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) -
4311 RTE_PKTMBUF_HEADROOM);
4312 buf_size = ROUND_UP(buf_size, 0x1 << 10);
4313 srrctl |= TXGBE_RXCFG_PKTLEN(buf_size);
4315 wr32(hw, TXGBE_RXCFG(rxq->reg_idx), srrctl);
4317 /* It adds dual VLAN length for supporting dual VLAN */
4318 if (dev->data->dev_conf.rxmode.max_rx_pkt_len +
4319 2 * TXGBE_VLAN_TAG_SIZE > buf_size)
4320 dev->data->scattered_rx = 1;
4321 if (rxq->offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
4322 rx_conf->offloads |= DEV_RX_OFFLOAD_VLAN_STRIP;
4325 if (rx_conf->offloads & DEV_RX_OFFLOAD_SCATTER)
4326 dev->data->scattered_rx = 1;
4329 * Device configured with multiple RX queues.
4331 txgbe_dev_mq_rx_configure(dev);
4334 * Setup the Checksum Register.
4335 * Disable Full-Packet Checksum which is mutually exclusive with RSS.
4336 * Enable IP/L4 checksum computation by hardware if requested to do so.
4338 rxcsum = rd32(hw, TXGBE_PSRCTL);
4339 rxcsum |= TXGBE_PSRCTL_PCSD;
4340 if (rx_conf->offloads & DEV_RX_OFFLOAD_CHECKSUM)
4341 rxcsum |= TXGBE_PSRCTL_L4CSUM;
4343 rxcsum &= ~TXGBE_PSRCTL_L4CSUM;
4345 wr32(hw, TXGBE_PSRCTL, rxcsum);
4347 if (hw->mac.type == txgbe_mac_raptor) {
4348 rdrxctl = rd32(hw, TXGBE_SECRXCTL);
4349 if (rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC)
4350 rdrxctl &= ~TXGBE_SECRXCTL_CRCSTRIP;
4352 rdrxctl |= TXGBE_SECRXCTL_CRCSTRIP;
4353 wr32(hw, TXGBE_SECRXCTL, rdrxctl);
4356 rc = txgbe_set_rsc(dev);
4360 txgbe_set_rx_function(dev);
4366 * Initializes Transmit Unit.
4369 txgbe_dev_tx_init(struct rte_eth_dev *dev)
4371 struct txgbe_hw *hw;
4372 struct txgbe_tx_queue *txq;
4376 PMD_INIT_FUNC_TRACE();
4377 hw = TXGBE_DEV_HW(dev);
4379 /* Setup the Base and Length of the Tx Descriptor Rings */
4380 for (i = 0; i < dev->data->nb_tx_queues; i++) {
4381 txq = dev->data->tx_queues[i];
4383 bus_addr = txq->tx_ring_phys_addr;
4384 wr32(hw, TXGBE_TXBAL(txq->reg_idx),
4385 (uint32_t)(bus_addr & BIT_MASK32));
4386 wr32(hw, TXGBE_TXBAH(txq->reg_idx),
4387 (uint32_t)(bus_addr >> 32));
4388 wr32m(hw, TXGBE_TXCFG(txq->reg_idx), TXGBE_TXCFG_BUFLEN_MASK,
4389 TXGBE_TXCFG_BUFLEN(txq->nb_tx_desc));
4390 /* Setup the HW Tx Head and TX Tail descriptor pointers */
4391 wr32(hw, TXGBE_TXRP(txq->reg_idx), 0);
4392 wr32(hw, TXGBE_TXWP(txq->reg_idx), 0);
4395 /* Device configured with multiple TX queues. */
4396 txgbe_dev_mq_tx_configure(dev);
4400 * Set up link loopback mode Tx->Rx.
4402 static inline void __rte_cold
4403 txgbe_setup_loopback_link_raptor(struct txgbe_hw *hw)
4405 PMD_INIT_FUNC_TRACE();
4407 wr32m(hw, TXGBE_MACRXCFG, TXGBE_MACRXCFG_LB, TXGBE_MACRXCFG_LB);
4413 * Start Transmit and Receive Units.
4416 txgbe_dev_rxtx_start(struct rte_eth_dev *dev)
4418 struct txgbe_hw *hw;
4419 struct txgbe_tx_queue *txq;
4420 struct txgbe_rx_queue *rxq;
4426 PMD_INIT_FUNC_TRACE();
4427 hw = TXGBE_DEV_HW(dev);
4429 for (i = 0; i < dev->data->nb_tx_queues; i++) {
4430 txq = dev->data->tx_queues[i];
4431 /* Setup Transmit Threshold Registers */
4432 wr32m(hw, TXGBE_TXCFG(txq->reg_idx),
4433 TXGBE_TXCFG_HTHRESH_MASK |
4434 TXGBE_TXCFG_WTHRESH_MASK,
4435 TXGBE_TXCFG_HTHRESH(txq->hthresh) |
4436 TXGBE_TXCFG_WTHRESH(txq->wthresh));
4439 dmatxctl = rd32(hw, TXGBE_DMATXCTRL);
4440 dmatxctl |= TXGBE_DMATXCTRL_ENA;
4441 wr32(hw, TXGBE_DMATXCTRL, dmatxctl);
4443 for (i = 0; i < dev->data->nb_tx_queues; i++) {
4444 txq = dev->data->tx_queues[i];
4445 if (!txq->tx_deferred_start) {
4446 ret = txgbe_dev_tx_queue_start(dev, i);
4452 for (i = 0; i < dev->data->nb_rx_queues; i++) {
4453 rxq = dev->data->rx_queues[i];
4454 if (!rxq->rx_deferred_start) {
4455 ret = txgbe_dev_rx_queue_start(dev, i);
4461 /* Enable Receive engine */
4462 rxctrl = rd32(hw, TXGBE_PBRXCTL);
4463 rxctrl |= TXGBE_PBRXCTL_ENA;
4464 hw->mac.enable_rx_dma(hw, rxctrl);
4466 /* If loopback mode is enabled, set up the link accordingly */
4467 if (hw->mac.type == txgbe_mac_raptor &&
4468 dev->data->dev_conf.lpbk_mode)
4469 txgbe_setup_loopback_link_raptor(hw);
4471 #ifdef RTE_LIB_SECURITY
4472 if ((dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_SECURITY) ||
4473 (dev->data->dev_conf.txmode.offloads & DEV_TX_OFFLOAD_SECURITY)) {
4474 ret = txgbe_crypto_enable_ipsec(dev);
4477 "txgbe_crypto_enable_ipsec fails with %d.",
4488 txgbe_dev_save_rx_queue(struct txgbe_hw *hw, uint16_t rx_queue_id)
4490 u32 *reg = &hw->q_rx_regs[rx_queue_id * 8];
4491 *(reg++) = rd32(hw, TXGBE_RXBAL(rx_queue_id));
4492 *(reg++) = rd32(hw, TXGBE_RXBAH(rx_queue_id));
4493 *(reg++) = rd32(hw, TXGBE_RXCFG(rx_queue_id));
4497 txgbe_dev_store_rx_queue(struct txgbe_hw *hw, uint16_t rx_queue_id)
4499 u32 *reg = &hw->q_rx_regs[rx_queue_id * 8];
4500 wr32(hw, TXGBE_RXBAL(rx_queue_id), *(reg++));
4501 wr32(hw, TXGBE_RXBAH(rx_queue_id), *(reg++));
4502 wr32(hw, TXGBE_RXCFG(rx_queue_id), *(reg++) & ~TXGBE_RXCFG_ENA);
4506 txgbe_dev_save_tx_queue(struct txgbe_hw *hw, uint16_t tx_queue_id)
4508 u32 *reg = &hw->q_tx_regs[tx_queue_id * 8];
4509 *(reg++) = rd32(hw, TXGBE_TXBAL(tx_queue_id));
4510 *(reg++) = rd32(hw, TXGBE_TXBAH(tx_queue_id));
4511 *(reg++) = rd32(hw, TXGBE_TXCFG(tx_queue_id));
4515 txgbe_dev_store_tx_queue(struct txgbe_hw *hw, uint16_t tx_queue_id)
4517 u32 *reg = &hw->q_tx_regs[tx_queue_id * 8];
4518 wr32(hw, TXGBE_TXBAL(tx_queue_id), *(reg++));
4519 wr32(hw, TXGBE_TXBAH(tx_queue_id), *(reg++));
4520 wr32(hw, TXGBE_TXCFG(tx_queue_id), *(reg++) & ~TXGBE_TXCFG_ENA);
4524 * Start Receive Units for specified queue.
4527 txgbe_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
4529 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
4530 struct txgbe_rx_queue *rxq;
4534 PMD_INIT_FUNC_TRACE();
4536 rxq = dev->data->rx_queues[rx_queue_id];
4538 /* Allocate buffers for descriptor rings */
4539 if (txgbe_alloc_rx_queue_mbufs(rxq) != 0) {
4540 PMD_INIT_LOG(ERR, "Could not alloc mbuf for queue:%d",
4544 rxdctl = rd32(hw, TXGBE_RXCFG(rxq->reg_idx));
4545 rxdctl |= TXGBE_RXCFG_ENA;
4546 wr32(hw, TXGBE_RXCFG(rxq->reg_idx), rxdctl);
4548 /* Wait until RX Enable ready */
4549 poll_ms = RTE_TXGBE_REGISTER_POLL_WAIT_10_MS;
4552 rxdctl = rd32(hw, TXGBE_RXCFG(rxq->reg_idx));
4553 } while (--poll_ms && !(rxdctl & TXGBE_RXCFG_ENA));
4555 PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", rx_queue_id);
4557 wr32(hw, TXGBE_RXRP(rxq->reg_idx), 0);
4558 wr32(hw, TXGBE_RXWP(rxq->reg_idx), rxq->nb_rx_desc - 1);
4559 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
4565 * Stop Receive Units for specified queue.
4568 txgbe_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
4570 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
4571 struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev);
4572 struct txgbe_rx_queue *rxq;
4576 PMD_INIT_FUNC_TRACE();
4578 rxq = dev->data->rx_queues[rx_queue_id];
4580 txgbe_dev_save_rx_queue(hw, rxq->reg_idx);
4581 wr32m(hw, TXGBE_RXCFG(rxq->reg_idx), TXGBE_RXCFG_ENA, 0);
4583 /* Wait until RX Enable bit clear */
4584 poll_ms = RTE_TXGBE_REGISTER_POLL_WAIT_10_MS;
4587 rxdctl = rd32(hw, TXGBE_RXCFG(rxq->reg_idx));
4588 } while (--poll_ms && (rxdctl & TXGBE_RXCFG_ENA));
4590 PMD_INIT_LOG(ERR, "Could not disable Rx Queue %d", rx_queue_id);
4592 rte_delay_us(RTE_TXGBE_WAIT_100_US);
4593 txgbe_dev_store_rx_queue(hw, rxq->reg_idx);
4595 txgbe_rx_queue_release_mbufs(rxq);
4596 txgbe_reset_rx_queue(adapter, rxq);
4597 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
4603 * Start Transmit Units for specified queue.
4606 txgbe_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
4608 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
4609 struct txgbe_tx_queue *txq;
4613 PMD_INIT_FUNC_TRACE();
4615 txq = dev->data->tx_queues[tx_queue_id];
4616 wr32m(hw, TXGBE_TXCFG(txq->reg_idx), TXGBE_TXCFG_ENA, TXGBE_TXCFG_ENA);
4618 /* Wait until TX Enable ready */
4619 poll_ms = RTE_TXGBE_REGISTER_POLL_WAIT_10_MS;
4622 txdctl = rd32(hw, TXGBE_TXCFG(txq->reg_idx));
4623 } while (--poll_ms && !(txdctl & TXGBE_TXCFG_ENA));
4625 PMD_INIT_LOG(ERR, "Could not enable "
4626 "Tx Queue %d", tx_queue_id);
4629 wr32(hw, TXGBE_TXWP(txq->reg_idx), txq->tx_tail);
4630 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
4636 * Stop Transmit Units for specified queue.
4639 txgbe_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
4641 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
4642 struct txgbe_tx_queue *txq;
4644 uint32_t txtdh, txtdt;
4647 PMD_INIT_FUNC_TRACE();
4649 txq = dev->data->tx_queues[tx_queue_id];
4651 /* Wait until TX queue is empty */
4652 poll_ms = RTE_TXGBE_REGISTER_POLL_WAIT_10_MS;
4654 rte_delay_us(RTE_TXGBE_WAIT_100_US);
4655 txtdh = rd32(hw, TXGBE_TXRP(txq->reg_idx));
4656 txtdt = rd32(hw, TXGBE_TXWP(txq->reg_idx));
4657 } while (--poll_ms && (txtdh != txtdt));
4660 "Tx Queue %d is not empty when stopping.",
4663 txgbe_dev_save_tx_queue(hw, txq->reg_idx);
4664 wr32m(hw, TXGBE_TXCFG(txq->reg_idx), TXGBE_TXCFG_ENA, 0);
4666 /* Wait until TX Enable bit clear */
4667 poll_ms = RTE_TXGBE_REGISTER_POLL_WAIT_10_MS;
4670 txdctl = rd32(hw, TXGBE_TXCFG(txq->reg_idx));
4671 } while (--poll_ms && (txdctl & TXGBE_TXCFG_ENA));
4673 PMD_INIT_LOG(ERR, "Could not disable Tx Queue %d",
4676 rte_delay_us(RTE_TXGBE_WAIT_100_US);
4677 txgbe_dev_store_tx_queue(hw, txq->reg_idx);
4679 if (txq->ops != NULL) {
4680 txq->ops->release_mbufs(txq);
4681 txq->ops->reset(txq);
4683 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
4689 txgbe_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
4690 struct rte_eth_rxq_info *qinfo)
4692 struct txgbe_rx_queue *rxq;
4694 rxq = dev->data->rx_queues[queue_id];
4696 qinfo->mp = rxq->mb_pool;
4697 qinfo->scattered_rx = dev->data->scattered_rx;
4698 qinfo->nb_desc = rxq->nb_rx_desc;
4700 qinfo->conf.rx_free_thresh = rxq->rx_free_thresh;
4701 qinfo->conf.rx_drop_en = rxq->drop_en;
4702 qinfo->conf.rx_deferred_start = rxq->rx_deferred_start;
4703 qinfo->conf.offloads = rxq->offloads;
4707 txgbe_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
4708 struct rte_eth_txq_info *qinfo)
4710 struct txgbe_tx_queue *txq;
4712 txq = dev->data->tx_queues[queue_id];
4714 qinfo->nb_desc = txq->nb_tx_desc;
4716 qinfo->conf.tx_thresh.pthresh = txq->pthresh;
4717 qinfo->conf.tx_thresh.hthresh = txq->hthresh;
4718 qinfo->conf.tx_thresh.wthresh = txq->wthresh;
4720 qinfo->conf.tx_free_thresh = txq->tx_free_thresh;
4721 qinfo->conf.offloads = txq->offloads;
4722 qinfo->conf.tx_deferred_start = txq->tx_deferred_start;
4726 txgbe_rss_conf_init(struct txgbe_rte_flow_rss_conf *out,
4727 const struct rte_flow_action_rss *in)
4729 if (in->key_len > RTE_DIM(out->key) ||
4730 in->queue_num > RTE_DIM(out->queue))
4732 out->conf = (struct rte_flow_action_rss){
4736 .key_len = in->key_len,
4737 .queue_num = in->queue_num,
4738 .key = memcpy(out->key, in->key, in->key_len),
4739 .queue = memcpy(out->queue, in->queue,
4740 sizeof(*in->queue) * in->queue_num),
4746 txgbe_action_rss_same(const struct rte_flow_action_rss *comp,
4747 const struct rte_flow_action_rss *with)
4749 return (comp->func == with->func &&
4750 comp->level == with->level &&
4751 comp->types == with->types &&
4752 comp->key_len == with->key_len &&
4753 comp->queue_num == with->queue_num &&
4754 !memcmp(comp->key, with->key, with->key_len) &&
4755 !memcmp(comp->queue, with->queue,
4756 sizeof(*with->queue) * with->queue_num));
4760 txgbe_config_rss_filter(struct rte_eth_dev *dev,
4761 struct txgbe_rte_flow_rss_conf *conf, bool add)
4763 struct txgbe_hw *hw;
4767 struct rte_eth_rss_conf rss_conf = {
4768 .rss_key = conf->conf.key_len ?
4769 (void *)(uintptr_t)conf->conf.key : NULL,
4770 .rss_key_len = conf->conf.key_len,
4771 .rss_hf = conf->conf.types,
4773 struct txgbe_filter_info *filter_info = TXGBE_DEV_FILTER(dev);
4775 PMD_INIT_FUNC_TRACE();
4776 hw = TXGBE_DEV_HW(dev);
4779 if (txgbe_action_rss_same(&filter_info->rss_info.conf,
4781 txgbe_rss_disable(dev);
4782 memset(&filter_info->rss_info, 0,
4783 sizeof(struct txgbe_rte_flow_rss_conf));
4789 if (filter_info->rss_info.conf.queue_num)
4791 /* Fill in redirection table
4792 * The byte-swap is needed because NIC registers are in
4793 * little-endian order.
4796 for (i = 0, j = 0; i < ETH_RSS_RETA_SIZE_128; i++, j++) {
4797 if (j == conf->conf.queue_num)
4799 reta = (reta >> 8) | LS32(conf->conf.queue[j], 24, 0xFF);
4801 wr32a(hw, TXGBE_REG_RSSTBL, i >> 2, reta);
4804 /* Configure the RSS key and the RSS protocols used to compute
4805 * the RSS hash of input packets.
4807 if ((rss_conf.rss_hf & TXGBE_RSS_OFFLOAD_ALL) == 0) {
4808 txgbe_rss_disable(dev);
4811 if (rss_conf.rss_key == NULL)
4812 rss_conf.rss_key = rss_intel_key; /* Default hash key */
4813 txgbe_dev_rss_hash_update(dev, &rss_conf);
4815 if (txgbe_rss_conf_init(&filter_info->rss_info, &conf->conf))