1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2015-2020
16 #include <rte_byteorder.h>
17 #include <rte_common.h>
18 #include <rte_cycles.h>
20 #include <rte_debug.h>
21 #include <rte_ethdev.h>
22 #include <ethdev_driver.h>
23 #include <rte_security_driver.h>
24 #include <rte_memzone.h>
25 #include <rte_atomic.h>
26 #include <rte_mempool.h>
27 #include <rte_malloc.h>
29 #include <rte_ether.h>
30 #include <rte_prefetch.h>
34 #include <rte_string_fns.h>
35 #include <rte_errno.h>
39 #include "txgbe_logs.h"
40 #include "base/txgbe.h"
41 #include "txgbe_ethdev.h"
42 #include "txgbe_rxtx.h"
44 #ifdef RTE_LIBRTE_IEEE1588
45 #define TXGBE_TX_IEEE1588_TMST PKT_TX_IEEE1588_TMST
47 #define TXGBE_TX_IEEE1588_TMST 0
50 /* Bit Mask to indicate what bits required for building TX context */
51 static const u64 TXGBE_TX_OFFLOAD_MASK = (PKT_TX_IP_CKSUM |
60 PKT_TX_OUTER_IP_CKSUM |
61 #ifdef RTE_LIB_SECURITY
64 TXGBE_TX_IEEE1588_TMST);
66 #define TXGBE_TX_OFFLOAD_NOTSUP_MASK \
67 (PKT_TX_OFFLOAD_MASK ^ TXGBE_TX_OFFLOAD_MASK)
70 * Prefetch a cache line into all cache levels.
72 #define rte_txgbe_prefetch(p) rte_prefetch0(p)
75 txgbe_is_vf(struct rte_eth_dev *dev)
77 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
79 switch (hw->mac.type) {
80 case txgbe_mac_raptor_vf:
87 /*********************************************************************
91 **********************************************************************/
94 * Check for descriptors with their DD bit set and free mbufs.
95 * Return the total number of buffers freed.
97 static __rte_always_inline int
98 txgbe_tx_free_bufs(struct txgbe_tx_queue *txq)
100 struct txgbe_tx_entry *txep;
103 struct rte_mbuf *m, *free[RTE_TXGBE_TX_MAX_FREE_BUF_SZ];
105 /* check DD bit on threshold descriptor */
106 status = txq->tx_ring[txq->tx_next_dd].dw3;
107 if (!(status & rte_cpu_to_le_32(TXGBE_TXD_DD))) {
108 if (txq->nb_tx_free >> 1 < txq->tx_free_thresh)
109 txgbe_set32_masked(txq->tdc_reg_addr,
110 TXGBE_TXCFG_FLUSH, TXGBE_TXCFG_FLUSH);
115 * first buffer to free from S/W ring is at index
116 * tx_next_dd - (tx_free_thresh-1)
118 txep = &txq->sw_ring[txq->tx_next_dd - (txq->tx_free_thresh - 1)];
119 for (i = 0; i < txq->tx_free_thresh; ++i, ++txep) {
120 /* free buffers one at a time */
121 m = rte_pktmbuf_prefree_seg(txep->mbuf);
124 if (unlikely(m == NULL))
127 if (nb_free >= RTE_TXGBE_TX_MAX_FREE_BUF_SZ ||
128 (nb_free > 0 && m->pool != free[0]->pool)) {
129 rte_mempool_put_bulk(free[0]->pool,
130 (void **)free, nb_free);
138 rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free);
140 /* buffers were freed, update counters */
141 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_free_thresh);
142 txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_free_thresh);
143 if (txq->tx_next_dd >= txq->nb_tx_desc)
144 txq->tx_next_dd = (uint16_t)(txq->tx_free_thresh - 1);
146 return txq->tx_free_thresh;
149 /* Populate 4 descriptors with data from 4 mbufs */
151 tx4(volatile struct txgbe_tx_desc *txdp, struct rte_mbuf **pkts)
153 uint64_t buf_dma_addr;
157 for (i = 0; i < 4; ++i, ++txdp, ++pkts) {
158 buf_dma_addr = rte_mbuf_data_iova(*pkts);
159 pkt_len = (*pkts)->data_len;
161 /* write data to descriptor */
162 txdp->qw0 = rte_cpu_to_le_64(buf_dma_addr);
163 txdp->dw2 = cpu_to_le32(TXGBE_TXD_FLAGS |
164 TXGBE_TXD_DATLEN(pkt_len));
165 txdp->dw3 = cpu_to_le32(TXGBE_TXD_PAYLEN(pkt_len));
167 rte_prefetch0(&(*pkts)->pool);
171 /* Populate 1 descriptor with data from 1 mbuf */
173 tx1(volatile struct txgbe_tx_desc *txdp, struct rte_mbuf **pkts)
175 uint64_t buf_dma_addr;
178 buf_dma_addr = rte_mbuf_data_iova(*pkts);
179 pkt_len = (*pkts)->data_len;
181 /* write data to descriptor */
182 txdp->qw0 = cpu_to_le64(buf_dma_addr);
183 txdp->dw2 = cpu_to_le32(TXGBE_TXD_FLAGS |
184 TXGBE_TXD_DATLEN(pkt_len));
185 txdp->dw3 = cpu_to_le32(TXGBE_TXD_PAYLEN(pkt_len));
187 rte_prefetch0(&(*pkts)->pool);
191 * Fill H/W descriptor ring with mbuf data.
192 * Copy mbuf pointers to the S/W ring.
195 txgbe_tx_fill_hw_ring(struct txgbe_tx_queue *txq, struct rte_mbuf **pkts,
198 volatile struct txgbe_tx_desc *txdp = &txq->tx_ring[txq->tx_tail];
199 struct txgbe_tx_entry *txep = &txq->sw_ring[txq->tx_tail];
200 const int N_PER_LOOP = 4;
201 const int N_PER_LOOP_MASK = N_PER_LOOP - 1;
202 int mainpart, leftover;
206 * Process most of the packets in chunks of N pkts. Any
207 * leftover packets will get processed one at a time.
209 mainpart = (nb_pkts & ((uint32_t)~N_PER_LOOP_MASK));
210 leftover = (nb_pkts & ((uint32_t)N_PER_LOOP_MASK));
211 for (i = 0; i < mainpart; i += N_PER_LOOP) {
212 /* Copy N mbuf pointers to the S/W ring */
213 for (j = 0; j < N_PER_LOOP; ++j)
214 (txep + i + j)->mbuf = *(pkts + i + j);
215 tx4(txdp + i, pkts + i);
218 if (unlikely(leftover > 0)) {
219 for (i = 0; i < leftover; ++i) {
220 (txep + mainpart + i)->mbuf = *(pkts + mainpart + i);
221 tx1(txdp + mainpart + i, pkts + mainpart + i);
226 static inline uint16_t
227 tx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
230 struct txgbe_tx_queue *txq = (struct txgbe_tx_queue *)tx_queue;
234 * Begin scanning the H/W ring for done descriptors when the
235 * number of available descriptors drops below tx_free_thresh. For
236 * each done descriptor, free the associated buffer.
238 if (txq->nb_tx_free < txq->tx_free_thresh)
239 txgbe_tx_free_bufs(txq);
241 /* Only use descriptors that are available */
242 nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts);
243 if (unlikely(nb_pkts == 0))
246 /* Use exactly nb_pkts descriptors */
247 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts);
250 * At this point, we know there are enough descriptors in the
251 * ring to transmit all the packets. This assumes that each
252 * mbuf contains a single segment, and that no new offloads
253 * are expected, which would require a new context descriptor.
257 * See if we're going to wrap-around. If so, handle the top
258 * of the descriptor ring first, then do the bottom. If not,
259 * the processing looks just like the "bottom" part anyway...
261 if ((txq->tx_tail + nb_pkts) > txq->nb_tx_desc) {
262 n = (uint16_t)(txq->nb_tx_desc - txq->tx_tail);
263 txgbe_tx_fill_hw_ring(txq, tx_pkts, n);
267 /* Fill H/W descriptor ring with mbuf data */
268 txgbe_tx_fill_hw_ring(txq, tx_pkts + n, (uint16_t)(nb_pkts - n));
269 txq->tx_tail = (uint16_t)(txq->tx_tail + (nb_pkts - n));
272 * Check for wrap-around. This would only happen if we used
273 * up to the last descriptor in the ring, no more, no less.
275 if (txq->tx_tail >= txq->nb_tx_desc)
278 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u",
279 (uint16_t)txq->port_id, (uint16_t)txq->queue_id,
280 (uint16_t)txq->tx_tail, (uint16_t)nb_pkts);
282 /* update tail pointer */
284 txgbe_set32_relaxed(txq->tdt_reg_addr, txq->tx_tail);
290 txgbe_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts,
295 /* Try to transmit at least chunks of TX_MAX_BURST pkts */
296 if (likely(nb_pkts <= RTE_PMD_TXGBE_TX_MAX_BURST))
297 return tx_xmit_pkts(tx_queue, tx_pkts, nb_pkts);
299 /* transmit more than the max burst, in chunks of TX_MAX_BURST */
304 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_TXGBE_TX_MAX_BURST);
305 ret = tx_xmit_pkts(tx_queue, &tx_pkts[nb_tx], n);
306 nb_tx = (uint16_t)(nb_tx + ret);
307 nb_pkts = (uint16_t)(nb_pkts - ret);
316 txgbe_set_xmit_ctx(struct txgbe_tx_queue *txq,
317 volatile struct txgbe_tx_ctx_desc *ctx_txd,
318 uint64_t ol_flags, union txgbe_tx_offload tx_offload,
319 __rte_unused uint64_t *mdata)
321 union txgbe_tx_offload tx_offload_mask;
322 uint32_t type_tucmd_mlhl;
323 uint32_t mss_l4len_idx;
325 uint32_t vlan_macip_lens;
326 uint32_t tunnel_seed;
328 ctx_idx = txq->ctx_curr;
329 tx_offload_mask.data[0] = 0;
330 tx_offload_mask.data[1] = 0;
332 /* Specify which HW CTX to upload. */
333 mss_l4len_idx = TXGBE_TXD_IDX(ctx_idx);
334 type_tucmd_mlhl = TXGBE_TXD_CTXT;
336 tx_offload_mask.ptid |= ~0;
337 type_tucmd_mlhl |= TXGBE_TXD_PTID(tx_offload.ptid);
339 /* check if TCP segmentation required for this packet */
340 if (ol_flags & PKT_TX_TCP_SEG) {
341 tx_offload_mask.l2_len |= ~0;
342 tx_offload_mask.l3_len |= ~0;
343 tx_offload_mask.l4_len |= ~0;
344 tx_offload_mask.tso_segsz |= ~0;
345 mss_l4len_idx |= TXGBE_TXD_MSS(tx_offload.tso_segsz);
346 mss_l4len_idx |= TXGBE_TXD_L4LEN(tx_offload.l4_len);
347 } else { /* no TSO, check if hardware checksum is needed */
348 if (ol_flags & PKT_TX_IP_CKSUM) {
349 tx_offload_mask.l2_len |= ~0;
350 tx_offload_mask.l3_len |= ~0;
353 switch (ol_flags & PKT_TX_L4_MASK) {
354 case PKT_TX_UDP_CKSUM:
356 TXGBE_TXD_L4LEN(sizeof(struct rte_udp_hdr));
357 tx_offload_mask.l2_len |= ~0;
358 tx_offload_mask.l3_len |= ~0;
360 case PKT_TX_TCP_CKSUM:
362 TXGBE_TXD_L4LEN(sizeof(struct rte_tcp_hdr));
363 tx_offload_mask.l2_len |= ~0;
364 tx_offload_mask.l3_len |= ~0;
366 case PKT_TX_SCTP_CKSUM:
368 TXGBE_TXD_L4LEN(sizeof(struct rte_sctp_hdr));
369 tx_offload_mask.l2_len |= ~0;
370 tx_offload_mask.l3_len |= ~0;
377 vlan_macip_lens = TXGBE_TXD_IPLEN(tx_offload.l3_len >> 1);
379 if (ol_flags & PKT_TX_TUNNEL_MASK) {
380 tx_offload_mask.outer_tun_len |= ~0;
381 tx_offload_mask.outer_l2_len |= ~0;
382 tx_offload_mask.outer_l3_len |= ~0;
383 tx_offload_mask.l2_len |= ~0;
384 tunnel_seed = TXGBE_TXD_ETUNLEN(tx_offload.outer_tun_len >> 1);
385 tunnel_seed |= TXGBE_TXD_EIPLEN(tx_offload.outer_l3_len >> 2);
387 switch (ol_flags & PKT_TX_TUNNEL_MASK) {
388 case PKT_TX_TUNNEL_IPIP:
389 /* for non UDP / GRE tunneling, set to 0b */
391 case PKT_TX_TUNNEL_VXLAN:
392 case PKT_TX_TUNNEL_GENEVE:
393 tunnel_seed |= TXGBE_TXD_ETYPE_UDP;
395 case PKT_TX_TUNNEL_GRE:
396 tunnel_seed |= TXGBE_TXD_ETYPE_GRE;
399 PMD_TX_LOG(ERR, "Tunnel type not supported");
402 vlan_macip_lens |= TXGBE_TXD_MACLEN(tx_offload.outer_l2_len);
405 vlan_macip_lens |= TXGBE_TXD_MACLEN(tx_offload.l2_len);
408 if (ol_flags & PKT_TX_VLAN_PKT) {
409 tx_offload_mask.vlan_tci |= ~0;
410 vlan_macip_lens |= TXGBE_TXD_VLAN(tx_offload.vlan_tci);
413 #ifdef RTE_LIB_SECURITY
414 if (ol_flags & PKT_TX_SEC_OFFLOAD) {
415 union txgbe_crypto_tx_desc_md *md =
416 (union txgbe_crypto_tx_desc_md *)mdata;
417 tunnel_seed |= TXGBE_TXD_IPSEC_SAIDX(md->sa_idx);
418 type_tucmd_mlhl |= md->enc ?
419 (TXGBE_TXD_IPSEC_ESP | TXGBE_TXD_IPSEC_ESPENC) : 0;
420 type_tucmd_mlhl |= TXGBE_TXD_IPSEC_ESPLEN(md->pad_len);
421 tx_offload_mask.sa_idx |= ~0;
422 tx_offload_mask.sec_pad_len |= ~0;
426 txq->ctx_cache[ctx_idx].flags = ol_flags;
427 txq->ctx_cache[ctx_idx].tx_offload.data[0] =
428 tx_offload_mask.data[0] & tx_offload.data[0];
429 txq->ctx_cache[ctx_idx].tx_offload.data[1] =
430 tx_offload_mask.data[1] & tx_offload.data[1];
431 txq->ctx_cache[ctx_idx].tx_offload_mask = tx_offload_mask;
433 ctx_txd->dw0 = rte_cpu_to_le_32(vlan_macip_lens);
434 ctx_txd->dw1 = rte_cpu_to_le_32(tunnel_seed);
435 ctx_txd->dw2 = rte_cpu_to_le_32(type_tucmd_mlhl);
436 ctx_txd->dw3 = rte_cpu_to_le_32(mss_l4len_idx);
440 * Check which hardware context can be used. Use the existing match
441 * or create a new context descriptor.
443 static inline uint32_t
444 what_ctx_update(struct txgbe_tx_queue *txq, uint64_t flags,
445 union txgbe_tx_offload tx_offload)
447 /* If match with the current used context */
448 if (likely(txq->ctx_cache[txq->ctx_curr].flags == flags &&
449 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] ==
450 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0]
451 & tx_offload.data[0])) &&
452 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] ==
453 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1]
454 & tx_offload.data[1]))))
455 return txq->ctx_curr;
457 /* What if match with the next context */
459 if (likely(txq->ctx_cache[txq->ctx_curr].flags == flags &&
460 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] ==
461 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0]
462 & tx_offload.data[0])) &&
463 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] ==
464 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1]
465 & tx_offload.data[1]))))
466 return txq->ctx_curr;
468 /* Mismatch, use the previous context */
469 return TXGBE_CTX_NUM;
472 static inline uint32_t
473 tx_desc_cksum_flags_to_olinfo(uint64_t ol_flags)
477 if ((ol_flags & PKT_TX_L4_MASK) != PKT_TX_L4_NO_CKSUM) {
479 tmp |= TXGBE_TXD_L4CS;
481 if (ol_flags & PKT_TX_IP_CKSUM) {
483 tmp |= TXGBE_TXD_IPCS;
485 if (ol_flags & PKT_TX_OUTER_IP_CKSUM) {
487 tmp |= TXGBE_TXD_EIPCS;
489 if (ol_flags & PKT_TX_TCP_SEG) {
491 /* implies IPv4 cksum */
492 if (ol_flags & PKT_TX_IPV4)
493 tmp |= TXGBE_TXD_IPCS;
494 tmp |= TXGBE_TXD_L4CS;
496 if (ol_flags & PKT_TX_VLAN_PKT)
502 static inline uint32_t
503 tx_desc_ol_flags_to_cmdtype(uint64_t ol_flags)
505 uint32_t cmdtype = 0;
507 if (ol_flags & PKT_TX_VLAN_PKT)
508 cmdtype |= TXGBE_TXD_VLE;
509 if (ol_flags & PKT_TX_TCP_SEG)
510 cmdtype |= TXGBE_TXD_TSE;
511 if (ol_flags & PKT_TX_MACSEC)
512 cmdtype |= TXGBE_TXD_LINKSEC;
516 static inline uint8_t
517 tx_desc_ol_flags_to_ptid(uint64_t oflags, uint32_t ptype)
522 return txgbe_encode_ptype(ptype);
524 /* Only support flags in TXGBE_TX_OFFLOAD_MASK */
525 tun = !!(oflags & PKT_TX_TUNNEL_MASK);
528 ptype = RTE_PTYPE_L2_ETHER;
529 if (oflags & PKT_TX_VLAN)
530 ptype |= RTE_PTYPE_L2_ETHER_VLAN;
533 if (oflags & (PKT_TX_OUTER_IPV4 | PKT_TX_OUTER_IP_CKSUM))
534 ptype |= RTE_PTYPE_L3_IPV4;
535 else if (oflags & (PKT_TX_OUTER_IPV6))
536 ptype |= RTE_PTYPE_L3_IPV6;
538 if (oflags & (PKT_TX_IPV4 | PKT_TX_IP_CKSUM))
539 ptype |= (tun ? RTE_PTYPE_INNER_L3_IPV4 : RTE_PTYPE_L3_IPV4);
540 else if (oflags & (PKT_TX_IPV6))
541 ptype |= (tun ? RTE_PTYPE_INNER_L3_IPV6 : RTE_PTYPE_L3_IPV6);
544 switch (oflags & (PKT_TX_L4_MASK)) {
545 case PKT_TX_TCP_CKSUM:
546 ptype |= (tun ? RTE_PTYPE_INNER_L4_TCP : RTE_PTYPE_L4_TCP);
548 case PKT_TX_UDP_CKSUM:
549 ptype |= (tun ? RTE_PTYPE_INNER_L4_UDP : RTE_PTYPE_L4_UDP);
551 case PKT_TX_SCTP_CKSUM:
552 ptype |= (tun ? RTE_PTYPE_INNER_L4_SCTP : RTE_PTYPE_L4_SCTP);
556 if (oflags & PKT_TX_TCP_SEG)
557 ptype |= (tun ? RTE_PTYPE_INNER_L4_TCP : RTE_PTYPE_L4_TCP);
560 switch (oflags & PKT_TX_TUNNEL_MASK) {
561 case PKT_TX_TUNNEL_VXLAN:
562 ptype |= RTE_PTYPE_L2_ETHER |
564 RTE_PTYPE_TUNNEL_VXLAN;
565 ptype |= RTE_PTYPE_INNER_L2_ETHER;
567 case PKT_TX_TUNNEL_GRE:
568 ptype |= RTE_PTYPE_L2_ETHER |
570 RTE_PTYPE_TUNNEL_GRE;
571 ptype |= RTE_PTYPE_INNER_L2_ETHER;
573 case PKT_TX_TUNNEL_GENEVE:
574 ptype |= RTE_PTYPE_L2_ETHER |
576 RTE_PTYPE_TUNNEL_GENEVE;
577 ptype |= RTE_PTYPE_INNER_L2_ETHER;
579 case PKT_TX_TUNNEL_VXLAN_GPE:
580 ptype |= RTE_PTYPE_L2_ETHER |
582 RTE_PTYPE_TUNNEL_VXLAN_GPE;
583 ptype |= RTE_PTYPE_INNER_L2_ETHER;
585 case PKT_TX_TUNNEL_IPIP:
586 case PKT_TX_TUNNEL_IP:
587 ptype |= RTE_PTYPE_L2_ETHER |
593 return txgbe_encode_ptype(ptype);
596 #ifndef DEFAULT_TX_FREE_THRESH
597 #define DEFAULT_TX_FREE_THRESH 32
600 /* Reset transmit descriptors after they have been used */
602 txgbe_xmit_cleanup(struct txgbe_tx_queue *txq)
604 struct txgbe_tx_entry *sw_ring = txq->sw_ring;
605 volatile struct txgbe_tx_desc *txr = txq->tx_ring;
606 uint16_t last_desc_cleaned = txq->last_desc_cleaned;
607 uint16_t nb_tx_desc = txq->nb_tx_desc;
608 uint16_t desc_to_clean_to;
609 uint16_t nb_tx_to_clean;
612 /* Determine the last descriptor needing to be cleaned */
613 desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->tx_free_thresh);
614 if (desc_to_clean_to >= nb_tx_desc)
615 desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc);
617 /* Check to make sure the last descriptor to clean is done */
618 desc_to_clean_to = sw_ring[desc_to_clean_to].last_id;
619 status = txr[desc_to_clean_to].dw3;
620 if (!(status & rte_cpu_to_le_32(TXGBE_TXD_DD))) {
621 PMD_TX_FREE_LOG(DEBUG,
622 "TX descriptor %4u is not done"
623 "(port=%d queue=%d)",
625 txq->port_id, txq->queue_id);
626 if (txq->nb_tx_free >> 1 < txq->tx_free_thresh)
627 txgbe_set32_masked(txq->tdc_reg_addr,
628 TXGBE_TXCFG_FLUSH, TXGBE_TXCFG_FLUSH);
629 /* Failed to clean any descriptors, better luck next time */
633 /* Figure out how many descriptors will be cleaned */
634 if (last_desc_cleaned > desc_to_clean_to)
635 nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) +
638 nb_tx_to_clean = (uint16_t)(desc_to_clean_to -
641 PMD_TX_FREE_LOG(DEBUG,
642 "Cleaning %4u TX descriptors: %4u to %4u "
643 "(port=%d queue=%d)",
644 nb_tx_to_clean, last_desc_cleaned, desc_to_clean_to,
645 txq->port_id, txq->queue_id);
648 * The last descriptor to clean is done, so that means all the
649 * descriptors from the last descriptor that was cleaned
650 * up to the last descriptor with the RS bit set
651 * are done. Only reset the threshold descriptor.
653 txr[desc_to_clean_to].dw3 = 0;
655 /* Update the txq to reflect the last descriptor that was cleaned */
656 txq->last_desc_cleaned = desc_to_clean_to;
657 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + nb_tx_to_clean);
663 static inline uint8_t
664 txgbe_get_tun_len(struct rte_mbuf *mbuf)
666 struct txgbe_genevehdr genevehdr;
667 const struct txgbe_genevehdr *gh;
670 switch (mbuf->ol_flags & PKT_TX_TUNNEL_MASK) {
671 case PKT_TX_TUNNEL_IPIP:
674 case PKT_TX_TUNNEL_VXLAN:
675 case PKT_TX_TUNNEL_VXLAN_GPE:
676 tun_len = sizeof(struct txgbe_udphdr)
677 + sizeof(struct txgbe_vxlanhdr);
679 case PKT_TX_TUNNEL_GRE:
680 tun_len = sizeof(struct txgbe_nvgrehdr);
682 case PKT_TX_TUNNEL_GENEVE:
683 gh = rte_pktmbuf_read(mbuf,
684 mbuf->outer_l2_len + mbuf->outer_l3_len,
685 sizeof(genevehdr), &genevehdr);
686 tun_len = sizeof(struct txgbe_udphdr)
687 + sizeof(struct txgbe_genevehdr)
688 + (gh->opt_len << 2);
698 txgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
701 struct txgbe_tx_queue *txq;
702 struct txgbe_tx_entry *sw_ring;
703 struct txgbe_tx_entry *txe, *txn;
704 volatile struct txgbe_tx_desc *txr;
705 volatile struct txgbe_tx_desc *txd;
706 struct rte_mbuf *tx_pkt;
707 struct rte_mbuf *m_seg;
708 uint64_t buf_dma_addr;
709 uint32_t olinfo_status;
710 uint32_t cmd_type_len;
721 union txgbe_tx_offload tx_offload;
722 #ifdef RTE_LIB_SECURITY
726 tx_offload.data[0] = 0;
727 tx_offload.data[1] = 0;
729 sw_ring = txq->sw_ring;
731 tx_id = txq->tx_tail;
732 txe = &sw_ring[tx_id];
734 /* Determine if the descriptor ring needs to be cleaned. */
735 if (txq->nb_tx_free < txq->tx_free_thresh)
736 txgbe_xmit_cleanup(txq);
738 rte_prefetch0(&txe->mbuf->pool);
741 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
744 pkt_len = tx_pkt->pkt_len;
747 * Determine how many (if any) context descriptors
748 * are needed for offload functionality.
750 ol_flags = tx_pkt->ol_flags;
751 #ifdef RTE_LIB_SECURITY
752 use_ipsec = txq->using_ipsec && (ol_flags & PKT_TX_SEC_OFFLOAD);
755 /* If hardware offload required */
756 tx_ol_req = ol_flags & TXGBE_TX_OFFLOAD_MASK;
758 tx_offload.ptid = tx_desc_ol_flags_to_ptid(tx_ol_req,
759 tx_pkt->packet_type);
760 tx_offload.l2_len = tx_pkt->l2_len;
761 tx_offload.l3_len = tx_pkt->l3_len;
762 tx_offload.l4_len = tx_pkt->l4_len;
763 tx_offload.vlan_tci = tx_pkt->vlan_tci;
764 tx_offload.tso_segsz = tx_pkt->tso_segsz;
765 tx_offload.outer_l2_len = tx_pkt->outer_l2_len;
766 tx_offload.outer_l3_len = tx_pkt->outer_l3_len;
767 tx_offload.outer_tun_len = txgbe_get_tun_len(tx_pkt);
769 #ifdef RTE_LIB_SECURITY
771 union txgbe_crypto_tx_desc_md *ipsec_mdata =
772 (union txgbe_crypto_tx_desc_md *)
773 rte_security_dynfield(tx_pkt);
774 tx_offload.sa_idx = ipsec_mdata->sa_idx;
775 tx_offload.sec_pad_len = ipsec_mdata->pad_len;
779 /* If new context need be built or reuse the exist ctx*/
780 ctx = what_ctx_update(txq, tx_ol_req, tx_offload);
781 /* Only allocate context descriptor if required */
782 new_ctx = (ctx == TXGBE_CTX_NUM);
787 * Keep track of how many descriptors are used this loop
788 * This will always be the number of segments + the number of
789 * Context descriptors required to transmit the packet
791 nb_used = (uint16_t)(tx_pkt->nb_segs + new_ctx);
794 * The number of descriptors that must be allocated for a
795 * packet is the number of segments of that packet, plus 1
796 * Context Descriptor for the hardware offload, if any.
797 * Determine the last TX descriptor to allocate in the TX ring
798 * for the packet, starting from the current position (tx_id)
801 tx_last = (uint16_t)(tx_id + nb_used - 1);
804 if (tx_last >= txq->nb_tx_desc)
805 tx_last = (uint16_t)(tx_last - txq->nb_tx_desc);
807 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u pktlen=%u"
808 " tx_first=%u tx_last=%u",
809 (uint16_t)txq->port_id,
810 (uint16_t)txq->queue_id,
816 * Make sure there are enough TX descriptors available to
817 * transmit the entire packet.
818 * nb_used better be less than or equal to txq->tx_free_thresh
820 if (nb_used > txq->nb_tx_free) {
821 PMD_TX_FREE_LOG(DEBUG,
822 "Not enough free TX descriptors "
823 "nb_used=%4u nb_free=%4u "
824 "(port=%d queue=%d)",
825 nb_used, txq->nb_tx_free,
826 txq->port_id, txq->queue_id);
828 if (txgbe_xmit_cleanup(txq) != 0) {
829 /* Could not clean any descriptors */
835 /* nb_used better be <= txq->tx_free_thresh */
836 if (unlikely(nb_used > txq->tx_free_thresh)) {
837 PMD_TX_FREE_LOG(DEBUG,
838 "The number of descriptors needed to "
839 "transmit the packet exceeds the "
840 "RS bit threshold. This will impact "
842 "nb_used=%4u nb_free=%4u "
843 "tx_free_thresh=%4u. "
844 "(port=%d queue=%d)",
845 nb_used, txq->nb_tx_free,
847 txq->port_id, txq->queue_id);
849 * Loop here until there are enough TX
850 * descriptors or until the ring cannot be
853 while (nb_used > txq->nb_tx_free) {
854 if (txgbe_xmit_cleanup(txq) != 0) {
856 * Could not clean any
868 * By now there are enough free TX descriptors to transmit
873 * Set common flags of all TX Data Descriptors.
875 * The following bits must be set in all Data Descriptors:
876 * - TXGBE_TXD_DTYP_DATA
877 * - TXGBE_TXD_DCMD_DEXT
879 * The following bits must be set in the first Data Descriptor
880 * and are ignored in the other ones:
881 * - TXGBE_TXD_DCMD_IFCS
882 * - TXGBE_TXD_MAC_1588
883 * - TXGBE_TXD_DCMD_VLE
885 * The following bits must only be set in the last Data
887 * - TXGBE_TXD_CMD_EOP
889 * The following bits can be set in any Data Descriptor, but
890 * are only set in the last Data Descriptor:
893 cmd_type_len = TXGBE_TXD_FCS;
895 #ifdef RTE_LIBRTE_IEEE1588
896 if (ol_flags & PKT_TX_IEEE1588_TMST)
897 cmd_type_len |= TXGBE_TXD_1588;
902 if (ol_flags & PKT_TX_TCP_SEG) {
903 /* when TSO is on, paylen in descriptor is the
904 * not the packet len but the tcp payload len
906 pkt_len -= (tx_offload.l2_len +
907 tx_offload.l3_len + tx_offload.l4_len);
909 (tx_pkt->ol_flags & PKT_TX_TUNNEL_MASK)
910 ? tx_offload.outer_l2_len +
911 tx_offload.outer_l3_len : 0;
915 * Setup the TX Advanced Context Descriptor if required
918 volatile struct txgbe_tx_ctx_desc *ctx_txd;
920 ctx_txd = (volatile struct txgbe_tx_ctx_desc *)
923 txn = &sw_ring[txe->next_id];
924 rte_prefetch0(&txn->mbuf->pool);
926 if (txe->mbuf != NULL) {
927 rte_pktmbuf_free_seg(txe->mbuf);
931 txgbe_set_xmit_ctx(txq, ctx_txd, tx_ol_req,
933 rte_security_dynfield(tx_pkt));
935 txe->last_id = tx_last;
936 tx_id = txe->next_id;
941 * Setup the TX Advanced Data Descriptor,
942 * This path will go through
943 * whatever new/reuse the context descriptor
945 cmd_type_len |= tx_desc_ol_flags_to_cmdtype(ol_flags);
947 tx_desc_cksum_flags_to_olinfo(ol_flags);
948 olinfo_status |= TXGBE_TXD_IDX(ctx);
951 olinfo_status |= TXGBE_TXD_PAYLEN(pkt_len);
952 #ifdef RTE_LIB_SECURITY
954 olinfo_status |= TXGBE_TXD_IPSEC;
960 txn = &sw_ring[txe->next_id];
961 rte_prefetch0(&txn->mbuf->pool);
963 if (txe->mbuf != NULL)
964 rte_pktmbuf_free_seg(txe->mbuf);
968 * Set up Transmit Data Descriptor.
970 slen = m_seg->data_len;
971 buf_dma_addr = rte_mbuf_data_iova(m_seg);
972 txd->qw0 = rte_cpu_to_le_64(buf_dma_addr);
973 txd->dw2 = rte_cpu_to_le_32(cmd_type_len | slen);
974 txd->dw3 = rte_cpu_to_le_32(olinfo_status);
975 txe->last_id = tx_last;
976 tx_id = txe->next_id;
979 } while (m_seg != NULL);
982 * The last packet data descriptor needs End Of Packet (EOP)
984 cmd_type_len |= TXGBE_TXD_EOP;
985 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_used);
987 txd->dw2 |= rte_cpu_to_le_32(cmd_type_len);
995 * Set the Transmit Descriptor Tail (TDT)
997 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u",
998 (uint16_t)txq->port_id, (uint16_t)txq->queue_id,
999 (uint16_t)tx_id, (uint16_t)nb_tx);
1000 txgbe_set32_relaxed(txq->tdt_reg_addr, tx_id);
1001 txq->tx_tail = tx_id;
1006 /*********************************************************************
1010 **********************************************************************/
1012 txgbe_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
1017 struct txgbe_tx_queue *txq = (struct txgbe_tx_queue *)tx_queue;
1019 for (i = 0; i < nb_pkts; i++) {
1021 ol_flags = m->ol_flags;
1024 * Check if packet meets requirements for number of segments
1026 * NOTE: for txgbe it's always (40 - WTHRESH) for both TSO and
1030 if (m->nb_segs > TXGBE_TX_MAX_SEG - txq->wthresh) {
1031 rte_errno = -EINVAL;
1035 if (ol_flags & TXGBE_TX_OFFLOAD_NOTSUP_MASK) {
1036 rte_errno = -ENOTSUP;
1040 #ifdef RTE_LIBRTE_ETHDEV_DEBUG
1041 ret = rte_validate_tx_offload(m);
1047 ret = rte_net_intel_cksum_prepare(m);
1057 /*********************************************************************
1061 **********************************************************************/
1062 /* @note: fix txgbe_dev_supported_ptypes_get() if any change here. */
1063 static inline uint32_t
1064 txgbe_rxd_pkt_info_to_pkt_type(uint32_t pkt_info, uint16_t ptid_mask)
1066 uint16_t ptid = TXGBE_RXD_PTID(pkt_info);
1070 return txgbe_decode_ptype(ptid);
1073 static inline uint64_t
1074 txgbe_rxd_pkt_info_to_pkt_flags(uint32_t pkt_info)
1076 static uint64_t ip_rss_types_map[16] __rte_cache_aligned = {
1077 0, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH,
1078 0, PKT_RX_RSS_HASH, 0, PKT_RX_RSS_HASH,
1079 PKT_RX_RSS_HASH, 0, 0, 0,
1080 0, 0, 0, PKT_RX_FDIR,
1082 #ifdef RTE_LIBRTE_IEEE1588
1083 static uint64_t ip_pkt_etqf_map[8] = {
1084 0, 0, 0, PKT_RX_IEEE1588_PTP,
1087 int etfid = txgbe_etflt_id(TXGBE_RXD_PTID(pkt_info));
1088 if (likely(-1 != etfid))
1089 return ip_pkt_etqf_map[etfid] |
1090 ip_rss_types_map[TXGBE_RXD_RSSTYPE(pkt_info)];
1092 return ip_rss_types_map[TXGBE_RXD_RSSTYPE(pkt_info)];
1094 return ip_rss_types_map[TXGBE_RXD_RSSTYPE(pkt_info)];
1098 static inline uint64_t
1099 rx_desc_status_to_pkt_flags(uint32_t rx_status, uint64_t vlan_flags)
1104 * Check if VLAN present only.
1105 * Do not check whether L3/L4 rx checksum done by NIC or not,
1106 * That can be found from rte_eth_rxmode.offloads flag
1108 pkt_flags = (rx_status & TXGBE_RXD_STAT_VLAN &&
1109 vlan_flags & PKT_RX_VLAN_STRIPPED)
1112 #ifdef RTE_LIBRTE_IEEE1588
1113 if (rx_status & TXGBE_RXD_STAT_1588)
1114 pkt_flags = pkt_flags | PKT_RX_IEEE1588_TMST;
1119 static inline uint64_t
1120 rx_desc_error_to_pkt_flags(uint32_t rx_status)
1122 uint64_t pkt_flags = 0;
1124 /* checksum offload can't be disabled */
1125 if (rx_status & TXGBE_RXD_STAT_IPCS) {
1126 pkt_flags |= (rx_status & TXGBE_RXD_ERR_IPCS
1127 ? PKT_RX_IP_CKSUM_BAD : PKT_RX_IP_CKSUM_GOOD);
1130 if (rx_status & TXGBE_RXD_STAT_L4CS) {
1131 pkt_flags |= (rx_status & TXGBE_RXD_ERR_L4CS
1132 ? PKT_RX_L4_CKSUM_BAD : PKT_RX_L4_CKSUM_GOOD);
1135 if (rx_status & TXGBE_RXD_STAT_EIPCS &&
1136 rx_status & TXGBE_RXD_ERR_EIPCS) {
1137 pkt_flags |= PKT_RX_EIP_CKSUM_BAD;
1140 #ifdef RTE_LIB_SECURITY
1141 if (rx_status & TXGBE_RXD_STAT_SECP) {
1142 pkt_flags |= PKT_RX_SEC_OFFLOAD;
1143 if (rx_status & TXGBE_RXD_ERR_SECERR)
1144 pkt_flags |= PKT_RX_SEC_OFFLOAD_FAILED;
1152 * LOOK_AHEAD defines how many desc statuses to check beyond the
1153 * current descriptor.
1154 * It must be a pound define for optimal performance.
1155 * Do not change the value of LOOK_AHEAD, as the txgbe_rx_scan_hw_ring
1156 * function only works with LOOK_AHEAD=8.
1158 #define LOOK_AHEAD 8
1159 #if (LOOK_AHEAD != 8)
1160 #error "PMD TXGBE: LOOK_AHEAD must be 8\n"
1163 txgbe_rx_scan_hw_ring(struct txgbe_rx_queue *rxq)
1165 volatile struct txgbe_rx_desc *rxdp;
1166 struct txgbe_rx_entry *rxep;
1167 struct rte_mbuf *mb;
1171 uint32_t s[LOOK_AHEAD];
1172 uint32_t pkt_info[LOOK_AHEAD];
1173 int i, j, nb_rx = 0;
1176 /* get references to current descriptor and S/W ring entry */
1177 rxdp = &rxq->rx_ring[rxq->rx_tail];
1178 rxep = &rxq->sw_ring[rxq->rx_tail];
1180 status = rxdp->qw1.lo.status;
1181 /* check to make sure there is at least 1 packet to receive */
1182 if (!(status & rte_cpu_to_le_32(TXGBE_RXD_STAT_DD)))
1186 * Scan LOOK_AHEAD descriptors at a time to determine which descriptors
1187 * reference packets that are ready to be received.
1189 for (i = 0; i < RTE_PMD_TXGBE_RX_MAX_BURST;
1190 i += LOOK_AHEAD, rxdp += LOOK_AHEAD, rxep += LOOK_AHEAD) {
1191 /* Read desc statuses backwards to avoid race condition */
1192 for (j = 0; j < LOOK_AHEAD; j++)
1193 s[j] = rte_le_to_cpu_32(rxdp[j].qw1.lo.status);
1195 rte_atomic_thread_fence(__ATOMIC_ACQUIRE);
1197 /* Compute how many status bits were set */
1198 for (nb_dd = 0; nb_dd < LOOK_AHEAD &&
1199 (s[nb_dd] & TXGBE_RXD_STAT_DD); nb_dd++)
1202 for (j = 0; j < nb_dd; j++)
1203 pkt_info[j] = rte_le_to_cpu_32(rxdp[j].qw0.dw0);
1207 /* Translate descriptor info to mbuf format */
1208 for (j = 0; j < nb_dd; ++j) {
1210 pkt_len = rte_le_to_cpu_16(rxdp[j].qw1.hi.len) -
1212 mb->data_len = pkt_len;
1213 mb->pkt_len = pkt_len;
1214 mb->vlan_tci = rte_le_to_cpu_16(rxdp[j].qw1.hi.tag);
1216 /* convert descriptor fields to rte mbuf flags */
1217 pkt_flags = rx_desc_status_to_pkt_flags(s[j],
1219 pkt_flags |= rx_desc_error_to_pkt_flags(s[j]);
1221 txgbe_rxd_pkt_info_to_pkt_flags(pkt_info[j]);
1222 mb->ol_flags = pkt_flags;
1224 txgbe_rxd_pkt_info_to_pkt_type(pkt_info[j],
1225 rxq->pkt_type_mask);
1227 if (likely(pkt_flags & PKT_RX_RSS_HASH))
1229 rte_le_to_cpu_32(rxdp[j].qw0.dw1);
1230 else if (pkt_flags & PKT_RX_FDIR) {
1231 mb->hash.fdir.hash =
1232 rte_le_to_cpu_16(rxdp[j].qw0.hi.csum) &
1233 TXGBE_ATR_HASH_MASK;
1235 rte_le_to_cpu_16(rxdp[j].qw0.hi.ipid);
1239 /* Move mbuf pointers from the S/W ring to the stage */
1240 for (j = 0; j < LOOK_AHEAD; ++j)
1241 rxq->rx_stage[i + j] = rxep[j].mbuf;
1243 /* stop if all requested packets could not be received */
1244 if (nb_dd != LOOK_AHEAD)
1248 /* clear software ring entries so we can cleanup correctly */
1249 for (i = 0; i < nb_rx; ++i)
1250 rxq->sw_ring[rxq->rx_tail + i].mbuf = NULL;
1256 txgbe_rx_alloc_bufs(struct txgbe_rx_queue *rxq, bool reset_mbuf)
1258 volatile struct txgbe_rx_desc *rxdp;
1259 struct txgbe_rx_entry *rxep;
1260 struct rte_mbuf *mb;
1265 /* allocate buffers in bulk directly into the S/W ring */
1266 alloc_idx = rxq->rx_free_trigger - (rxq->rx_free_thresh - 1);
1267 rxep = &rxq->sw_ring[alloc_idx];
1268 diag = rte_mempool_get_bulk(rxq->mb_pool, (void *)rxep,
1269 rxq->rx_free_thresh);
1270 if (unlikely(diag != 0))
1273 rxdp = &rxq->rx_ring[alloc_idx];
1274 for (i = 0; i < rxq->rx_free_thresh; ++i) {
1275 /* populate the static rte mbuf fields */
1278 mb->port = rxq->port_id;
1280 rte_mbuf_refcnt_set(mb, 1);
1281 mb->data_off = RTE_PKTMBUF_HEADROOM;
1283 /* populate the descriptors */
1284 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mb));
1285 TXGBE_RXD_HDRADDR(&rxdp[i], 0);
1286 TXGBE_RXD_PKTADDR(&rxdp[i], dma_addr);
1289 /* update state of internal queue structure */
1290 rxq->rx_free_trigger = rxq->rx_free_trigger + rxq->rx_free_thresh;
1291 if (rxq->rx_free_trigger >= rxq->nb_rx_desc)
1292 rxq->rx_free_trigger = rxq->rx_free_thresh - 1;
1298 static inline uint16_t
1299 txgbe_rx_fill_from_stage(struct txgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
1302 struct rte_mbuf **stage = &rxq->rx_stage[rxq->rx_next_avail];
1305 /* how many packets are ready to return? */
1306 nb_pkts = (uint16_t)RTE_MIN(nb_pkts, rxq->rx_nb_avail);
1308 /* copy mbuf pointers to the application's packet list */
1309 for (i = 0; i < nb_pkts; ++i)
1310 rx_pkts[i] = stage[i];
1312 /* update internal queue state */
1313 rxq->rx_nb_avail = (uint16_t)(rxq->rx_nb_avail - nb_pkts);
1314 rxq->rx_next_avail = (uint16_t)(rxq->rx_next_avail + nb_pkts);
1319 static inline uint16_t
1320 txgbe_rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
1323 struct txgbe_rx_queue *rxq = (struct txgbe_rx_queue *)rx_queue;
1324 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id];
1327 /* Any previously recv'd pkts will be returned from the Rx stage */
1328 if (rxq->rx_nb_avail)
1329 return txgbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
1331 /* Scan the H/W ring for packets to receive */
1332 nb_rx = (uint16_t)txgbe_rx_scan_hw_ring(rxq);
1334 /* update internal queue state */
1335 rxq->rx_next_avail = 0;
1336 rxq->rx_nb_avail = nb_rx;
1337 rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_rx);
1339 /* if required, allocate new buffers to replenish descriptors */
1340 if (rxq->rx_tail > rxq->rx_free_trigger) {
1341 uint16_t cur_free_trigger = rxq->rx_free_trigger;
1343 if (txgbe_rx_alloc_bufs(rxq, true) != 0) {
1346 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u "
1347 "queue_id=%u", (uint16_t)rxq->port_id,
1348 (uint16_t)rxq->queue_id);
1350 dev->data->rx_mbuf_alloc_failed +=
1351 rxq->rx_free_thresh;
1354 * Need to rewind any previous receives if we cannot
1355 * allocate new buffers to replenish the old ones.
1357 rxq->rx_nb_avail = 0;
1358 rxq->rx_tail = (uint16_t)(rxq->rx_tail - nb_rx);
1359 for (i = 0, j = rxq->rx_tail; i < nb_rx; ++i, ++j)
1360 rxq->sw_ring[j].mbuf = rxq->rx_stage[i];
1365 /* update tail pointer */
1367 txgbe_set32_relaxed(rxq->rdt_reg_addr, cur_free_trigger);
1370 if (rxq->rx_tail >= rxq->nb_rx_desc)
1373 /* received any packets this loop? */
1374 if (rxq->rx_nb_avail)
1375 return txgbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
1380 /* split requests into chunks of size RTE_PMD_TXGBE_RX_MAX_BURST */
1382 txgbe_recv_pkts_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
1387 if (unlikely(nb_pkts == 0))
1390 if (likely(nb_pkts <= RTE_PMD_TXGBE_RX_MAX_BURST))
1391 return txgbe_rx_recv_pkts(rx_queue, rx_pkts, nb_pkts);
1393 /* request is relatively large, chunk it up */
1398 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_TXGBE_RX_MAX_BURST);
1399 ret = txgbe_rx_recv_pkts(rx_queue, &rx_pkts[nb_rx], n);
1400 nb_rx = (uint16_t)(nb_rx + ret);
1401 nb_pkts = (uint16_t)(nb_pkts - ret);
1410 txgbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
1413 struct txgbe_rx_queue *rxq;
1414 volatile struct txgbe_rx_desc *rx_ring;
1415 volatile struct txgbe_rx_desc *rxdp;
1416 struct txgbe_rx_entry *sw_ring;
1417 struct txgbe_rx_entry *rxe;
1418 struct rte_mbuf *rxm;
1419 struct rte_mbuf *nmb;
1420 struct txgbe_rx_desc rxd;
1433 rx_id = rxq->rx_tail;
1434 rx_ring = rxq->rx_ring;
1435 sw_ring = rxq->sw_ring;
1436 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id];
1437 while (nb_rx < nb_pkts) {
1439 * The order of operations here is important as the DD status
1440 * bit must not be read after any other descriptor fields.
1441 * rx_ring and rxdp are pointing to volatile data so the order
1442 * of accesses cannot be reordered by the compiler. If they were
1443 * not volatile, they could be reordered which could lead to
1444 * using invalid descriptor fields when read from rxd.
1446 rxdp = &rx_ring[rx_id];
1447 staterr = rxdp->qw1.lo.status;
1448 if (!(staterr & rte_cpu_to_le_32(TXGBE_RXD_STAT_DD)))
1455 * If the TXGBE_RXD_STAT_EOP flag is not set, the RX packet
1456 * is likely to be invalid and to be dropped by the various
1457 * validation checks performed by the network stack.
1459 * Allocate a new mbuf to replenish the RX ring descriptor.
1460 * If the allocation fails:
1461 * - arrange for that RX descriptor to be the first one
1462 * being parsed the next time the receive function is
1463 * invoked [on the same queue].
1465 * - Stop parsing the RX ring and return immediately.
1467 * This policy do not drop the packet received in the RX
1468 * descriptor for which the allocation of a new mbuf failed.
1469 * Thus, it allows that packet to be later retrieved if
1470 * mbuf have been freed in the mean time.
1471 * As a side effect, holding RX descriptors instead of
1472 * systematically giving them back to the NIC may lead to
1473 * RX ring exhaustion situations.
1474 * However, the NIC can gracefully prevent such situations
1475 * to happen by sending specific "back-pressure" flow control
1476 * frames to its peer(s).
1478 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u "
1479 "ext_err_stat=0x%08x pkt_len=%u",
1480 (uint16_t)rxq->port_id, (uint16_t)rxq->queue_id,
1481 (uint16_t)rx_id, (uint32_t)staterr,
1482 (uint16_t)rte_le_to_cpu_16(rxd.qw1.hi.len));
1484 nmb = rte_mbuf_raw_alloc(rxq->mb_pool);
1486 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u "
1487 "queue_id=%u", (uint16_t)rxq->port_id,
1488 (uint16_t)rxq->queue_id);
1489 dev->data->rx_mbuf_alloc_failed++;
1494 rxe = &sw_ring[rx_id];
1496 if (rx_id == rxq->nb_rx_desc)
1499 /* Prefetch next mbuf while processing current one. */
1500 rte_txgbe_prefetch(sw_ring[rx_id].mbuf);
1503 * When next RX descriptor is on a cache-line boundary,
1504 * prefetch the next 4 RX descriptors and the next 8 pointers
1507 if ((rx_id & 0x3) == 0) {
1508 rte_txgbe_prefetch(&rx_ring[rx_id]);
1509 rte_txgbe_prefetch(&sw_ring[rx_id]);
1514 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
1515 TXGBE_RXD_HDRADDR(rxdp, 0);
1516 TXGBE_RXD_PKTADDR(rxdp, dma_addr);
1519 * Initialize the returned mbuf.
1520 * 1) setup generic mbuf fields:
1521 * - number of segments,
1524 * - RX port identifier.
1525 * 2) integrate hardware offload data, if any:
1526 * - RSS flag & hash,
1527 * - IP checksum flag,
1528 * - VLAN TCI, if any,
1531 pkt_len = (uint16_t)(rte_le_to_cpu_16(rxd.qw1.hi.len) -
1533 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1534 rte_packet_prefetch((char *)rxm->buf_addr + rxm->data_off);
1537 rxm->pkt_len = pkt_len;
1538 rxm->data_len = pkt_len;
1539 rxm->port = rxq->port_id;
1541 pkt_info = rte_le_to_cpu_32(rxd.qw0.dw0);
1542 /* Only valid if PKT_RX_VLAN set in pkt_flags */
1543 rxm->vlan_tci = rte_le_to_cpu_16(rxd.qw1.hi.tag);
1545 pkt_flags = rx_desc_status_to_pkt_flags(staterr,
1547 pkt_flags |= rx_desc_error_to_pkt_flags(staterr);
1548 pkt_flags |= txgbe_rxd_pkt_info_to_pkt_flags(pkt_info);
1549 rxm->ol_flags = pkt_flags;
1550 rxm->packet_type = txgbe_rxd_pkt_info_to_pkt_type(pkt_info,
1551 rxq->pkt_type_mask);
1553 if (likely(pkt_flags & PKT_RX_RSS_HASH)) {
1554 rxm->hash.rss = rte_le_to_cpu_32(rxd.qw0.dw1);
1555 } else if (pkt_flags & PKT_RX_FDIR) {
1556 rxm->hash.fdir.hash =
1557 rte_le_to_cpu_16(rxd.qw0.hi.csum) &
1558 TXGBE_ATR_HASH_MASK;
1559 rxm->hash.fdir.id = rte_le_to_cpu_16(rxd.qw0.hi.ipid);
1562 * Store the mbuf address into the next entry of the array
1563 * of returned packets.
1565 rx_pkts[nb_rx++] = rxm;
1567 rxq->rx_tail = rx_id;
1570 * If the number of free RX descriptors is greater than the RX free
1571 * threshold of the queue, advance the Receive Descriptor Tail (RDT)
1573 * Update the RDT with the value of the last processed RX descriptor
1574 * minus 1, to guarantee that the RDT register is never equal to the
1575 * RDH register, which creates a "full" ring situation from the
1576 * hardware point of view...
1578 nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold);
1579 if (nb_hold > rxq->rx_free_thresh) {
1580 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u "
1581 "nb_hold=%u nb_rx=%u",
1582 (uint16_t)rxq->port_id, (uint16_t)rxq->queue_id,
1583 (uint16_t)rx_id, (uint16_t)nb_hold,
1585 rx_id = (uint16_t)((rx_id == 0) ?
1586 (rxq->nb_rx_desc - 1) : (rx_id - 1));
1587 txgbe_set32(rxq->rdt_reg_addr, rx_id);
1590 rxq->nb_rx_hold = nb_hold;
1595 * txgbe_fill_cluster_head_buf - fill the first mbuf of the returned packet
1597 * Fill the following info in the HEAD buffer of the Rx cluster:
1598 * - RX port identifier
1599 * - hardware offload data, if any:
1601 * - IP checksum flag
1602 * - VLAN TCI, if any
1604 * @head HEAD of the packet cluster
1605 * @desc HW descriptor to get data from
1606 * @rxq Pointer to the Rx queue
1609 txgbe_fill_cluster_head_buf(struct rte_mbuf *head, struct txgbe_rx_desc *desc,
1610 struct txgbe_rx_queue *rxq, uint32_t staterr)
1615 head->port = rxq->port_id;
1617 /* The vlan_tci field is only valid when PKT_RX_VLAN is
1618 * set in the pkt_flags field.
1620 head->vlan_tci = rte_le_to_cpu_16(desc->qw1.hi.tag);
1621 pkt_info = rte_le_to_cpu_32(desc->qw0.dw0);
1622 pkt_flags = rx_desc_status_to_pkt_flags(staterr, rxq->vlan_flags);
1623 pkt_flags |= rx_desc_error_to_pkt_flags(staterr);
1624 pkt_flags |= txgbe_rxd_pkt_info_to_pkt_flags(pkt_info);
1625 head->ol_flags = pkt_flags;
1626 head->packet_type = txgbe_rxd_pkt_info_to_pkt_type(pkt_info,
1627 rxq->pkt_type_mask);
1629 if (likely(pkt_flags & PKT_RX_RSS_HASH)) {
1630 head->hash.rss = rte_le_to_cpu_32(desc->qw0.dw1);
1631 } else if (pkt_flags & PKT_RX_FDIR) {
1632 head->hash.fdir.hash = rte_le_to_cpu_16(desc->qw0.hi.csum)
1633 & TXGBE_ATR_HASH_MASK;
1634 head->hash.fdir.id = rte_le_to_cpu_16(desc->qw0.hi.ipid);
1639 * txgbe_recv_pkts_lro - receive handler for and LRO case.
1641 * @rx_queue Rx queue handle
1642 * @rx_pkts table of received packets
1643 * @nb_pkts size of rx_pkts table
1644 * @bulk_alloc if TRUE bulk allocation is used for a HW ring refilling
1646 * Handles the Rx HW ring completions when RSC feature is configured. Uses an
1647 * additional ring of txgbe_rsc_entry's that will hold the relevant RSC info.
1649 * We use the same logic as in Linux and in FreeBSD txgbe drivers:
1650 * 1) When non-EOP RSC completion arrives:
1651 * a) Update the HEAD of the current RSC aggregation cluster with the new
1652 * segment's data length.
1653 * b) Set the "next" pointer of the current segment to point to the segment
1654 * at the NEXTP index.
1655 * c) Pass the HEAD of RSC aggregation cluster on to the next NEXTP entry
1656 * in the sw_rsc_ring.
1657 * 2) When EOP arrives we just update the cluster's total length and offload
1658 * flags and deliver the cluster up to the upper layers. In our case - put it
1659 * in the rx_pkts table.
1661 * Returns the number of received packets/clusters (according to the "bulk
1662 * receive" interface).
1664 static inline uint16_t
1665 txgbe_recv_pkts_lro(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts,
1668 struct txgbe_rx_queue *rxq = rx_queue;
1669 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id];
1670 volatile struct txgbe_rx_desc *rx_ring = rxq->rx_ring;
1671 struct txgbe_rx_entry *sw_ring = rxq->sw_ring;
1672 struct txgbe_scattered_rx_entry *sw_sc_ring = rxq->sw_sc_ring;
1673 uint16_t rx_id = rxq->rx_tail;
1675 uint16_t nb_hold = rxq->nb_rx_hold;
1676 uint16_t prev_id = rxq->rx_tail;
1678 while (nb_rx < nb_pkts) {
1680 struct txgbe_rx_entry *rxe;
1681 struct txgbe_scattered_rx_entry *sc_entry;
1682 struct txgbe_scattered_rx_entry *next_sc_entry = NULL;
1683 struct txgbe_rx_entry *next_rxe = NULL;
1684 struct rte_mbuf *first_seg;
1685 struct rte_mbuf *rxm;
1686 struct rte_mbuf *nmb = NULL;
1687 struct txgbe_rx_desc rxd;
1690 volatile struct txgbe_rx_desc *rxdp;
1695 * The code in this whole file uses the volatile pointer to
1696 * ensure the read ordering of the status and the rest of the
1697 * descriptor fields (on the compiler level only!!!). This is so
1698 * UGLY - why not to just use the compiler barrier instead? DPDK
1699 * even has the rte_compiler_barrier() for that.
1701 * But most importantly this is just wrong because this doesn't
1702 * ensure memory ordering in a general case at all. For
1703 * instance, DPDK is supposed to work on Power CPUs where
1704 * compiler barrier may just not be enough!
1706 * I tried to write only this function properly to have a
1707 * starting point (as a part of an LRO/RSC series) but the
1708 * compiler cursed at me when I tried to cast away the
1709 * "volatile" from rx_ring (yes, it's volatile too!!!). So, I'm
1710 * keeping it the way it is for now.
1712 * The code in this file is broken in so many other places and
1713 * will just not work on a big endian CPU anyway therefore the
1714 * lines below will have to be revisited together with the rest
1718 * - Get rid of "volatile" and let the compiler do its job.
1719 * - Use the proper memory barrier (rte_rmb()) to ensure the
1720 * memory ordering below.
1722 rxdp = &rx_ring[rx_id];
1723 staterr = rte_le_to_cpu_32(rxdp->qw1.lo.status);
1725 if (!(staterr & TXGBE_RXD_STAT_DD))
1730 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u "
1731 "staterr=0x%x data_len=%u",
1732 rxq->port_id, rxq->queue_id, rx_id, staterr,
1733 rte_le_to_cpu_16(rxd.qw1.hi.len));
1736 nmb = rte_mbuf_raw_alloc(rxq->mb_pool);
1738 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed "
1739 "port_id=%u queue_id=%u",
1740 rxq->port_id, rxq->queue_id);
1742 dev->data->rx_mbuf_alloc_failed++;
1745 } else if (nb_hold > rxq->rx_free_thresh) {
1746 uint16_t next_rdt = rxq->rx_free_trigger;
1748 if (!txgbe_rx_alloc_bufs(rxq, false)) {
1750 txgbe_set32_relaxed(rxq->rdt_reg_addr,
1752 nb_hold -= rxq->rx_free_thresh;
1754 PMD_RX_LOG(DEBUG, "RX bulk alloc failed "
1755 "port_id=%u queue_id=%u",
1756 rxq->port_id, rxq->queue_id);
1758 dev->data->rx_mbuf_alloc_failed++;
1764 rxe = &sw_ring[rx_id];
1765 eop = staterr & TXGBE_RXD_STAT_EOP;
1767 next_id = rx_id + 1;
1768 if (next_id == rxq->nb_rx_desc)
1771 /* Prefetch next mbuf while processing current one. */
1772 rte_txgbe_prefetch(sw_ring[next_id].mbuf);
1775 * When next RX descriptor is on a cache-line boundary,
1776 * prefetch the next 4 RX descriptors and the next 4 pointers
1779 if ((next_id & 0x3) == 0) {
1780 rte_txgbe_prefetch(&rx_ring[next_id]);
1781 rte_txgbe_prefetch(&sw_ring[next_id]);
1788 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
1790 * Update RX descriptor with the physical address of the
1791 * new data buffer of the new allocated mbuf.
1795 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1796 TXGBE_RXD_HDRADDR(rxdp, 0);
1797 TXGBE_RXD_PKTADDR(rxdp, dma);
1803 * Set data length & data buffer address of mbuf.
1805 data_len = rte_le_to_cpu_16(rxd.qw1.hi.len);
1806 rxm->data_len = data_len;
1811 * Get next descriptor index:
1812 * - For RSC it's in the NEXTP field.
1813 * - For a scattered packet - it's just a following
1816 if (TXGBE_RXD_RSCCNT(rxd.qw0.dw0))
1817 nextp_id = TXGBE_RXD_NEXTP(staterr);
1821 next_sc_entry = &sw_sc_ring[nextp_id];
1822 next_rxe = &sw_ring[nextp_id];
1823 rte_txgbe_prefetch(next_rxe);
1826 sc_entry = &sw_sc_ring[rx_id];
1827 first_seg = sc_entry->fbuf;
1828 sc_entry->fbuf = NULL;
1831 * If this is the first buffer of the received packet,
1832 * set the pointer to the first mbuf of the packet and
1833 * initialize its context.
1834 * Otherwise, update the total length and the number of segments
1835 * of the current scattered packet, and update the pointer to
1836 * the last mbuf of the current packet.
1838 if (first_seg == NULL) {
1840 first_seg->pkt_len = data_len;
1841 first_seg->nb_segs = 1;
1843 first_seg->pkt_len += data_len;
1844 first_seg->nb_segs++;
1851 * If this is not the last buffer of the received packet, update
1852 * the pointer to the first mbuf at the NEXTP entry in the
1853 * sw_sc_ring and continue to parse the RX ring.
1855 if (!eop && next_rxe) {
1856 rxm->next = next_rxe->mbuf;
1857 next_sc_entry->fbuf = first_seg;
1861 /* Initialize the first mbuf of the returned packet */
1862 txgbe_fill_cluster_head_buf(first_seg, &rxd, rxq, staterr);
1865 * Deal with the case, when HW CRC srip is disabled.
1866 * That can't happen when LRO is enabled, but still could
1867 * happen for scattered RX mode.
1869 first_seg->pkt_len -= rxq->crc_len;
1870 if (unlikely(rxm->data_len <= rxq->crc_len)) {
1871 struct rte_mbuf *lp;
1873 for (lp = first_seg; lp->next != rxm; lp = lp->next)
1876 first_seg->nb_segs--;
1877 lp->data_len -= rxq->crc_len - rxm->data_len;
1879 rte_pktmbuf_free_seg(rxm);
1881 rxm->data_len -= rxq->crc_len;
1884 /* Prefetch data of first segment, if configured to do so. */
1885 rte_packet_prefetch((char *)first_seg->buf_addr +
1886 first_seg->data_off);
1889 * Store the mbuf address into the next entry of the array
1890 * of returned packets.
1892 rx_pkts[nb_rx++] = first_seg;
1896 * Record index of the next RX descriptor to probe.
1898 rxq->rx_tail = rx_id;
1901 * If the number of free RX descriptors is greater than the RX free
1902 * threshold of the queue, advance the Receive Descriptor Tail (RDT)
1904 * Update the RDT with the value of the last processed RX descriptor
1905 * minus 1, to guarantee that the RDT register is never equal to the
1906 * RDH register, which creates a "full" ring situation from the
1907 * hardware point of view...
1909 if (!bulk_alloc && nb_hold > rxq->rx_free_thresh) {
1910 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u "
1911 "nb_hold=%u nb_rx=%u",
1912 rxq->port_id, rxq->queue_id, rx_id, nb_hold, nb_rx);
1915 txgbe_set32_relaxed(rxq->rdt_reg_addr, prev_id);
1919 rxq->nb_rx_hold = nb_hold;
1924 txgbe_recv_pkts_lro_single_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
1927 return txgbe_recv_pkts_lro(rx_queue, rx_pkts, nb_pkts, false);
1931 txgbe_recv_pkts_lro_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
1934 return txgbe_recv_pkts_lro(rx_queue, rx_pkts, nb_pkts, true);
1938 txgbe_get_rx_queue_offloads(struct rte_eth_dev *dev __rte_unused)
1940 return DEV_RX_OFFLOAD_VLAN_STRIP;
1944 txgbe_get_rx_port_offloads(struct rte_eth_dev *dev)
1947 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
1948 struct rte_eth_dev_sriov *sriov = &RTE_ETH_DEV_SRIOV(dev);
1950 offloads = DEV_RX_OFFLOAD_IPV4_CKSUM |
1951 DEV_RX_OFFLOAD_UDP_CKSUM |
1952 DEV_RX_OFFLOAD_TCP_CKSUM |
1953 DEV_RX_OFFLOAD_KEEP_CRC |
1954 DEV_RX_OFFLOAD_JUMBO_FRAME |
1955 DEV_RX_OFFLOAD_VLAN_FILTER |
1956 DEV_RX_OFFLOAD_RSS_HASH |
1957 DEV_RX_OFFLOAD_SCATTER;
1959 if (!txgbe_is_vf(dev))
1960 offloads |= (DEV_RX_OFFLOAD_VLAN_FILTER |
1961 DEV_RX_OFFLOAD_QINQ_STRIP |
1962 DEV_RX_OFFLOAD_VLAN_EXTEND);
1965 * RSC is only supported by PF devices in a non-SR-IOV
1968 if (hw->mac.type == txgbe_mac_raptor && !sriov->active)
1969 offloads |= DEV_RX_OFFLOAD_TCP_LRO;
1971 if (hw->mac.type == txgbe_mac_raptor)
1972 offloads |= DEV_RX_OFFLOAD_MACSEC_STRIP;
1974 offloads |= DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM;
1976 #ifdef RTE_LIB_SECURITY
1977 if (dev->security_ctx)
1978 offloads |= DEV_RX_OFFLOAD_SECURITY;
1984 static void __rte_cold
1985 txgbe_tx_queue_release_mbufs(struct txgbe_tx_queue *txq)
1989 if (txq->sw_ring != NULL) {
1990 for (i = 0; i < txq->nb_tx_desc; i++) {
1991 if (txq->sw_ring[i].mbuf != NULL) {
1992 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
1993 txq->sw_ring[i].mbuf = NULL;
2000 txgbe_tx_done_cleanup_full(struct txgbe_tx_queue *txq, uint32_t free_cnt)
2002 struct txgbe_tx_entry *swr_ring = txq->sw_ring;
2003 uint16_t i, tx_last, tx_id;
2004 uint16_t nb_tx_free_last;
2005 uint16_t nb_tx_to_clean;
2008 /* Start free mbuf from the next of tx_tail */
2009 tx_last = txq->tx_tail;
2010 tx_id = swr_ring[tx_last].next_id;
2012 if (txq->nb_tx_free == 0 && txgbe_xmit_cleanup(txq))
2015 nb_tx_to_clean = txq->nb_tx_free;
2016 nb_tx_free_last = txq->nb_tx_free;
2018 free_cnt = txq->nb_tx_desc;
2020 /* Loop through swr_ring to count the amount of
2021 * freeable mubfs and packets.
2023 for (pkt_cnt = 0; pkt_cnt < free_cnt; ) {
2024 for (i = 0; i < nb_tx_to_clean &&
2025 pkt_cnt < free_cnt &&
2026 tx_id != tx_last; i++) {
2027 if (swr_ring[tx_id].mbuf != NULL) {
2028 rte_pktmbuf_free_seg(swr_ring[tx_id].mbuf);
2029 swr_ring[tx_id].mbuf = NULL;
2032 * last segment in the packet,
2033 * increment packet count
2035 pkt_cnt += (swr_ring[tx_id].last_id == tx_id);
2038 tx_id = swr_ring[tx_id].next_id;
2041 if (pkt_cnt < free_cnt) {
2042 if (txgbe_xmit_cleanup(txq))
2045 nb_tx_to_clean = txq->nb_tx_free - nb_tx_free_last;
2046 nb_tx_free_last = txq->nb_tx_free;
2050 return (int)pkt_cnt;
2054 txgbe_tx_done_cleanup_simple(struct txgbe_tx_queue *txq,
2059 if (free_cnt == 0 || free_cnt > txq->nb_tx_desc)
2060 free_cnt = txq->nb_tx_desc;
2062 cnt = free_cnt - free_cnt % txq->tx_free_thresh;
2064 for (i = 0; i < cnt; i += n) {
2065 if (txq->nb_tx_desc - txq->nb_tx_free < txq->tx_free_thresh)
2068 n = txgbe_tx_free_bufs(txq);
2078 txgbe_dev_tx_done_cleanup(void *tx_queue, uint32_t free_cnt)
2080 struct txgbe_tx_queue *txq = (struct txgbe_tx_queue *)tx_queue;
2081 if (txq->offloads == 0 &&
2082 #ifdef RTE_LIB_SECURITY
2083 !(txq->using_ipsec) &&
2085 txq->tx_free_thresh >= RTE_PMD_TXGBE_TX_MAX_BURST)
2086 return txgbe_tx_done_cleanup_simple(txq, free_cnt);
2088 return txgbe_tx_done_cleanup_full(txq, free_cnt);
2091 static void __rte_cold
2092 txgbe_tx_free_swring(struct txgbe_tx_queue *txq)
2095 txq->sw_ring != NULL)
2096 rte_free(txq->sw_ring);
2099 static void __rte_cold
2100 txgbe_tx_queue_release(struct txgbe_tx_queue *txq)
2102 if (txq != NULL && txq->ops != NULL) {
2103 txq->ops->release_mbufs(txq);
2104 txq->ops->free_swring(txq);
2110 txgbe_dev_tx_queue_release(void *txq)
2112 txgbe_tx_queue_release(txq);
2115 /* (Re)set dynamic txgbe_tx_queue fields to defaults */
2116 static void __rte_cold
2117 txgbe_reset_tx_queue(struct txgbe_tx_queue *txq)
2119 static const struct txgbe_tx_desc zeroed_desc = {0};
2120 struct txgbe_tx_entry *txe = txq->sw_ring;
2123 /* Zero out HW ring memory */
2124 for (i = 0; i < txq->nb_tx_desc; i++)
2125 txq->tx_ring[i] = zeroed_desc;
2127 /* Initialize SW ring entries */
2128 prev = (uint16_t)(txq->nb_tx_desc - 1);
2129 for (i = 0; i < txq->nb_tx_desc; i++) {
2130 volatile struct txgbe_tx_desc *txd = &txq->tx_ring[i];
2132 txd->dw3 = rte_cpu_to_le_32(TXGBE_TXD_DD);
2135 txe[prev].next_id = i;
2139 txq->tx_next_dd = (uint16_t)(txq->tx_free_thresh - 1);
2143 * Always allow 1 descriptor to be un-allocated to avoid
2144 * a H/W race condition
2146 txq->last_desc_cleaned = (uint16_t)(txq->nb_tx_desc - 1);
2147 txq->nb_tx_free = (uint16_t)(txq->nb_tx_desc - 1);
2149 memset((void *)&txq->ctx_cache, 0,
2150 TXGBE_CTX_NUM * sizeof(struct txgbe_ctx_info));
2153 static const struct txgbe_txq_ops def_txq_ops = {
2154 .release_mbufs = txgbe_tx_queue_release_mbufs,
2155 .free_swring = txgbe_tx_free_swring,
2156 .reset = txgbe_reset_tx_queue,
2159 /* Takes an ethdev and a queue and sets up the tx function to be used based on
2160 * the queue parameters. Used in tx_queue_setup by primary process and then
2161 * in dev_init by secondary process when attaching to an existing ethdev.
2164 txgbe_set_tx_function(struct rte_eth_dev *dev, struct txgbe_tx_queue *txq)
2166 /* Use a simple Tx queue (no offloads, no multi segs) if possible */
2167 if (txq->offloads == 0 &&
2168 #ifdef RTE_LIB_SECURITY
2169 !(txq->using_ipsec) &&
2171 txq->tx_free_thresh >= RTE_PMD_TXGBE_TX_MAX_BURST) {
2172 PMD_INIT_LOG(DEBUG, "Using simple tx code path");
2173 dev->tx_pkt_burst = txgbe_xmit_pkts_simple;
2174 dev->tx_pkt_prepare = NULL;
2176 PMD_INIT_LOG(DEBUG, "Using full-featured tx code path");
2178 " - offloads = 0x%" PRIx64,
2181 " - tx_free_thresh = %lu [RTE_PMD_TXGBE_TX_MAX_BURST=%lu]",
2182 (unsigned long)txq->tx_free_thresh,
2183 (unsigned long)RTE_PMD_TXGBE_TX_MAX_BURST);
2184 dev->tx_pkt_burst = txgbe_xmit_pkts;
2185 dev->tx_pkt_prepare = txgbe_prep_pkts;
2190 txgbe_get_tx_queue_offloads(struct rte_eth_dev *dev)
2198 txgbe_get_tx_port_offloads(struct rte_eth_dev *dev)
2200 uint64_t tx_offload_capa;
2203 DEV_TX_OFFLOAD_VLAN_INSERT |
2204 DEV_TX_OFFLOAD_IPV4_CKSUM |
2205 DEV_TX_OFFLOAD_UDP_CKSUM |
2206 DEV_TX_OFFLOAD_TCP_CKSUM |
2207 DEV_TX_OFFLOAD_SCTP_CKSUM |
2208 DEV_TX_OFFLOAD_TCP_TSO |
2209 DEV_TX_OFFLOAD_UDP_TSO |
2210 DEV_TX_OFFLOAD_UDP_TNL_TSO |
2211 DEV_TX_OFFLOAD_IP_TNL_TSO |
2212 DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
2213 DEV_TX_OFFLOAD_GRE_TNL_TSO |
2214 DEV_TX_OFFLOAD_IPIP_TNL_TSO |
2215 DEV_TX_OFFLOAD_GENEVE_TNL_TSO |
2216 DEV_TX_OFFLOAD_MULTI_SEGS;
2218 if (!txgbe_is_vf(dev))
2219 tx_offload_capa |= DEV_TX_OFFLOAD_QINQ_INSERT;
2221 tx_offload_capa |= DEV_TX_OFFLOAD_MACSEC_INSERT;
2223 tx_offload_capa |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM;
2225 #ifdef RTE_LIB_SECURITY
2226 if (dev->security_ctx)
2227 tx_offload_capa |= DEV_TX_OFFLOAD_SECURITY;
2229 return tx_offload_capa;
2233 txgbe_dev_tx_queue_setup(struct rte_eth_dev *dev,
2236 unsigned int socket_id,
2237 const struct rte_eth_txconf *tx_conf)
2239 const struct rte_memzone *tz;
2240 struct txgbe_tx_queue *txq;
2241 struct txgbe_hw *hw;
2242 uint16_t tx_free_thresh;
2245 PMD_INIT_FUNC_TRACE();
2246 hw = TXGBE_DEV_HW(dev);
2248 offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads;
2251 * Validate number of transmit descriptors.
2252 * It must not exceed hardware maximum, and must be multiple
2255 if (nb_desc % TXGBE_TXD_ALIGN != 0 ||
2256 nb_desc > TXGBE_RING_DESC_MAX ||
2257 nb_desc < TXGBE_RING_DESC_MIN) {
2262 * The TX descriptor ring will be cleaned after txq->tx_free_thresh
2263 * descriptors are used or if the number of descriptors required
2264 * to transmit a packet is greater than the number of free TX
2266 * One descriptor in the TX ring is used as a sentinel to avoid a
2267 * H/W race condition, hence the maximum threshold constraints.
2268 * When set to zero use default values.
2270 tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ?
2271 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH);
2272 if (tx_free_thresh >= (nb_desc - 3)) {
2273 PMD_INIT_LOG(ERR, "tx_free_thresh must be less than the number of "
2274 "TX descriptors minus 3. (tx_free_thresh=%u "
2275 "port=%d queue=%d)",
2276 (unsigned int)tx_free_thresh,
2277 (int)dev->data->port_id, (int)queue_idx);
2281 if ((nb_desc % tx_free_thresh) != 0) {
2282 PMD_INIT_LOG(ERR, "tx_free_thresh must be a divisor of the "
2283 "number of TX descriptors. (tx_free_thresh=%u "
2284 "port=%d queue=%d)", (unsigned int)tx_free_thresh,
2285 (int)dev->data->port_id, (int)queue_idx);
2289 /* Free memory prior to re-allocation if needed... */
2290 if (dev->data->tx_queues[queue_idx] != NULL) {
2291 txgbe_tx_queue_release(dev->data->tx_queues[queue_idx]);
2292 dev->data->tx_queues[queue_idx] = NULL;
2295 /* First allocate the tx queue data structure */
2296 txq = rte_zmalloc_socket("ethdev TX queue",
2297 sizeof(struct txgbe_tx_queue),
2298 RTE_CACHE_LINE_SIZE, socket_id);
2303 * Allocate TX ring hardware descriptors. A memzone large enough to
2304 * handle the maximum ring size is allocated in order to allow for
2305 * resizing in later calls to the queue setup function.
2307 tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx,
2308 sizeof(struct txgbe_tx_desc) * TXGBE_RING_DESC_MAX,
2309 TXGBE_ALIGN, socket_id);
2311 txgbe_tx_queue_release(txq);
2315 txq->nb_tx_desc = nb_desc;
2316 txq->tx_free_thresh = tx_free_thresh;
2317 txq->pthresh = tx_conf->tx_thresh.pthresh;
2318 txq->hthresh = tx_conf->tx_thresh.hthresh;
2319 txq->wthresh = tx_conf->tx_thresh.wthresh;
2320 txq->queue_id = queue_idx;
2321 txq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ?
2322 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx);
2323 txq->port_id = dev->data->port_id;
2324 txq->offloads = offloads;
2325 txq->ops = &def_txq_ops;
2326 txq->tx_deferred_start = tx_conf->tx_deferred_start;
2327 #ifdef RTE_LIB_SECURITY
2328 txq->using_ipsec = !!(dev->data->dev_conf.txmode.offloads &
2329 DEV_TX_OFFLOAD_SECURITY);
2332 /* Modification to set tail pointer for virtual function
2333 * if vf is detected.
2335 if (hw->mac.type == txgbe_mac_raptor_vf) {
2336 txq->tdt_reg_addr = TXGBE_REG_ADDR(hw, TXGBE_TXWP(queue_idx));
2337 txq->tdc_reg_addr = TXGBE_REG_ADDR(hw, TXGBE_TXCFG(queue_idx));
2339 txq->tdt_reg_addr = TXGBE_REG_ADDR(hw,
2340 TXGBE_TXWP(txq->reg_idx));
2341 txq->tdc_reg_addr = TXGBE_REG_ADDR(hw,
2342 TXGBE_TXCFG(txq->reg_idx));
2345 txq->tx_ring_phys_addr = TMZ_PADDR(tz);
2346 txq->tx_ring = (struct txgbe_tx_desc *)TMZ_VADDR(tz);
2348 /* Allocate software ring */
2349 txq->sw_ring = rte_zmalloc_socket("txq->sw_ring",
2350 sizeof(struct txgbe_tx_entry) * nb_desc,
2351 RTE_CACHE_LINE_SIZE, socket_id);
2352 if (txq->sw_ring == NULL) {
2353 txgbe_tx_queue_release(txq);
2356 PMD_INIT_LOG(DEBUG, "sw_ring=%p hw_ring=%p dma_addr=0x%" PRIx64,
2357 txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr);
2359 /* set up scalar TX function as appropriate */
2360 txgbe_set_tx_function(dev, txq);
2362 txq->ops->reset(txq);
2364 dev->data->tx_queues[queue_idx] = txq;
2370 * txgbe_free_sc_cluster - free the not-yet-completed scattered cluster
2372 * The "next" pointer of the last segment of (not-yet-completed) RSC clusters
2373 * in the sw_rsc_ring is not set to NULL but rather points to the next
2374 * mbuf of this RSC aggregation (that has not been completed yet and still
2375 * resides on the HW ring). So, instead of calling for rte_pktmbuf_free() we
2376 * will just free first "nb_segs" segments of the cluster explicitly by calling
2377 * an rte_pktmbuf_free_seg().
2379 * @m scattered cluster head
2381 static void __rte_cold
2382 txgbe_free_sc_cluster(struct rte_mbuf *m)
2384 uint16_t i, nb_segs = m->nb_segs;
2385 struct rte_mbuf *next_seg;
2387 for (i = 0; i < nb_segs; i++) {
2389 rte_pktmbuf_free_seg(m);
2394 static void __rte_cold
2395 txgbe_rx_queue_release_mbufs(struct txgbe_rx_queue *rxq)
2399 if (rxq->sw_ring != NULL) {
2400 for (i = 0; i < rxq->nb_rx_desc; i++) {
2401 if (rxq->sw_ring[i].mbuf != NULL) {
2402 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
2403 rxq->sw_ring[i].mbuf = NULL;
2406 if (rxq->rx_nb_avail) {
2407 for (i = 0; i < rxq->rx_nb_avail; ++i) {
2408 struct rte_mbuf *mb;
2410 mb = rxq->rx_stage[rxq->rx_next_avail + i];
2411 rte_pktmbuf_free_seg(mb);
2413 rxq->rx_nb_avail = 0;
2417 if (rxq->sw_sc_ring)
2418 for (i = 0; i < rxq->nb_rx_desc; i++)
2419 if (rxq->sw_sc_ring[i].fbuf) {
2420 txgbe_free_sc_cluster(rxq->sw_sc_ring[i].fbuf);
2421 rxq->sw_sc_ring[i].fbuf = NULL;
2425 static void __rte_cold
2426 txgbe_rx_queue_release(struct txgbe_rx_queue *rxq)
2429 txgbe_rx_queue_release_mbufs(rxq);
2430 rte_free(rxq->sw_ring);
2431 rte_free(rxq->sw_sc_ring);
2437 txgbe_dev_rx_queue_release(void *rxq)
2439 txgbe_rx_queue_release(rxq);
2443 * Check if Rx Burst Bulk Alloc function can be used.
2445 * 0: the preconditions are satisfied and the bulk allocation function
2447 * -EINVAL: the preconditions are NOT satisfied and the default Rx burst
2448 * function must be used.
2450 static inline int __rte_cold
2451 check_rx_burst_bulk_alloc_preconditions(struct txgbe_rx_queue *rxq)
2456 * Make sure the following pre-conditions are satisfied:
2457 * rxq->rx_free_thresh >= RTE_PMD_TXGBE_RX_MAX_BURST
2458 * rxq->rx_free_thresh < rxq->nb_rx_desc
2459 * (rxq->nb_rx_desc % rxq->rx_free_thresh) == 0
2460 * Scattered packets are not supported. This should be checked
2461 * outside of this function.
2463 if (!(rxq->rx_free_thresh >= RTE_PMD_TXGBE_RX_MAX_BURST)) {
2464 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
2465 "rxq->rx_free_thresh=%d, "
2466 "RTE_PMD_TXGBE_RX_MAX_BURST=%d",
2467 rxq->rx_free_thresh, RTE_PMD_TXGBE_RX_MAX_BURST);
2469 } else if (!(rxq->rx_free_thresh < rxq->nb_rx_desc)) {
2470 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
2471 "rxq->rx_free_thresh=%d, "
2472 "rxq->nb_rx_desc=%d",
2473 rxq->rx_free_thresh, rxq->nb_rx_desc);
2475 } else if (!((rxq->nb_rx_desc % rxq->rx_free_thresh) == 0)) {
2476 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
2477 "rxq->nb_rx_desc=%d, "
2478 "rxq->rx_free_thresh=%d",
2479 rxq->nb_rx_desc, rxq->rx_free_thresh);
2486 /* Reset dynamic txgbe_rx_queue fields back to defaults */
2487 static void __rte_cold
2488 txgbe_reset_rx_queue(struct txgbe_adapter *adapter, struct txgbe_rx_queue *rxq)
2490 static const struct txgbe_rx_desc zeroed_desc = {
2491 {{0}, {0} }, {{0}, {0} } };
2493 uint16_t len = rxq->nb_rx_desc;
2496 * By default, the Rx queue setup function allocates enough memory for
2497 * TXGBE_RING_DESC_MAX. The Rx Burst bulk allocation function requires
2498 * extra memory at the end of the descriptor ring to be zero'd out.
2500 if (adapter->rx_bulk_alloc_allowed)
2501 /* zero out extra memory */
2502 len += RTE_PMD_TXGBE_RX_MAX_BURST;
2505 * Zero out HW ring memory. Zero out extra memory at the end of
2506 * the H/W ring so look-ahead logic in Rx Burst bulk alloc function
2507 * reads extra memory as zeros.
2509 for (i = 0; i < len; i++)
2510 rxq->rx_ring[i] = zeroed_desc;
2513 * initialize extra software ring entries. Space for these extra
2514 * entries is always allocated
2516 memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf));
2517 for (i = rxq->nb_rx_desc; i < len; ++i)
2518 rxq->sw_ring[i].mbuf = &rxq->fake_mbuf;
2520 rxq->rx_nb_avail = 0;
2521 rxq->rx_next_avail = 0;
2522 rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1);
2524 rxq->nb_rx_hold = 0;
2525 rxq->pkt_first_seg = NULL;
2526 rxq->pkt_last_seg = NULL;
2530 txgbe_dev_rx_queue_setup(struct rte_eth_dev *dev,
2533 unsigned int socket_id,
2534 const struct rte_eth_rxconf *rx_conf,
2535 struct rte_mempool *mp)
2537 const struct rte_memzone *rz;
2538 struct txgbe_rx_queue *rxq;
2539 struct txgbe_hw *hw;
2541 struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev);
2544 PMD_INIT_FUNC_TRACE();
2545 hw = TXGBE_DEV_HW(dev);
2547 offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads;
2550 * Validate number of receive descriptors.
2551 * It must not exceed hardware maximum, and must be multiple
2554 if (nb_desc % TXGBE_RXD_ALIGN != 0 ||
2555 nb_desc > TXGBE_RING_DESC_MAX ||
2556 nb_desc < TXGBE_RING_DESC_MIN) {
2560 /* Free memory prior to re-allocation if needed... */
2561 if (dev->data->rx_queues[queue_idx] != NULL) {
2562 txgbe_rx_queue_release(dev->data->rx_queues[queue_idx]);
2563 dev->data->rx_queues[queue_idx] = NULL;
2566 /* First allocate the rx queue data structure */
2567 rxq = rte_zmalloc_socket("ethdev RX queue",
2568 sizeof(struct txgbe_rx_queue),
2569 RTE_CACHE_LINE_SIZE, socket_id);
2573 rxq->nb_rx_desc = nb_desc;
2574 rxq->rx_free_thresh = rx_conf->rx_free_thresh;
2575 rxq->queue_id = queue_idx;
2576 rxq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ?
2577 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx);
2578 rxq->port_id = dev->data->port_id;
2579 if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC)
2580 rxq->crc_len = RTE_ETHER_CRC_LEN;
2583 rxq->drop_en = rx_conf->rx_drop_en;
2584 rxq->rx_deferred_start = rx_conf->rx_deferred_start;
2585 rxq->offloads = offloads;
2588 * The packet type in RX descriptor is different for different NICs.
2589 * So set different masks for different NICs.
2591 rxq->pkt_type_mask = TXGBE_PTID_MASK;
2594 * Allocate RX ring hardware descriptors. A memzone large enough to
2595 * handle the maximum ring size is allocated in order to allow for
2596 * resizing in later calls to the queue setup function.
2598 rz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx,
2599 RX_RING_SZ, TXGBE_ALIGN, socket_id);
2601 txgbe_rx_queue_release(rxq);
2606 * Zero init all the descriptors in the ring.
2608 memset(rz->addr, 0, RX_RING_SZ);
2611 * Modified to setup VFRDT for Virtual Function
2613 if (hw->mac.type == txgbe_mac_raptor_vf) {
2615 TXGBE_REG_ADDR(hw, TXGBE_RXWP(queue_idx));
2617 TXGBE_REG_ADDR(hw, TXGBE_RXRP(queue_idx));
2620 TXGBE_REG_ADDR(hw, TXGBE_RXWP(rxq->reg_idx));
2622 TXGBE_REG_ADDR(hw, TXGBE_RXRP(rxq->reg_idx));
2625 rxq->rx_ring_phys_addr = TMZ_PADDR(rz);
2626 rxq->rx_ring = (struct txgbe_rx_desc *)TMZ_VADDR(rz);
2629 * Certain constraints must be met in order to use the bulk buffer
2630 * allocation Rx burst function. If any of Rx queues doesn't meet them
2631 * the feature should be disabled for the whole port.
2633 if (check_rx_burst_bulk_alloc_preconditions(rxq)) {
2634 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Rx Bulk Alloc "
2635 "preconditions - canceling the feature for "
2636 "the whole port[%d]",
2637 rxq->queue_id, rxq->port_id);
2638 adapter->rx_bulk_alloc_allowed = false;
2642 * Allocate software ring. Allow for space at the end of the
2643 * S/W ring to make sure look-ahead logic in bulk alloc Rx burst
2644 * function does not access an invalid memory region.
2647 if (adapter->rx_bulk_alloc_allowed)
2648 len += RTE_PMD_TXGBE_RX_MAX_BURST;
2650 rxq->sw_ring = rte_zmalloc_socket("rxq->sw_ring",
2651 sizeof(struct txgbe_rx_entry) * len,
2652 RTE_CACHE_LINE_SIZE, socket_id);
2653 if (!rxq->sw_ring) {
2654 txgbe_rx_queue_release(rxq);
2659 * Always allocate even if it's not going to be needed in order to
2660 * simplify the code.
2662 * This ring is used in LRO and Scattered Rx cases and Scattered Rx may
2663 * be requested in txgbe_dev_rx_init(), which is called later from
2667 rte_zmalloc_socket("rxq->sw_sc_ring",
2668 sizeof(struct txgbe_scattered_rx_entry) * len,
2669 RTE_CACHE_LINE_SIZE, socket_id);
2670 if (!rxq->sw_sc_ring) {
2671 txgbe_rx_queue_release(rxq);
2675 PMD_INIT_LOG(DEBUG, "sw_ring=%p sw_sc_ring=%p hw_ring=%p "
2676 "dma_addr=0x%" PRIx64,
2677 rxq->sw_ring, rxq->sw_sc_ring, rxq->rx_ring,
2678 rxq->rx_ring_phys_addr);
2680 dev->data->rx_queues[queue_idx] = rxq;
2682 txgbe_reset_rx_queue(adapter, rxq);
2688 txgbe_dev_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id)
2690 #define TXGBE_RXQ_SCAN_INTERVAL 4
2691 volatile struct txgbe_rx_desc *rxdp;
2692 struct txgbe_rx_queue *rxq;
2695 rxq = dev->data->rx_queues[rx_queue_id];
2696 rxdp = &rxq->rx_ring[rxq->rx_tail];
2698 while ((desc < rxq->nb_rx_desc) &&
2699 (rxdp->qw1.lo.status &
2700 rte_cpu_to_le_32(TXGBE_RXD_STAT_DD))) {
2701 desc += TXGBE_RXQ_SCAN_INTERVAL;
2702 rxdp += TXGBE_RXQ_SCAN_INTERVAL;
2703 if (rxq->rx_tail + desc >= rxq->nb_rx_desc)
2704 rxdp = &(rxq->rx_ring[rxq->rx_tail +
2705 desc - rxq->nb_rx_desc]);
2712 txgbe_dev_rx_descriptor_status(void *rx_queue, uint16_t offset)
2714 struct txgbe_rx_queue *rxq = rx_queue;
2715 volatile uint32_t *status;
2716 uint32_t nb_hold, desc;
2718 if (unlikely(offset >= rxq->nb_rx_desc))
2721 nb_hold = rxq->nb_rx_hold;
2722 if (offset >= rxq->nb_rx_desc - nb_hold)
2723 return RTE_ETH_RX_DESC_UNAVAIL;
2725 desc = rxq->rx_tail + offset;
2726 if (desc >= rxq->nb_rx_desc)
2727 desc -= rxq->nb_rx_desc;
2729 status = &rxq->rx_ring[desc].qw1.lo.status;
2730 if (*status & rte_cpu_to_le_32(TXGBE_RXD_STAT_DD))
2731 return RTE_ETH_RX_DESC_DONE;
2733 return RTE_ETH_RX_DESC_AVAIL;
2737 txgbe_dev_tx_descriptor_status(void *tx_queue, uint16_t offset)
2739 struct txgbe_tx_queue *txq = tx_queue;
2740 volatile uint32_t *status;
2743 if (unlikely(offset >= txq->nb_tx_desc))
2746 desc = txq->tx_tail + offset;
2747 if (desc >= txq->nb_tx_desc) {
2748 desc -= txq->nb_tx_desc;
2749 if (desc >= txq->nb_tx_desc)
2750 desc -= txq->nb_tx_desc;
2753 status = &txq->tx_ring[desc].dw3;
2754 if (*status & rte_cpu_to_le_32(TXGBE_TXD_DD))
2755 return RTE_ETH_TX_DESC_DONE;
2757 return RTE_ETH_TX_DESC_FULL;
2761 txgbe_dev_clear_queues(struct rte_eth_dev *dev)
2764 struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev);
2766 PMD_INIT_FUNC_TRACE();
2768 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2769 struct txgbe_tx_queue *txq = dev->data->tx_queues[i];
2772 txq->ops->release_mbufs(txq);
2773 txq->ops->reset(txq);
2777 for (i = 0; i < dev->data->nb_rx_queues; i++) {
2778 struct txgbe_rx_queue *rxq = dev->data->rx_queues[i];
2781 txgbe_rx_queue_release_mbufs(rxq);
2782 txgbe_reset_rx_queue(adapter, rxq);
2788 txgbe_dev_free_queues(struct rte_eth_dev *dev)
2792 PMD_INIT_FUNC_TRACE();
2794 for (i = 0; i < dev->data->nb_rx_queues; i++) {
2795 txgbe_dev_rx_queue_release(dev->data->rx_queues[i]);
2796 dev->data->rx_queues[i] = NULL;
2798 dev->data->nb_rx_queues = 0;
2800 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2801 txgbe_dev_tx_queue_release(dev->data->tx_queues[i]);
2802 dev->data->tx_queues[i] = NULL;
2804 dev->data->nb_tx_queues = 0;
2808 * Receive Side Scaling (RSS)
2811 * The source and destination IP addresses of the IP header and the source
2812 * and destination ports of TCP/UDP headers, if any, of received packets are
2813 * hashed against a configurable random key to compute a 32-bit RSS hash result.
2814 * The seven (7) LSBs of the 32-bit hash result are used as an index into a
2815 * 128-entry redirection table (RETA). Each entry of the RETA provides a 3-bit
2816 * RSS output index which is used as the RX queue index where to store the
2818 * The following output is supplied in the RX write-back descriptor:
2819 * - 32-bit result of the Microsoft RSS hash function,
2820 * - 4-bit RSS type field.
2824 * Used as the default key.
2826 static uint8_t rss_intel_key[40] = {
2827 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
2828 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
2829 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
2830 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
2831 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
2835 txgbe_rss_disable(struct rte_eth_dev *dev)
2837 struct txgbe_hw *hw;
2839 hw = TXGBE_DEV_HW(dev);
2840 if (hw->mac.type == txgbe_mac_raptor_vf)
2841 wr32m(hw, TXGBE_VFPLCFG, TXGBE_VFPLCFG_RSSENA, 0);
2843 wr32m(hw, TXGBE_RACTL, TXGBE_RACTL_RSSENA, 0);
2847 txgbe_dev_rss_hash_update(struct rte_eth_dev *dev,
2848 struct rte_eth_rss_conf *rss_conf)
2850 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
2857 if (!txgbe_rss_update_sp(hw->mac.type)) {
2858 PMD_DRV_LOG(ERR, "RSS hash update is not supported on this "
2863 hash_key = rss_conf->rss_key;
2865 /* Fill in RSS hash key */
2866 for (i = 0; i < 10; i++) {
2867 rss_key = LS32(hash_key[(i * 4) + 0], 0, 0xFF);
2868 rss_key |= LS32(hash_key[(i * 4) + 1], 8, 0xFF);
2869 rss_key |= LS32(hash_key[(i * 4) + 2], 16, 0xFF);
2870 rss_key |= LS32(hash_key[(i * 4) + 3], 24, 0xFF);
2871 wr32a(hw, TXGBE_REG_RSSKEY, i, rss_key);
2875 /* Set configured hashing protocols */
2876 rss_hf = rss_conf->rss_hf & TXGBE_RSS_OFFLOAD_ALL;
2877 mrqc = rd32(hw, TXGBE_RACTL);
2878 mrqc &= ~TXGBE_RACTL_RSSMASK;
2879 if (rss_hf & ETH_RSS_IPV4)
2880 mrqc |= TXGBE_RACTL_RSSIPV4;
2881 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
2882 mrqc |= TXGBE_RACTL_RSSIPV4TCP;
2883 if (rss_hf & ETH_RSS_IPV6 ||
2884 rss_hf & ETH_RSS_IPV6_EX)
2885 mrqc |= TXGBE_RACTL_RSSIPV6;
2886 if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP ||
2887 rss_hf & ETH_RSS_IPV6_TCP_EX)
2888 mrqc |= TXGBE_RACTL_RSSIPV6TCP;
2889 if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP)
2890 mrqc |= TXGBE_RACTL_RSSIPV4UDP;
2891 if (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP ||
2892 rss_hf & ETH_RSS_IPV6_UDP_EX)
2893 mrqc |= TXGBE_RACTL_RSSIPV6UDP;
2896 mrqc |= TXGBE_RACTL_RSSENA;
2898 mrqc &= ~TXGBE_RACTL_RSSENA;
2900 wr32(hw, TXGBE_RACTL, mrqc);
2906 txgbe_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
2907 struct rte_eth_rss_conf *rss_conf)
2909 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
2916 hash_key = rss_conf->rss_key;
2918 /* Return RSS hash key */
2919 for (i = 0; i < 10; i++) {
2920 rss_key = rd32a(hw, TXGBE_REG_RSSKEY, i);
2921 hash_key[(i * 4) + 0] = RS32(rss_key, 0, 0xFF);
2922 hash_key[(i * 4) + 1] = RS32(rss_key, 8, 0xFF);
2923 hash_key[(i * 4) + 2] = RS32(rss_key, 16, 0xFF);
2924 hash_key[(i * 4) + 3] = RS32(rss_key, 24, 0xFF);
2929 mrqc = rd32(hw, TXGBE_RACTL);
2930 if (mrqc & TXGBE_RACTL_RSSIPV4)
2931 rss_hf |= ETH_RSS_IPV4;
2932 if (mrqc & TXGBE_RACTL_RSSIPV4TCP)
2933 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP;
2934 if (mrqc & TXGBE_RACTL_RSSIPV6)
2935 rss_hf |= ETH_RSS_IPV6 |
2937 if (mrqc & TXGBE_RACTL_RSSIPV6TCP)
2938 rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP |
2939 ETH_RSS_IPV6_TCP_EX;
2940 if (mrqc & TXGBE_RACTL_RSSIPV4UDP)
2941 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP;
2942 if (mrqc & TXGBE_RACTL_RSSIPV6UDP)
2943 rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP |
2944 ETH_RSS_IPV6_UDP_EX;
2945 if (!(mrqc & TXGBE_RACTL_RSSENA))
2948 rss_hf &= TXGBE_RSS_OFFLOAD_ALL;
2950 rss_conf->rss_hf = rss_hf;
2955 txgbe_rss_configure(struct rte_eth_dev *dev)
2957 struct rte_eth_rss_conf rss_conf;
2958 struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev);
2959 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
2964 PMD_INIT_FUNC_TRACE();
2967 * Fill in redirection table
2968 * The byte-swap is needed because NIC registers are in
2969 * little-endian order.
2971 if (adapter->rss_reta_updated == 0) {
2973 for (i = 0, j = 0; i < ETH_RSS_RETA_SIZE_128; i++, j++) {
2974 if (j == dev->data->nb_rx_queues)
2976 reta = (reta >> 8) | LS32(j, 24, 0xFF);
2978 wr32a(hw, TXGBE_REG_RSSTBL, i >> 2, reta);
2982 * Configure the RSS key and the RSS protocols used to compute
2983 * the RSS hash of input packets.
2985 rss_conf = dev->data->dev_conf.rx_adv_conf.rss_conf;
2986 if (rss_conf.rss_key == NULL)
2987 rss_conf.rss_key = rss_intel_key; /* Default hash key */
2988 txgbe_dev_rss_hash_update(dev, &rss_conf);
2991 #define NUM_VFTA_REGISTERS 128
2992 #define NIC_RX_BUFFER_SIZE 0x200
2995 txgbe_vmdq_dcb_configure(struct rte_eth_dev *dev)
2997 struct rte_eth_vmdq_dcb_conf *cfg;
2998 struct txgbe_hw *hw;
2999 enum rte_eth_nb_pools num_pools;
3000 uint32_t mrqc, vt_ctl, queue_mapping, vlanctrl;
3002 uint8_t nb_tcs; /* number of traffic classes */
3005 PMD_INIT_FUNC_TRACE();
3006 hw = TXGBE_DEV_HW(dev);
3007 cfg = &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf;
3008 num_pools = cfg->nb_queue_pools;
3009 /* Check we have a valid number of pools */
3010 if (num_pools != ETH_16_POOLS && num_pools != ETH_32_POOLS) {
3011 txgbe_rss_disable(dev);
3014 /* 16 pools -> 8 traffic classes, 32 pools -> 4 traffic classes */
3015 nb_tcs = (uint8_t)(ETH_VMDQ_DCB_NUM_QUEUES / (int)num_pools);
3018 * split rx buffer up into sections, each for 1 traffic class
3020 pbsize = (uint16_t)(NIC_RX_BUFFER_SIZE / nb_tcs);
3021 for (i = 0; i < nb_tcs; i++) {
3022 uint32_t rxpbsize = rd32(hw, TXGBE_PBRXSIZE(i));
3024 rxpbsize &= (~(0x3FF << 10));
3025 /* clear 10 bits. */
3026 rxpbsize |= (pbsize << 10); /* set value */
3027 wr32(hw, TXGBE_PBRXSIZE(i), rxpbsize);
3029 /* zero alloc all unused TCs */
3030 for (i = nb_tcs; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3031 uint32_t rxpbsize = rd32(hw, TXGBE_PBRXSIZE(i));
3033 rxpbsize &= (~(0x3FF << 10));
3034 /* clear 10 bits. */
3035 wr32(hw, TXGBE_PBRXSIZE(i), rxpbsize);
3038 if (num_pools == ETH_16_POOLS) {
3039 mrqc = TXGBE_PORTCTL_NUMTC_8;
3040 mrqc |= TXGBE_PORTCTL_NUMVT_16;
3042 mrqc = TXGBE_PORTCTL_NUMTC_4;
3043 mrqc |= TXGBE_PORTCTL_NUMVT_32;
3045 wr32m(hw, TXGBE_PORTCTL,
3046 TXGBE_PORTCTL_NUMTC_MASK | TXGBE_PORTCTL_NUMVT_MASK, mrqc);
3048 vt_ctl = TXGBE_POOLCTL_RPLEN;
3049 if (cfg->enable_default_pool)
3050 vt_ctl |= TXGBE_POOLCTL_DEFPL(cfg->default_pool);
3052 vt_ctl |= TXGBE_POOLCTL_DEFDSA;
3054 wr32(hw, TXGBE_POOLCTL, vt_ctl);
3057 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++)
3059 * mapping is done with 3 bits per priority,
3060 * so shift by i*3 each time
3062 queue_mapping |= ((cfg->dcb_tc[i] & 0x07) << (i * 3));
3064 wr32(hw, TXGBE_RPUP2TC, queue_mapping);
3066 wr32(hw, TXGBE_ARBRXCTL, TXGBE_ARBRXCTL_RRM);
3068 /* enable vlan filtering and allow all vlan tags through */
3069 vlanctrl = rd32(hw, TXGBE_VLANCTL);
3070 vlanctrl |= TXGBE_VLANCTL_VFE; /* enable vlan filters */
3071 wr32(hw, TXGBE_VLANCTL, vlanctrl);
3073 /* enable all vlan filters */
3074 for (i = 0; i < NUM_VFTA_REGISTERS; i++)
3075 wr32(hw, TXGBE_VLANTBL(i), 0xFFFFFFFF);
3077 wr32(hw, TXGBE_POOLRXENA(0),
3078 num_pools == ETH_16_POOLS ? 0xFFFF : 0xFFFFFFFF);
3080 wr32(hw, TXGBE_ETHADDRIDX, 0);
3081 wr32(hw, TXGBE_ETHADDRASSL, 0xFFFFFFFF);
3082 wr32(hw, TXGBE_ETHADDRASSH, 0xFFFFFFFF);
3084 /* set up filters for vlan tags as configured */
3085 for (i = 0; i < cfg->nb_pool_maps; i++) {
3086 /* set vlan id in VF register and set the valid bit */
3087 wr32(hw, TXGBE_PSRVLANIDX, i);
3088 wr32(hw, TXGBE_PSRVLAN, (TXGBE_PSRVLAN_EA |
3089 (cfg->pool_map[i].vlan_id & 0xFFF)));
3091 wr32(hw, TXGBE_PSRVLANPLM(0), cfg->pool_map[i].pools);
3096 * txgbe_dcb_config_tx_hw_config - Configure general DCB TX parameters
3097 * @dev: pointer to eth_dev structure
3098 * @dcb_config: pointer to txgbe_dcb_config structure
3101 txgbe_dcb_tx_hw_config(struct rte_eth_dev *dev,
3102 struct txgbe_dcb_config *dcb_config)
3105 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
3107 PMD_INIT_FUNC_TRACE();
3109 /* Disable the Tx desc arbiter */
3110 reg = rd32(hw, TXGBE_ARBTXCTL);
3111 reg |= TXGBE_ARBTXCTL_DIA;
3112 wr32(hw, TXGBE_ARBTXCTL, reg);
3114 /* Enable DCB for Tx with 8 TCs */
3115 reg = rd32(hw, TXGBE_PORTCTL);
3116 reg &= TXGBE_PORTCTL_NUMTC_MASK;
3117 reg |= TXGBE_PORTCTL_DCB;
3118 if (dcb_config->num_tcs.pg_tcs == 8)
3119 reg |= TXGBE_PORTCTL_NUMTC_8;
3121 reg |= TXGBE_PORTCTL_NUMTC_4;
3123 wr32(hw, TXGBE_PORTCTL, reg);
3125 /* Enable the Tx desc arbiter */
3126 reg = rd32(hw, TXGBE_ARBTXCTL);
3127 reg &= ~TXGBE_ARBTXCTL_DIA;
3128 wr32(hw, TXGBE_ARBTXCTL, reg);
3132 * txgbe_vmdq_dcb_hw_tx_config - Configure general VMDQ+DCB TX parameters
3133 * @dev: pointer to rte_eth_dev structure
3134 * @dcb_config: pointer to txgbe_dcb_config structure
3137 txgbe_vmdq_dcb_hw_tx_config(struct rte_eth_dev *dev,
3138 struct txgbe_dcb_config *dcb_config)
3140 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3141 &dev->data->dev_conf.tx_adv_conf.vmdq_dcb_tx_conf;
3142 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
3144 PMD_INIT_FUNC_TRACE();
3145 /*PF VF Transmit Enable*/
3146 wr32(hw, TXGBE_POOLTXENA(0),
3147 vmdq_tx_conf->nb_queue_pools ==
3148 ETH_16_POOLS ? 0xFFFF : 0xFFFFFFFF);
3150 /*Configure general DCB TX parameters*/
3151 txgbe_dcb_tx_hw_config(dev, dcb_config);
3155 txgbe_vmdq_dcb_rx_config(struct rte_eth_dev *dev,
3156 struct txgbe_dcb_config *dcb_config)
3158 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3159 &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf;
3160 struct txgbe_dcb_tc_config *tc;
3163 /* convert rte_eth_conf.rx_adv_conf to struct txgbe_dcb_config */
3164 if (vmdq_rx_conf->nb_queue_pools == ETH_16_POOLS) {
3165 dcb_config->num_tcs.pg_tcs = ETH_8_TCS;
3166 dcb_config->num_tcs.pfc_tcs = ETH_8_TCS;
3168 dcb_config->num_tcs.pg_tcs = ETH_4_TCS;
3169 dcb_config->num_tcs.pfc_tcs = ETH_4_TCS;
3172 /* Initialize User Priority to Traffic Class mapping */
3173 for (j = 0; j < TXGBE_DCB_TC_MAX; j++) {
3174 tc = &dcb_config->tc_config[j];
3175 tc->path[TXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0;
3178 /* User Priority to Traffic Class mapping */
3179 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3180 j = vmdq_rx_conf->dcb_tc[i];
3181 tc = &dcb_config->tc_config[j];
3182 tc->path[TXGBE_DCB_RX_CONFIG].up_to_tc_bitmap |=
3188 txgbe_dcb_vt_tx_config(struct rte_eth_dev *dev,
3189 struct txgbe_dcb_config *dcb_config)
3191 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3192 &dev->data->dev_conf.tx_adv_conf.vmdq_dcb_tx_conf;
3193 struct txgbe_dcb_tc_config *tc;
3196 /* convert rte_eth_conf.rx_adv_conf to struct txgbe_dcb_config */
3197 if (vmdq_tx_conf->nb_queue_pools == ETH_16_POOLS) {
3198 dcb_config->num_tcs.pg_tcs = ETH_8_TCS;
3199 dcb_config->num_tcs.pfc_tcs = ETH_8_TCS;
3201 dcb_config->num_tcs.pg_tcs = ETH_4_TCS;
3202 dcb_config->num_tcs.pfc_tcs = ETH_4_TCS;
3205 /* Initialize User Priority to Traffic Class mapping */
3206 for (j = 0; j < TXGBE_DCB_TC_MAX; j++) {
3207 tc = &dcb_config->tc_config[j];
3208 tc->path[TXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0;
3211 /* User Priority to Traffic Class mapping */
3212 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3213 j = vmdq_tx_conf->dcb_tc[i];
3214 tc = &dcb_config->tc_config[j];
3215 tc->path[TXGBE_DCB_TX_CONFIG].up_to_tc_bitmap |=
3221 txgbe_dcb_rx_config(struct rte_eth_dev *dev,
3222 struct txgbe_dcb_config *dcb_config)
3224 struct rte_eth_dcb_rx_conf *rx_conf =
3225 &dev->data->dev_conf.rx_adv_conf.dcb_rx_conf;
3226 struct txgbe_dcb_tc_config *tc;
3229 dcb_config->num_tcs.pg_tcs = (uint8_t)rx_conf->nb_tcs;
3230 dcb_config->num_tcs.pfc_tcs = (uint8_t)rx_conf->nb_tcs;
3232 /* Initialize User Priority to Traffic Class mapping */
3233 for (j = 0; j < TXGBE_DCB_TC_MAX; j++) {
3234 tc = &dcb_config->tc_config[j];
3235 tc->path[TXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0;
3238 /* User Priority to Traffic Class mapping */
3239 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3240 j = rx_conf->dcb_tc[i];
3241 tc = &dcb_config->tc_config[j];
3242 tc->path[TXGBE_DCB_RX_CONFIG].up_to_tc_bitmap |=
3248 txgbe_dcb_tx_config(struct rte_eth_dev *dev,
3249 struct txgbe_dcb_config *dcb_config)
3251 struct rte_eth_dcb_tx_conf *tx_conf =
3252 &dev->data->dev_conf.tx_adv_conf.dcb_tx_conf;
3253 struct txgbe_dcb_tc_config *tc;
3256 dcb_config->num_tcs.pg_tcs = (uint8_t)tx_conf->nb_tcs;
3257 dcb_config->num_tcs.pfc_tcs = (uint8_t)tx_conf->nb_tcs;
3259 /* Initialize User Priority to Traffic Class mapping */
3260 for (j = 0; j < TXGBE_DCB_TC_MAX; j++) {
3261 tc = &dcb_config->tc_config[j];
3262 tc->path[TXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0;
3265 /* User Priority to Traffic Class mapping */
3266 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3267 j = tx_conf->dcb_tc[i];
3268 tc = &dcb_config->tc_config[j];
3269 tc->path[TXGBE_DCB_TX_CONFIG].up_to_tc_bitmap |=
3275 * txgbe_dcb_rx_hw_config - Configure general DCB RX HW parameters
3276 * @dev: pointer to eth_dev structure
3277 * @dcb_config: pointer to txgbe_dcb_config structure
3280 txgbe_dcb_rx_hw_config(struct rte_eth_dev *dev,
3281 struct txgbe_dcb_config *dcb_config)
3287 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
3289 PMD_INIT_FUNC_TRACE();
3291 * Disable the arbiter before changing parameters
3292 * (always enable recycle mode; WSP)
3294 reg = TXGBE_ARBRXCTL_RRM | TXGBE_ARBRXCTL_WSP | TXGBE_ARBRXCTL_DIA;
3295 wr32(hw, TXGBE_ARBRXCTL, reg);
3297 reg = rd32(hw, TXGBE_PORTCTL);
3298 reg &= ~(TXGBE_PORTCTL_NUMTC_MASK | TXGBE_PORTCTL_NUMVT_MASK);
3299 if (dcb_config->num_tcs.pg_tcs == 4) {
3300 reg |= TXGBE_PORTCTL_NUMTC_4;
3301 if (dcb_config->vt_mode)
3302 reg |= TXGBE_PORTCTL_NUMVT_32;
3304 wr32(hw, TXGBE_POOLCTL, 0);
3307 if (dcb_config->num_tcs.pg_tcs == 8) {
3308 reg |= TXGBE_PORTCTL_NUMTC_8;
3309 if (dcb_config->vt_mode)
3310 reg |= TXGBE_PORTCTL_NUMVT_16;
3312 wr32(hw, TXGBE_POOLCTL, 0);
3315 wr32(hw, TXGBE_PORTCTL, reg);
3317 if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
3318 /* Disable drop for all queues in VMDQ mode*/
3319 for (q = 0; q < TXGBE_MAX_RX_QUEUE_NUM; q++) {
3320 u32 val = 1 << (q % 32);
3321 wr32m(hw, TXGBE_QPRXDROP(q / 32), val, val);
3324 /* Enable drop for all queues in SRIOV mode */
3325 for (q = 0; q < TXGBE_MAX_RX_QUEUE_NUM; q++) {
3326 u32 val = 1 << (q % 32);
3327 wr32m(hw, TXGBE_QPRXDROP(q / 32), val, val);
3331 /* VLNCTL: enable vlan filtering and allow all vlan tags through */
3332 vlanctrl = rd32(hw, TXGBE_VLANCTL);
3333 vlanctrl |= TXGBE_VLANCTL_VFE; /* enable vlan filters */
3334 wr32(hw, TXGBE_VLANCTL, vlanctrl);
3336 /* VLANTBL - enable all vlan filters */
3337 for (i = 0; i < NUM_VFTA_REGISTERS; i++)
3338 wr32(hw, TXGBE_VLANTBL(i), 0xFFFFFFFF);
3341 * Configure Rx packet plane (recycle mode; WSP) and
3344 reg = TXGBE_ARBRXCTL_RRM | TXGBE_ARBRXCTL_WSP;
3345 wr32(hw, TXGBE_ARBRXCTL, reg);
3349 txgbe_dcb_hw_arbite_rx_config(struct txgbe_hw *hw, uint16_t *refill,
3350 uint16_t *max, uint8_t *bwg_id, uint8_t *tsa, uint8_t *map)
3352 txgbe_dcb_config_rx_arbiter_raptor(hw, refill, max, bwg_id,
3357 txgbe_dcb_hw_arbite_tx_config(struct txgbe_hw *hw, uint16_t *refill,
3358 uint16_t *max, uint8_t *bwg_id, uint8_t *tsa, uint8_t *map)
3360 switch (hw->mac.type) {
3361 case txgbe_mac_raptor:
3362 txgbe_dcb_config_tx_desc_arbiter_raptor(hw, refill,
3364 txgbe_dcb_config_tx_data_arbiter_raptor(hw, refill,
3365 max, bwg_id, tsa, map);
3372 #define DCB_RX_CONFIG 1
3373 #define DCB_TX_CONFIG 1
3374 #define DCB_TX_PB 1024
3376 * txgbe_dcb_hw_configure - Enable DCB and configure
3377 * general DCB in VT mode and non-VT mode parameters
3378 * @dev: pointer to rte_eth_dev structure
3379 * @dcb_config: pointer to txgbe_dcb_config structure
3382 txgbe_dcb_hw_configure(struct rte_eth_dev *dev,
3383 struct txgbe_dcb_config *dcb_config)
3386 uint8_t i, pfc_en, nb_tcs;
3387 uint16_t pbsize, rx_buffer_size;
3388 uint8_t config_dcb_rx = 0;
3389 uint8_t config_dcb_tx = 0;
3390 uint8_t tsa[TXGBE_DCB_TC_MAX] = {0};
3391 uint8_t bwgid[TXGBE_DCB_TC_MAX] = {0};
3392 uint16_t refill[TXGBE_DCB_TC_MAX] = {0};
3393 uint16_t max[TXGBE_DCB_TC_MAX] = {0};
3394 uint8_t map[TXGBE_DCB_TC_MAX] = {0};
3395 struct txgbe_dcb_tc_config *tc;
3396 uint32_t max_frame = dev->data->mtu +
3397 RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN;
3398 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
3399 struct txgbe_bw_conf *bw_conf = TXGBE_DEV_BW_CONF(dev);
3401 switch (dev->data->dev_conf.rxmode.mq_mode) {
3402 case ETH_MQ_RX_VMDQ_DCB:
3403 dcb_config->vt_mode = true;
3404 config_dcb_rx = DCB_RX_CONFIG;
3406 * get dcb and VT rx configuration parameters
3409 txgbe_vmdq_dcb_rx_config(dev, dcb_config);
3410 /*Configure general VMDQ and DCB RX parameters*/
3411 txgbe_vmdq_dcb_configure(dev);
3414 case ETH_MQ_RX_DCB_RSS:
3415 dcb_config->vt_mode = false;
3416 config_dcb_rx = DCB_RX_CONFIG;
3417 /* Get dcb TX configuration parameters from rte_eth_conf */
3418 txgbe_dcb_rx_config(dev, dcb_config);
3419 /*Configure general DCB RX parameters*/
3420 txgbe_dcb_rx_hw_config(dev, dcb_config);
3423 PMD_INIT_LOG(ERR, "Incorrect DCB RX mode configuration");
3426 switch (dev->data->dev_conf.txmode.mq_mode) {
3427 case ETH_MQ_TX_VMDQ_DCB:
3428 dcb_config->vt_mode = true;
3429 config_dcb_tx = DCB_TX_CONFIG;
3430 /* get DCB and VT TX configuration parameters
3433 txgbe_dcb_vt_tx_config(dev, dcb_config);
3434 /* Configure general VMDQ and DCB TX parameters */
3435 txgbe_vmdq_dcb_hw_tx_config(dev, dcb_config);
3439 dcb_config->vt_mode = false;
3440 config_dcb_tx = DCB_TX_CONFIG;
3441 /* get DCB TX configuration parameters from rte_eth_conf */
3442 txgbe_dcb_tx_config(dev, dcb_config);
3443 /* Configure general DCB TX parameters */
3444 txgbe_dcb_tx_hw_config(dev, dcb_config);
3447 PMD_INIT_LOG(ERR, "Incorrect DCB TX mode configuration");
3451 nb_tcs = dcb_config->num_tcs.pfc_tcs;
3453 txgbe_dcb_unpack_map_cee(dcb_config, TXGBE_DCB_RX_CONFIG, map);
3454 if (nb_tcs == ETH_4_TCS) {
3455 /* Avoid un-configured priority mapping to TC0 */
3457 uint8_t mask = 0xFF;
3459 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES - 4; i++)
3460 mask = (uint8_t)(mask & (~(1 << map[i])));
3461 for (i = 0; mask && (i < TXGBE_DCB_TC_MAX); i++) {
3462 if ((mask & 0x1) && j < ETH_DCB_NUM_USER_PRIORITIES)
3466 /* Re-configure 4 TCs BW */
3467 for (i = 0; i < nb_tcs; i++) {
3468 tc = &dcb_config->tc_config[i];
3469 if (bw_conf->tc_num != nb_tcs)
3470 tc->path[TXGBE_DCB_TX_CONFIG].bwg_percent =
3471 (uint8_t)(100 / nb_tcs);
3472 tc->path[TXGBE_DCB_RX_CONFIG].bwg_percent =
3473 (uint8_t)(100 / nb_tcs);
3475 for (; i < TXGBE_DCB_TC_MAX; i++) {
3476 tc = &dcb_config->tc_config[i];
3477 tc->path[TXGBE_DCB_TX_CONFIG].bwg_percent = 0;
3478 tc->path[TXGBE_DCB_RX_CONFIG].bwg_percent = 0;
3481 /* Re-configure 8 TCs BW */
3482 for (i = 0; i < nb_tcs; i++) {
3483 tc = &dcb_config->tc_config[i];
3484 if (bw_conf->tc_num != nb_tcs)
3485 tc->path[TXGBE_DCB_TX_CONFIG].bwg_percent =
3486 (uint8_t)(100 / nb_tcs + (i & 1));
3487 tc->path[TXGBE_DCB_RX_CONFIG].bwg_percent =
3488 (uint8_t)(100 / nb_tcs + (i & 1));
3492 rx_buffer_size = NIC_RX_BUFFER_SIZE;
3494 if (config_dcb_rx) {
3495 /* Set RX buffer size */
3496 pbsize = (uint16_t)(rx_buffer_size / nb_tcs);
3497 uint32_t rxpbsize = pbsize << 10;
3499 for (i = 0; i < nb_tcs; i++)
3500 wr32(hw, TXGBE_PBRXSIZE(i), rxpbsize);
3502 /* zero alloc all unused TCs */
3503 for (; i < ETH_DCB_NUM_USER_PRIORITIES; i++)
3504 wr32(hw, TXGBE_PBRXSIZE(i), 0);
3506 if (config_dcb_tx) {
3507 /* Only support an equally distributed
3508 * Tx packet buffer strategy.
3510 uint32_t txpktsize = TXGBE_PBTXSIZE_MAX / nb_tcs;
3511 uint32_t txpbthresh = (txpktsize / DCB_TX_PB) -
3512 TXGBE_TXPKT_SIZE_MAX;
3514 for (i = 0; i < nb_tcs; i++) {
3515 wr32(hw, TXGBE_PBTXSIZE(i), txpktsize);
3516 wr32(hw, TXGBE_PBTXDMATH(i), txpbthresh);
3518 /* Clear unused TCs, if any, to zero buffer size*/
3519 for (; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3520 wr32(hw, TXGBE_PBTXSIZE(i), 0);
3521 wr32(hw, TXGBE_PBTXDMATH(i), 0);
3525 /*Calculates traffic class credits*/
3526 txgbe_dcb_calculate_tc_credits_cee(hw, dcb_config, max_frame,
3527 TXGBE_DCB_TX_CONFIG);
3528 txgbe_dcb_calculate_tc_credits_cee(hw, dcb_config, max_frame,
3529 TXGBE_DCB_RX_CONFIG);
3531 if (config_dcb_rx) {
3532 /* Unpack CEE standard containers */
3533 txgbe_dcb_unpack_refill_cee(dcb_config,
3534 TXGBE_DCB_RX_CONFIG, refill);
3535 txgbe_dcb_unpack_max_cee(dcb_config, max);
3536 txgbe_dcb_unpack_bwgid_cee(dcb_config,
3537 TXGBE_DCB_RX_CONFIG, bwgid);
3538 txgbe_dcb_unpack_tsa_cee(dcb_config,
3539 TXGBE_DCB_RX_CONFIG, tsa);
3540 /* Configure PG(ETS) RX */
3541 txgbe_dcb_hw_arbite_rx_config(hw, refill, max, bwgid, tsa, map);
3544 if (config_dcb_tx) {
3545 /* Unpack CEE standard containers */
3546 txgbe_dcb_unpack_refill_cee(dcb_config,
3547 TXGBE_DCB_TX_CONFIG, refill);
3548 txgbe_dcb_unpack_max_cee(dcb_config, max);
3549 txgbe_dcb_unpack_bwgid_cee(dcb_config,
3550 TXGBE_DCB_TX_CONFIG, bwgid);
3551 txgbe_dcb_unpack_tsa_cee(dcb_config,
3552 TXGBE_DCB_TX_CONFIG, tsa);
3553 /* Configure PG(ETS) TX */
3554 txgbe_dcb_hw_arbite_tx_config(hw, refill, max, bwgid, tsa, map);
3557 /* Configure queue statistics registers */
3558 txgbe_dcb_config_tc_stats_raptor(hw, dcb_config);
3560 /* Check if the PFC is supported */
3561 if (dev->data->dev_conf.dcb_capability_en & ETH_DCB_PFC_SUPPORT) {
3562 pbsize = (uint16_t)(rx_buffer_size / nb_tcs);
3563 for (i = 0; i < nb_tcs; i++) {
3564 /* If the TC count is 8,
3565 * and the default high_water is 48,
3566 * the low_water is 16 as default.
3568 hw->fc.high_water[i] = (pbsize * 3) / 4;
3569 hw->fc.low_water[i] = pbsize / 4;
3570 /* Enable pfc for this TC */
3571 tc = &dcb_config->tc_config[i];
3572 tc->pfc = txgbe_dcb_pfc_enabled;
3574 txgbe_dcb_unpack_pfc_cee(dcb_config, map, &pfc_en);
3575 if (dcb_config->num_tcs.pfc_tcs == ETH_4_TCS)
3577 ret = txgbe_dcb_config_pfc(hw, pfc_en, map);
3583 void txgbe_configure_pb(struct rte_eth_dev *dev)
3585 struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
3586 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
3589 int tc = dev_conf->rx_adv_conf.dcb_rx_conf.nb_tcs;
3591 /* Reserve 256KB(/512KB) rx buffer for fdir */
3594 hw->mac.setup_pba(hw, tc, hdrm, PBA_STRATEGY_EQUAL);
3597 void txgbe_configure_port(struct rte_eth_dev *dev)
3599 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
3601 uint16_t tpids[8] = {RTE_ETHER_TYPE_VLAN, RTE_ETHER_TYPE_QINQ,
3606 PMD_INIT_FUNC_TRACE();
3608 /* default outer vlan tpid */
3609 wr32(hw, TXGBE_EXTAG,
3610 TXGBE_EXTAG_ETAG(RTE_ETHER_TYPE_ETAG) |
3611 TXGBE_EXTAG_VLAN(RTE_ETHER_TYPE_QINQ));
3613 /* default inner vlan tpid */
3614 wr32m(hw, TXGBE_VLANCTL,
3615 TXGBE_VLANCTL_TPID_MASK,
3616 TXGBE_VLANCTL_TPID(RTE_ETHER_TYPE_VLAN));
3617 wr32m(hw, TXGBE_DMATXCTRL,
3618 TXGBE_DMATXCTRL_TPID_MASK,
3619 TXGBE_DMATXCTRL_TPID(RTE_ETHER_TYPE_VLAN));
3621 /* default vlan tpid filters */
3622 for (i = 0; i < 8; i++) {
3623 wr32m(hw, TXGBE_TAGTPID(i / 2),
3624 (i % 2 ? TXGBE_TAGTPID_MSB_MASK
3625 : TXGBE_TAGTPID_LSB_MASK),
3626 (i % 2 ? TXGBE_TAGTPID_MSB(tpids[i])
3627 : TXGBE_TAGTPID_LSB(tpids[i])));
3630 /* default vxlan port */
3631 wr32(hw, TXGBE_VXLANPORT, 4789);
3635 * txgbe_configure_dcb - Configure DCB Hardware
3636 * @dev: pointer to rte_eth_dev
3638 void txgbe_configure_dcb(struct rte_eth_dev *dev)
3640 struct txgbe_dcb_config *dcb_cfg = TXGBE_DEV_DCB_CONFIG(dev);
3641 struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
3643 PMD_INIT_FUNC_TRACE();
3645 /* check support mq_mode for DCB */
3646 if (dev_conf->rxmode.mq_mode != ETH_MQ_RX_VMDQ_DCB &&
3647 dev_conf->rxmode.mq_mode != ETH_MQ_RX_DCB &&
3648 dev_conf->rxmode.mq_mode != ETH_MQ_RX_DCB_RSS)
3651 if (dev->data->nb_rx_queues > ETH_DCB_NUM_QUEUES)
3654 /** Configure DCB hardware **/
3655 txgbe_dcb_hw_configure(dev, dcb_cfg);
3659 * VMDq only support for 10 GbE NIC.
3662 txgbe_vmdq_rx_hw_configure(struct rte_eth_dev *dev)
3664 struct rte_eth_vmdq_rx_conf *cfg;
3665 struct txgbe_hw *hw;
3666 enum rte_eth_nb_pools num_pools;
3667 uint32_t mrqc, vt_ctl, vlanctrl;
3671 PMD_INIT_FUNC_TRACE();
3672 hw = TXGBE_DEV_HW(dev);
3673 cfg = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
3674 num_pools = cfg->nb_queue_pools;
3676 txgbe_rss_disable(dev);
3679 mrqc = TXGBE_PORTCTL_NUMVT_64;
3680 wr32m(hw, TXGBE_PORTCTL, TXGBE_PORTCTL_NUMVT_MASK, mrqc);
3682 /* turn on virtualisation and set the default pool */
3683 vt_ctl = TXGBE_POOLCTL_RPLEN;
3684 if (cfg->enable_default_pool)
3685 vt_ctl |= TXGBE_POOLCTL_DEFPL(cfg->default_pool);
3687 vt_ctl |= TXGBE_POOLCTL_DEFDSA;
3689 wr32(hw, TXGBE_POOLCTL, vt_ctl);
3691 for (i = 0; i < (int)num_pools; i++) {
3692 vmolr = txgbe_convert_vm_rx_mask_to_val(cfg->rx_mode, vmolr);
3693 wr32(hw, TXGBE_POOLETHCTL(i), vmolr);
3696 /* enable vlan filtering and allow all vlan tags through */
3697 vlanctrl = rd32(hw, TXGBE_VLANCTL);
3698 vlanctrl |= TXGBE_VLANCTL_VFE; /* enable vlan filters */
3699 wr32(hw, TXGBE_VLANCTL, vlanctrl);
3701 /* enable all vlan filters */
3702 for (i = 0; i < NUM_VFTA_REGISTERS; i++)
3703 wr32(hw, TXGBE_VLANTBL(i), UINT32_MAX);
3705 /* pool enabling for receive - 64 */
3706 wr32(hw, TXGBE_POOLRXENA(0), UINT32_MAX);
3707 if (num_pools == ETH_64_POOLS)
3708 wr32(hw, TXGBE_POOLRXENA(1), UINT32_MAX);
3711 * allow pools to read specific mac addresses
3712 * In this case, all pools should be able to read from mac addr 0
3714 wr32(hw, TXGBE_ETHADDRIDX, 0);
3715 wr32(hw, TXGBE_ETHADDRASSL, 0xFFFFFFFF);
3716 wr32(hw, TXGBE_ETHADDRASSH, 0xFFFFFFFF);
3718 /* set up filters for vlan tags as configured */
3719 for (i = 0; i < cfg->nb_pool_maps; i++) {
3720 /* set vlan id in VF register and set the valid bit */
3721 wr32(hw, TXGBE_PSRVLANIDX, i);
3722 wr32(hw, TXGBE_PSRVLAN, (TXGBE_PSRVLAN_EA |
3723 TXGBE_PSRVLAN_VID(cfg->pool_map[i].vlan_id)));
3725 * Put the allowed pools in VFB reg. As we only have 16 or 64
3726 * pools, we only need to use the first half of the register
3729 if (((cfg->pool_map[i].pools >> 32) & UINT32_MAX) == 0)
3730 wr32(hw, TXGBE_PSRVLANPLM(0),
3731 (cfg->pool_map[i].pools & UINT32_MAX));
3733 wr32(hw, TXGBE_PSRVLANPLM(1),
3734 ((cfg->pool_map[i].pools >> 32) & UINT32_MAX));
3737 /* Tx General Switch Control Enables VMDQ loopback */
3738 if (cfg->enable_loop_back) {
3739 wr32(hw, TXGBE_PSRCTL, TXGBE_PSRCTL_LBENA);
3740 for (i = 0; i < 64; i++)
3741 wr32m(hw, TXGBE_POOLETHCTL(i),
3742 TXGBE_POOLETHCTL_LLB, TXGBE_POOLETHCTL_LLB);
3749 * txgbe_vmdq_tx_hw_configure - Configure general VMDq TX parameters
3750 * @hw: pointer to hardware structure
3753 txgbe_vmdq_tx_hw_configure(struct txgbe_hw *hw)
3758 PMD_INIT_FUNC_TRACE();
3759 /*PF VF Transmit Enable*/
3760 wr32(hw, TXGBE_POOLTXENA(0), UINT32_MAX);
3761 wr32(hw, TXGBE_POOLTXENA(1), UINT32_MAX);
3763 /* Disable the Tx desc arbiter */
3764 reg = rd32(hw, TXGBE_ARBTXCTL);
3765 reg |= TXGBE_ARBTXCTL_DIA;
3766 wr32(hw, TXGBE_ARBTXCTL, reg);
3768 wr32m(hw, TXGBE_PORTCTL, TXGBE_PORTCTL_NUMVT_MASK,
3769 TXGBE_PORTCTL_NUMVT_64);
3771 /* Disable drop for all queues */
3772 for (q = 0; q < 128; q++) {
3773 u32 val = 1 << (q % 32);
3774 wr32m(hw, TXGBE_QPRXDROP(q / 32), val, val);
3777 /* Enable the Tx desc arbiter */
3778 reg = rd32(hw, TXGBE_ARBTXCTL);
3779 reg &= ~TXGBE_ARBTXCTL_DIA;
3780 wr32(hw, TXGBE_ARBTXCTL, reg);
3785 static int __rte_cold
3786 txgbe_alloc_rx_queue_mbufs(struct txgbe_rx_queue *rxq)
3788 struct txgbe_rx_entry *rxe = rxq->sw_ring;
3792 /* Initialize software ring entries */
3793 for (i = 0; i < rxq->nb_rx_desc; i++) {
3794 volatile struct txgbe_rx_desc *rxd;
3795 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mb_pool);
3798 PMD_INIT_LOG(ERR, "RX mbuf alloc failed queue_id=%u",
3799 (unsigned int)rxq->queue_id);
3803 mbuf->data_off = RTE_PKTMBUF_HEADROOM;
3804 mbuf->port = rxq->port_id;
3807 rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf));
3808 rxd = &rxq->rx_ring[i];
3809 TXGBE_RXD_HDRADDR(rxd, 0);
3810 TXGBE_RXD_PKTADDR(rxd, dma_addr);
3818 txgbe_config_vf_rss(struct rte_eth_dev *dev)
3820 struct txgbe_hw *hw;
3823 txgbe_rss_configure(dev);
3825 hw = TXGBE_DEV_HW(dev);
3828 mrqc = rd32(hw, TXGBE_PORTCTL);
3829 mrqc &= ~(TXGBE_PORTCTL_NUMTC_MASK | TXGBE_PORTCTL_NUMVT_MASK);
3830 switch (RTE_ETH_DEV_SRIOV(dev).active) {
3832 mrqc |= TXGBE_PORTCTL_NUMVT_64;
3836 mrqc |= TXGBE_PORTCTL_NUMVT_32;
3840 PMD_INIT_LOG(ERR, "Invalid pool number in IOV mode with VMDQ RSS");
3844 wr32(hw, TXGBE_PORTCTL, mrqc);
3850 txgbe_config_vf_default(struct rte_eth_dev *dev)
3852 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
3855 mrqc = rd32(hw, TXGBE_PORTCTL);
3856 mrqc &= ~(TXGBE_PORTCTL_NUMTC_MASK | TXGBE_PORTCTL_NUMVT_MASK);
3857 switch (RTE_ETH_DEV_SRIOV(dev).active) {
3859 mrqc |= TXGBE_PORTCTL_NUMVT_64;
3863 mrqc |= TXGBE_PORTCTL_NUMVT_32;
3867 mrqc |= TXGBE_PORTCTL_NUMVT_16;
3871 "invalid pool number in IOV mode");
3875 wr32(hw, TXGBE_PORTCTL, mrqc);
3881 txgbe_dev_mq_rx_configure(struct rte_eth_dev *dev)
3883 if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
3885 * SRIOV inactive scheme
3886 * any DCB/RSS w/o VMDq multi-queue setting
3888 switch (dev->data->dev_conf.rxmode.mq_mode) {
3890 case ETH_MQ_RX_DCB_RSS:
3891 case ETH_MQ_RX_VMDQ_RSS:
3892 txgbe_rss_configure(dev);
3895 case ETH_MQ_RX_VMDQ_DCB:
3896 txgbe_vmdq_dcb_configure(dev);
3899 case ETH_MQ_RX_VMDQ_ONLY:
3900 txgbe_vmdq_rx_hw_configure(dev);
3903 case ETH_MQ_RX_NONE:
3905 /* if mq_mode is none, disable rss mode.*/
3906 txgbe_rss_disable(dev);
3910 /* SRIOV active scheme
3911 * Support RSS together with SRIOV.
3913 switch (dev->data->dev_conf.rxmode.mq_mode) {
3915 case ETH_MQ_RX_VMDQ_RSS:
3916 txgbe_config_vf_rss(dev);
3918 case ETH_MQ_RX_VMDQ_DCB:
3920 /* In SRIOV, the configuration is the same as VMDq case */
3921 txgbe_vmdq_dcb_configure(dev);
3923 /* DCB/RSS together with SRIOV is not supported */
3924 case ETH_MQ_RX_VMDQ_DCB_RSS:
3925 case ETH_MQ_RX_DCB_RSS:
3927 "Could not support DCB/RSS with VMDq & SRIOV");
3930 txgbe_config_vf_default(dev);
3939 txgbe_dev_mq_tx_configure(struct rte_eth_dev *dev)
3941 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
3945 /* disable arbiter */
3946 rttdcs = rd32(hw, TXGBE_ARBTXCTL);
3947 rttdcs |= TXGBE_ARBTXCTL_DIA;
3948 wr32(hw, TXGBE_ARBTXCTL, rttdcs);
3950 if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
3952 * SRIOV inactive scheme
3953 * any DCB w/o VMDq multi-queue setting
3955 if (dev->data->dev_conf.txmode.mq_mode == ETH_MQ_TX_VMDQ_ONLY)
3956 txgbe_vmdq_tx_hw_configure(hw);
3958 wr32m(hw, TXGBE_PORTCTL, TXGBE_PORTCTL_NUMVT_MASK, 0);
3960 switch (RTE_ETH_DEV_SRIOV(dev).active) {
3962 * SRIOV active scheme
3963 * FIXME if support DCB together with VMDq & SRIOV
3966 mtqc = TXGBE_PORTCTL_NUMVT_64;
3969 mtqc = TXGBE_PORTCTL_NUMVT_32;
3972 mtqc = TXGBE_PORTCTL_NUMVT_16;
3976 PMD_INIT_LOG(ERR, "invalid pool number in IOV mode");
3978 wr32m(hw, TXGBE_PORTCTL, TXGBE_PORTCTL_NUMVT_MASK, mtqc);
3981 /* re-enable arbiter */
3982 rttdcs &= ~TXGBE_ARBTXCTL_DIA;
3983 wr32(hw, TXGBE_ARBTXCTL, rttdcs);
3989 * txgbe_get_rscctl_maxdesc
3991 * @pool Memory pool of the Rx queue
3993 static inline uint32_t
3994 txgbe_get_rscctl_maxdesc(struct rte_mempool *pool)
3996 struct rte_pktmbuf_pool_private *mp_priv = rte_mempool_get_priv(pool);
3999 RTE_IPV4_MAX_PKT_LEN /
4000 (mp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM);
4003 return TXGBE_RXCFG_RSCMAX_16;
4004 else if (maxdesc >= 8)
4005 return TXGBE_RXCFG_RSCMAX_8;
4006 else if (maxdesc >= 4)
4007 return TXGBE_RXCFG_RSCMAX_4;
4009 return TXGBE_RXCFG_RSCMAX_1;
4013 * txgbe_set_rsc - configure RSC related port HW registers
4015 * Configures the port's RSC related registers.
4019 * Returns 0 in case of success or a non-zero error code
4022 txgbe_set_rsc(struct rte_eth_dev *dev)
4024 struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode;
4025 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
4026 struct rte_eth_dev_info dev_info = { 0 };
4027 bool rsc_capable = false;
4033 dev->dev_ops->dev_infos_get(dev, &dev_info);
4034 if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TCP_LRO)
4037 if (!rsc_capable && (rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO)) {
4038 PMD_INIT_LOG(CRIT, "LRO is requested on HW that doesn't "
4043 /* RSC global configuration */
4045 if ((rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC) &&
4046 (rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO)) {
4047 PMD_INIT_LOG(CRIT, "LRO can't be enabled when HW CRC "
4052 rfctl = rd32(hw, TXGBE_PSRCTL);
4053 if (rsc_capable && (rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO))
4054 rfctl &= ~TXGBE_PSRCTL_RSCDIA;
4056 rfctl |= TXGBE_PSRCTL_RSCDIA;
4057 wr32(hw, TXGBE_PSRCTL, rfctl);
4059 /* If LRO hasn't been requested - we are done here. */
4060 if (!(rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO))
4063 /* Set PSRCTL.RSCACK bit */
4064 rdrxctl = rd32(hw, TXGBE_PSRCTL);
4065 rdrxctl |= TXGBE_PSRCTL_RSCACK;
4066 wr32(hw, TXGBE_PSRCTL, rdrxctl);
4068 /* Per-queue RSC configuration */
4069 for (i = 0; i < dev->data->nb_rx_queues; i++) {
4070 struct txgbe_rx_queue *rxq = dev->data->rx_queues[i];
4072 rd32(hw, TXGBE_RXCFG(rxq->reg_idx));
4074 rd32(hw, TXGBE_POOLRSS(rxq->reg_idx));
4076 rd32(hw, TXGBE_ITR(rxq->reg_idx));
4079 * txgbe PMD doesn't support header-split at the moment.
4081 srrctl &= ~TXGBE_RXCFG_HDRLEN_MASK;
4082 srrctl |= TXGBE_RXCFG_HDRLEN(128);
4085 * TODO: Consider setting the Receive Descriptor Minimum
4086 * Threshold Size for an RSC case. This is not an obviously
4087 * beneficiary option but the one worth considering...
4090 srrctl |= TXGBE_RXCFG_RSCENA;
4091 srrctl &= ~TXGBE_RXCFG_RSCMAX_MASK;
4092 srrctl |= txgbe_get_rscctl_maxdesc(rxq->mb_pool);
4093 psrtype |= TXGBE_POOLRSS_L4HDR;
4096 * RSC: Set ITR interval corresponding to 2K ints/s.
4098 * Full-sized RSC aggregations for a 10Gb/s link will
4099 * arrive at about 20K aggregation/s rate.
4101 * 2K inst/s rate will make only 10% of the
4102 * aggregations to be closed due to the interrupt timer
4103 * expiration for a streaming at wire-speed case.
4105 * For a sparse streaming case this setting will yield
4106 * at most 500us latency for a single RSC aggregation.
4108 eitr &= ~TXGBE_ITR_IVAL_MASK;
4109 eitr |= TXGBE_ITR_IVAL_10G(TXGBE_QUEUE_ITR_INTERVAL_DEFAULT);
4110 eitr |= TXGBE_ITR_WRDSA;
4112 wr32(hw, TXGBE_RXCFG(rxq->reg_idx), srrctl);
4113 wr32(hw, TXGBE_POOLRSS(rxq->reg_idx), psrtype);
4114 wr32(hw, TXGBE_ITR(rxq->reg_idx), eitr);
4117 * RSC requires the mapping of the queue to the
4120 txgbe_set_ivar_map(hw, 0, rxq->reg_idx, i);
4125 PMD_INIT_LOG(DEBUG, "enabling LRO mode");
4131 txgbe_set_rx_function(struct rte_eth_dev *dev)
4134 struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev);
4137 * Initialize the appropriate LRO callback.
4139 * If all queues satisfy the bulk allocation preconditions
4140 * (adapter->rx_bulk_alloc_allowed is TRUE) then we may use
4141 * bulk allocation. Otherwise use a single allocation version.
4143 if (dev->data->lro) {
4144 if (adapter->rx_bulk_alloc_allowed) {
4145 PMD_INIT_LOG(DEBUG, "LRO is requested. Using a bulk "
4146 "allocation version");
4147 dev->rx_pkt_burst = txgbe_recv_pkts_lro_bulk_alloc;
4149 PMD_INIT_LOG(DEBUG, "LRO is requested. Using a single "
4150 "allocation version");
4151 dev->rx_pkt_burst = txgbe_recv_pkts_lro_single_alloc;
4153 } else if (dev->data->scattered_rx) {
4155 * Set the non-LRO scattered callback: there are bulk and
4156 * single allocation versions.
4158 if (adapter->rx_bulk_alloc_allowed) {
4159 PMD_INIT_LOG(DEBUG, "Using a Scattered with bulk "
4160 "allocation callback (port=%d).",
4161 dev->data->port_id);
4162 dev->rx_pkt_burst = txgbe_recv_pkts_lro_bulk_alloc;
4164 PMD_INIT_LOG(DEBUG, "Using Regular (non-vector, "
4165 "single allocation) "
4166 "Scattered Rx callback "
4168 dev->data->port_id);
4170 dev->rx_pkt_burst = txgbe_recv_pkts_lro_single_alloc;
4173 * Below we set "simple" callbacks according to port/queues parameters.
4174 * If parameters allow we are going to choose between the following
4177 * - Single buffer allocation (the simplest one)
4179 } else if (adapter->rx_bulk_alloc_allowed) {
4180 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are "
4181 "satisfied. Rx Burst Bulk Alloc function "
4182 "will be used on port=%d.",
4183 dev->data->port_id);
4185 dev->rx_pkt_burst = txgbe_recv_pkts_bulk_alloc;
4187 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are not "
4188 "satisfied, or Scattered Rx is requested "
4190 dev->data->port_id);
4192 dev->rx_pkt_burst = txgbe_recv_pkts;
4195 #ifdef RTE_LIB_SECURITY
4196 for (i = 0; i < dev->data->nb_rx_queues; i++) {
4197 struct txgbe_rx_queue *rxq = dev->data->rx_queues[i];
4199 rxq->using_ipsec = !!(dev->data->dev_conf.rxmode.offloads &
4200 DEV_RX_OFFLOAD_SECURITY);
4206 * Initializes Receive Unit.
4209 txgbe_dev_rx_init(struct rte_eth_dev *dev)
4211 struct txgbe_hw *hw;
4212 struct txgbe_rx_queue *rxq;
4221 struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode;
4224 PMD_INIT_FUNC_TRACE();
4225 hw = TXGBE_DEV_HW(dev);
4228 * Make sure receives are disabled while setting
4229 * up the RX context (registers, descriptor rings, etc.).
4231 wr32m(hw, TXGBE_MACRXCFG, TXGBE_MACRXCFG_ENA, 0);
4232 wr32m(hw, TXGBE_PBRXCTL, TXGBE_PBRXCTL_ENA, 0);
4234 /* Enable receipt of broadcasted frames */
4235 fctrl = rd32(hw, TXGBE_PSRCTL);
4236 fctrl |= TXGBE_PSRCTL_BCA;
4237 wr32(hw, TXGBE_PSRCTL, fctrl);
4240 * Configure CRC stripping, if any.
4242 hlreg0 = rd32(hw, TXGBE_SECRXCTL);
4243 if (rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC)
4244 hlreg0 &= ~TXGBE_SECRXCTL_CRCSTRIP;
4246 hlreg0 |= TXGBE_SECRXCTL_CRCSTRIP;
4247 wr32(hw, TXGBE_SECRXCTL, hlreg0);
4250 * Configure jumbo frame support, if any.
4252 if (rx_conf->offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) {
4253 wr32m(hw, TXGBE_FRMSZ, TXGBE_FRMSZ_MAX_MASK,
4254 TXGBE_FRMSZ_MAX(rx_conf->max_rx_pkt_len));
4256 wr32m(hw, TXGBE_FRMSZ, TXGBE_FRMSZ_MAX_MASK,
4257 TXGBE_FRMSZ_MAX(TXGBE_FRAME_SIZE_DFT));
4261 * If loopback mode is configured, set LPBK bit.
4263 hlreg0 = rd32(hw, TXGBE_PSRCTL);
4264 if (hw->mac.type == txgbe_mac_raptor &&
4265 dev->data->dev_conf.lpbk_mode)
4266 hlreg0 |= TXGBE_PSRCTL_LBENA;
4268 hlreg0 &= ~TXGBE_PSRCTL_LBENA;
4270 wr32(hw, TXGBE_PSRCTL, hlreg0);
4273 * Assume no header split and no VLAN strip support
4274 * on any Rx queue first .
4276 rx_conf->offloads &= ~DEV_RX_OFFLOAD_VLAN_STRIP;
4278 /* Setup RX queues */
4279 for (i = 0; i < dev->data->nb_rx_queues; i++) {
4280 rxq = dev->data->rx_queues[i];
4283 * Reset crc_len in case it was changed after queue setup by a
4284 * call to configure.
4286 if (rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC)
4287 rxq->crc_len = RTE_ETHER_CRC_LEN;
4291 /* Setup the Base and Length of the Rx Descriptor Rings */
4292 bus_addr = rxq->rx_ring_phys_addr;
4293 wr32(hw, TXGBE_RXBAL(rxq->reg_idx),
4294 (uint32_t)(bus_addr & BIT_MASK32));
4295 wr32(hw, TXGBE_RXBAH(rxq->reg_idx),
4296 (uint32_t)(bus_addr >> 32));
4297 wr32(hw, TXGBE_RXRP(rxq->reg_idx), 0);
4298 wr32(hw, TXGBE_RXWP(rxq->reg_idx), 0);
4300 srrctl = TXGBE_RXCFG_RNGLEN(rxq->nb_rx_desc);
4302 /* Set if packets are dropped when no descriptors available */
4304 srrctl |= TXGBE_RXCFG_DROP;
4307 * Configure the RX buffer size in the PKTLEN field of
4308 * the RXCFG register of the queue.
4309 * The value is in 1 KB resolution. Valid values can be from
4312 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) -
4313 RTE_PKTMBUF_HEADROOM);
4314 buf_size = ROUND_UP(buf_size, 0x1 << 10);
4315 srrctl |= TXGBE_RXCFG_PKTLEN(buf_size);
4317 wr32(hw, TXGBE_RXCFG(rxq->reg_idx), srrctl);
4319 /* It adds dual VLAN length for supporting dual VLAN */
4320 if (dev->data->dev_conf.rxmode.max_rx_pkt_len +
4321 2 * TXGBE_VLAN_TAG_SIZE > buf_size)
4322 dev->data->scattered_rx = 1;
4323 if (rxq->offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
4324 rx_conf->offloads |= DEV_RX_OFFLOAD_VLAN_STRIP;
4327 if (rx_conf->offloads & DEV_RX_OFFLOAD_SCATTER)
4328 dev->data->scattered_rx = 1;
4331 * Device configured with multiple RX queues.
4333 txgbe_dev_mq_rx_configure(dev);
4336 * Setup the Checksum Register.
4337 * Disable Full-Packet Checksum which is mutually exclusive with RSS.
4338 * Enable IP/L4 checksum computation by hardware if requested to do so.
4340 rxcsum = rd32(hw, TXGBE_PSRCTL);
4341 rxcsum |= TXGBE_PSRCTL_PCSD;
4342 if (rx_conf->offloads & DEV_RX_OFFLOAD_CHECKSUM)
4343 rxcsum |= TXGBE_PSRCTL_L4CSUM;
4345 rxcsum &= ~TXGBE_PSRCTL_L4CSUM;
4347 wr32(hw, TXGBE_PSRCTL, rxcsum);
4349 if (hw->mac.type == txgbe_mac_raptor) {
4350 rdrxctl = rd32(hw, TXGBE_SECRXCTL);
4351 if (rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC)
4352 rdrxctl &= ~TXGBE_SECRXCTL_CRCSTRIP;
4354 rdrxctl |= TXGBE_SECRXCTL_CRCSTRIP;
4355 wr32(hw, TXGBE_SECRXCTL, rdrxctl);
4358 rc = txgbe_set_rsc(dev);
4362 txgbe_set_rx_function(dev);
4368 * Initializes Transmit Unit.
4371 txgbe_dev_tx_init(struct rte_eth_dev *dev)
4373 struct txgbe_hw *hw;
4374 struct txgbe_tx_queue *txq;
4378 PMD_INIT_FUNC_TRACE();
4379 hw = TXGBE_DEV_HW(dev);
4381 /* Setup the Base and Length of the Tx Descriptor Rings */
4382 for (i = 0; i < dev->data->nb_tx_queues; i++) {
4383 txq = dev->data->tx_queues[i];
4385 bus_addr = txq->tx_ring_phys_addr;
4386 wr32(hw, TXGBE_TXBAL(txq->reg_idx),
4387 (uint32_t)(bus_addr & BIT_MASK32));
4388 wr32(hw, TXGBE_TXBAH(txq->reg_idx),
4389 (uint32_t)(bus_addr >> 32));
4390 wr32m(hw, TXGBE_TXCFG(txq->reg_idx), TXGBE_TXCFG_BUFLEN_MASK,
4391 TXGBE_TXCFG_BUFLEN(txq->nb_tx_desc));
4392 /* Setup the HW Tx Head and TX Tail descriptor pointers */
4393 wr32(hw, TXGBE_TXRP(txq->reg_idx), 0);
4394 wr32(hw, TXGBE_TXWP(txq->reg_idx), 0);
4397 /* Device configured with multiple TX queues. */
4398 txgbe_dev_mq_tx_configure(dev);
4402 * Set up link loopback mode Tx->Rx.
4404 static inline void __rte_cold
4405 txgbe_setup_loopback_link_raptor(struct txgbe_hw *hw)
4407 PMD_INIT_FUNC_TRACE();
4409 wr32m(hw, TXGBE_MACRXCFG, TXGBE_MACRXCFG_LB, TXGBE_MACRXCFG_LB);
4415 * Start Transmit and Receive Units.
4418 txgbe_dev_rxtx_start(struct rte_eth_dev *dev)
4420 struct txgbe_hw *hw;
4421 struct txgbe_tx_queue *txq;
4422 struct txgbe_rx_queue *rxq;
4428 PMD_INIT_FUNC_TRACE();
4429 hw = TXGBE_DEV_HW(dev);
4431 for (i = 0; i < dev->data->nb_tx_queues; i++) {
4432 txq = dev->data->tx_queues[i];
4433 /* Setup Transmit Threshold Registers */
4434 wr32m(hw, TXGBE_TXCFG(txq->reg_idx),
4435 TXGBE_TXCFG_HTHRESH_MASK |
4436 TXGBE_TXCFG_WTHRESH_MASK,
4437 TXGBE_TXCFG_HTHRESH(txq->hthresh) |
4438 TXGBE_TXCFG_WTHRESH(txq->wthresh));
4441 dmatxctl = rd32(hw, TXGBE_DMATXCTRL);
4442 dmatxctl |= TXGBE_DMATXCTRL_ENA;
4443 wr32(hw, TXGBE_DMATXCTRL, dmatxctl);
4445 for (i = 0; i < dev->data->nb_tx_queues; i++) {
4446 txq = dev->data->tx_queues[i];
4447 if (!txq->tx_deferred_start) {
4448 ret = txgbe_dev_tx_queue_start(dev, i);
4454 for (i = 0; i < dev->data->nb_rx_queues; i++) {
4455 rxq = dev->data->rx_queues[i];
4456 if (!rxq->rx_deferred_start) {
4457 ret = txgbe_dev_rx_queue_start(dev, i);
4463 /* Enable Receive engine */
4464 rxctrl = rd32(hw, TXGBE_PBRXCTL);
4465 rxctrl |= TXGBE_PBRXCTL_ENA;
4466 hw->mac.enable_rx_dma(hw, rxctrl);
4468 /* If loopback mode is enabled, set up the link accordingly */
4469 if (hw->mac.type == txgbe_mac_raptor &&
4470 dev->data->dev_conf.lpbk_mode)
4471 txgbe_setup_loopback_link_raptor(hw);
4473 #ifdef RTE_LIB_SECURITY
4474 if ((dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_SECURITY) ||
4475 (dev->data->dev_conf.txmode.offloads & DEV_TX_OFFLOAD_SECURITY)) {
4476 ret = txgbe_crypto_enable_ipsec(dev);
4479 "txgbe_crypto_enable_ipsec fails with %d.",
4490 txgbe_dev_save_rx_queue(struct txgbe_hw *hw, uint16_t rx_queue_id)
4492 u32 *reg = &hw->q_rx_regs[rx_queue_id * 8];
4493 *(reg++) = rd32(hw, TXGBE_RXBAL(rx_queue_id));
4494 *(reg++) = rd32(hw, TXGBE_RXBAH(rx_queue_id));
4495 *(reg++) = rd32(hw, TXGBE_RXCFG(rx_queue_id));
4499 txgbe_dev_store_rx_queue(struct txgbe_hw *hw, uint16_t rx_queue_id)
4501 u32 *reg = &hw->q_rx_regs[rx_queue_id * 8];
4502 wr32(hw, TXGBE_RXBAL(rx_queue_id), *(reg++));
4503 wr32(hw, TXGBE_RXBAH(rx_queue_id), *(reg++));
4504 wr32(hw, TXGBE_RXCFG(rx_queue_id), *(reg++) & ~TXGBE_RXCFG_ENA);
4508 txgbe_dev_save_tx_queue(struct txgbe_hw *hw, uint16_t tx_queue_id)
4510 u32 *reg = &hw->q_tx_regs[tx_queue_id * 8];
4511 *(reg++) = rd32(hw, TXGBE_TXBAL(tx_queue_id));
4512 *(reg++) = rd32(hw, TXGBE_TXBAH(tx_queue_id));
4513 *(reg++) = rd32(hw, TXGBE_TXCFG(tx_queue_id));
4517 txgbe_dev_store_tx_queue(struct txgbe_hw *hw, uint16_t tx_queue_id)
4519 u32 *reg = &hw->q_tx_regs[tx_queue_id * 8];
4520 wr32(hw, TXGBE_TXBAL(tx_queue_id), *(reg++));
4521 wr32(hw, TXGBE_TXBAH(tx_queue_id), *(reg++));
4522 wr32(hw, TXGBE_TXCFG(tx_queue_id), *(reg++) & ~TXGBE_TXCFG_ENA);
4526 * Start Receive Units for specified queue.
4529 txgbe_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
4531 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
4532 struct txgbe_rx_queue *rxq;
4536 PMD_INIT_FUNC_TRACE();
4538 rxq = dev->data->rx_queues[rx_queue_id];
4540 /* Allocate buffers for descriptor rings */
4541 if (txgbe_alloc_rx_queue_mbufs(rxq) != 0) {
4542 PMD_INIT_LOG(ERR, "Could not alloc mbuf for queue:%d",
4546 rxdctl = rd32(hw, TXGBE_RXCFG(rxq->reg_idx));
4547 rxdctl |= TXGBE_RXCFG_ENA;
4548 wr32(hw, TXGBE_RXCFG(rxq->reg_idx), rxdctl);
4550 /* Wait until RX Enable ready */
4551 poll_ms = RTE_TXGBE_REGISTER_POLL_WAIT_10_MS;
4554 rxdctl = rd32(hw, TXGBE_RXCFG(rxq->reg_idx));
4555 } while (--poll_ms && !(rxdctl & TXGBE_RXCFG_ENA));
4557 PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", rx_queue_id);
4559 wr32(hw, TXGBE_RXRP(rxq->reg_idx), 0);
4560 wr32(hw, TXGBE_RXWP(rxq->reg_idx), rxq->nb_rx_desc - 1);
4561 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
4567 * Stop Receive Units for specified queue.
4570 txgbe_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
4572 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
4573 struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev);
4574 struct txgbe_rx_queue *rxq;
4578 PMD_INIT_FUNC_TRACE();
4580 rxq = dev->data->rx_queues[rx_queue_id];
4582 txgbe_dev_save_rx_queue(hw, rxq->reg_idx);
4583 wr32m(hw, TXGBE_RXCFG(rxq->reg_idx), TXGBE_RXCFG_ENA, 0);
4585 /* Wait until RX Enable bit clear */
4586 poll_ms = RTE_TXGBE_REGISTER_POLL_WAIT_10_MS;
4589 rxdctl = rd32(hw, TXGBE_RXCFG(rxq->reg_idx));
4590 } while (--poll_ms && (rxdctl & TXGBE_RXCFG_ENA));
4592 PMD_INIT_LOG(ERR, "Could not disable Rx Queue %d", rx_queue_id);
4594 rte_delay_us(RTE_TXGBE_WAIT_100_US);
4595 txgbe_dev_store_rx_queue(hw, rxq->reg_idx);
4597 txgbe_rx_queue_release_mbufs(rxq);
4598 txgbe_reset_rx_queue(adapter, rxq);
4599 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
4605 * Start Transmit Units for specified queue.
4608 txgbe_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
4610 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
4611 struct txgbe_tx_queue *txq;
4615 PMD_INIT_FUNC_TRACE();
4617 txq = dev->data->tx_queues[tx_queue_id];
4618 wr32m(hw, TXGBE_TXCFG(txq->reg_idx), TXGBE_TXCFG_ENA, TXGBE_TXCFG_ENA);
4620 /* Wait until TX Enable ready */
4621 poll_ms = RTE_TXGBE_REGISTER_POLL_WAIT_10_MS;
4624 txdctl = rd32(hw, TXGBE_TXCFG(txq->reg_idx));
4625 } while (--poll_ms && !(txdctl & TXGBE_TXCFG_ENA));
4627 PMD_INIT_LOG(ERR, "Could not enable "
4628 "Tx Queue %d", tx_queue_id);
4631 wr32(hw, TXGBE_TXWP(txq->reg_idx), txq->tx_tail);
4632 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
4638 * Stop Transmit Units for specified queue.
4641 txgbe_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
4643 struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
4644 struct txgbe_tx_queue *txq;
4646 uint32_t txtdh, txtdt;
4649 PMD_INIT_FUNC_TRACE();
4651 txq = dev->data->tx_queues[tx_queue_id];
4653 /* Wait until TX queue is empty */
4654 poll_ms = RTE_TXGBE_REGISTER_POLL_WAIT_10_MS;
4656 rte_delay_us(RTE_TXGBE_WAIT_100_US);
4657 txtdh = rd32(hw, TXGBE_TXRP(txq->reg_idx));
4658 txtdt = rd32(hw, TXGBE_TXWP(txq->reg_idx));
4659 } while (--poll_ms && (txtdh != txtdt));
4662 "Tx Queue %d is not empty when stopping.",
4665 txgbe_dev_save_tx_queue(hw, txq->reg_idx);
4666 wr32m(hw, TXGBE_TXCFG(txq->reg_idx), TXGBE_TXCFG_ENA, 0);
4668 /* Wait until TX Enable bit clear */
4669 poll_ms = RTE_TXGBE_REGISTER_POLL_WAIT_10_MS;
4672 txdctl = rd32(hw, TXGBE_TXCFG(txq->reg_idx));
4673 } while (--poll_ms && (txdctl & TXGBE_TXCFG_ENA));
4675 PMD_INIT_LOG(ERR, "Could not disable Tx Queue %d",
4678 rte_delay_us(RTE_TXGBE_WAIT_100_US);
4679 txgbe_dev_store_tx_queue(hw, txq->reg_idx);
4681 if (txq->ops != NULL) {
4682 txq->ops->release_mbufs(txq);
4683 txq->ops->reset(txq);
4685 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
4691 txgbe_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
4692 struct rte_eth_rxq_info *qinfo)
4694 struct txgbe_rx_queue *rxq;
4696 rxq = dev->data->rx_queues[queue_id];
4698 qinfo->mp = rxq->mb_pool;
4699 qinfo->scattered_rx = dev->data->scattered_rx;
4700 qinfo->nb_desc = rxq->nb_rx_desc;
4702 qinfo->conf.rx_free_thresh = rxq->rx_free_thresh;
4703 qinfo->conf.rx_drop_en = rxq->drop_en;
4704 qinfo->conf.rx_deferred_start = rxq->rx_deferred_start;
4705 qinfo->conf.offloads = rxq->offloads;
4709 txgbe_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
4710 struct rte_eth_txq_info *qinfo)
4712 struct txgbe_tx_queue *txq;
4714 txq = dev->data->tx_queues[queue_id];
4716 qinfo->nb_desc = txq->nb_tx_desc;
4718 qinfo->conf.tx_thresh.pthresh = txq->pthresh;
4719 qinfo->conf.tx_thresh.hthresh = txq->hthresh;
4720 qinfo->conf.tx_thresh.wthresh = txq->wthresh;
4722 qinfo->conf.tx_free_thresh = txq->tx_free_thresh;
4723 qinfo->conf.offloads = txq->offloads;
4724 qinfo->conf.tx_deferred_start = txq->tx_deferred_start;
4728 * [VF] Initializes Receive Unit.
4731 txgbevf_dev_rx_init(struct rte_eth_dev *dev)
4733 struct txgbe_hw *hw;
4734 struct txgbe_rx_queue *rxq;
4735 struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
4737 uint32_t srrctl, psrtype;
4742 PMD_INIT_FUNC_TRACE();
4743 hw = TXGBE_DEV_HW(dev);
4745 if (rte_is_power_of_2(dev->data->nb_rx_queues) == 0) {
4746 PMD_INIT_LOG(ERR, "The number of Rx queue invalid, "
4747 "it should be power of 2");
4751 if (dev->data->nb_rx_queues > hw->mac.max_rx_queues) {
4752 PMD_INIT_LOG(ERR, "The number of Rx queue invalid, "
4753 "it should be equal to or less than %d",
4754 hw->mac.max_rx_queues);
4759 * When the VF driver issues a TXGBE_VF_RESET request, the PF driver
4760 * disables the VF receipt of packets if the PF MTU is > 1500.
4761 * This is done to deal with limitations that imposes
4762 * the PF and all VFs to share the same MTU.
4763 * Then, the PF driver enables again the VF receipt of packet when
4764 * the VF driver issues a TXGBE_VF_SET_LPE request.
4765 * In the meantime, the VF device cannot be used, even if the VF driver
4766 * and the Guest VM network stack are ready to accept packets with a
4767 * size up to the PF MTU.
4768 * As a work-around to this PF behaviour, force the call to
4769 * txgbevf_rlpml_set_vf even if jumbo frames are not used. This way,
4770 * VF packets received can work in all cases.
4772 if (txgbevf_rlpml_set_vf(hw,
4773 (uint16_t)dev->data->dev_conf.rxmode.max_rx_pkt_len)) {
4774 PMD_INIT_LOG(ERR, "Set max packet length to %d failed.",
4775 dev->data->dev_conf.rxmode.max_rx_pkt_len);
4780 * Assume no header split and no VLAN strip support
4781 * on any Rx queue first .
4783 rxmode->offloads &= ~DEV_RX_OFFLOAD_VLAN_STRIP;
4785 /* Set PSR type for VF RSS according to max Rx queue */
4786 psrtype = TXGBE_VFPLCFG_PSRL4HDR |
4787 TXGBE_VFPLCFG_PSRL4HDR |
4788 TXGBE_VFPLCFG_PSRL2HDR |
4789 TXGBE_VFPLCFG_PSRTUNHDR |
4790 TXGBE_VFPLCFG_PSRTUNMAC;
4791 wr32(hw, TXGBE_VFPLCFG, TXGBE_VFPLCFG_PSR(psrtype));
4793 /* Setup RX queues */
4794 for (i = 0; i < dev->data->nb_rx_queues; i++) {
4795 rxq = dev->data->rx_queues[i];
4797 /* Allocate buffers for descriptor rings */
4798 ret = txgbe_alloc_rx_queue_mbufs(rxq);
4802 /* Setup the Base and Length of the Rx Descriptor Rings */
4803 bus_addr = rxq->rx_ring_phys_addr;
4805 wr32(hw, TXGBE_RXBAL(i),
4806 (uint32_t)(bus_addr & BIT_MASK32));
4807 wr32(hw, TXGBE_RXBAH(i),
4808 (uint32_t)(bus_addr >> 32));
4809 wr32(hw, TXGBE_RXRP(i), 0);
4810 wr32(hw, TXGBE_RXWP(i), 0);
4812 /* Configure the RXCFG register */
4813 srrctl = TXGBE_RXCFG_RNGLEN(rxq->nb_rx_desc);
4815 /* Set if packets are dropped when no descriptors available */
4817 srrctl |= TXGBE_RXCFG_DROP;
4820 * Configure the RX buffer size in the PKTLEN field of
4821 * the RXCFG register of the queue.
4822 * The value is in 1 KB resolution. Valid values can be from
4825 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) -
4826 RTE_PKTMBUF_HEADROOM);
4827 buf_size = ROUND_UP(buf_size, 1 << 10);
4828 srrctl |= TXGBE_RXCFG_PKTLEN(buf_size);
4831 * VF modification to write virtual function RXCFG register
4833 wr32(hw, TXGBE_RXCFG(i), srrctl);
4835 if (rxmode->offloads & DEV_RX_OFFLOAD_SCATTER ||
4836 /* It adds dual VLAN length for supporting dual VLAN */
4837 (rxmode->max_rx_pkt_len +
4838 2 * TXGBE_VLAN_TAG_SIZE) > buf_size) {
4839 if (!dev->data->scattered_rx)
4840 PMD_INIT_LOG(DEBUG, "forcing scatter mode");
4841 dev->data->scattered_rx = 1;
4844 if (rxq->offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
4845 rxmode->offloads |= DEV_RX_OFFLOAD_VLAN_STRIP;
4849 * Device configured with multiple RX queues.
4851 txgbe_dev_mq_rx_configure(dev);
4853 txgbe_set_rx_function(dev);
4859 * [VF] Initializes Transmit Unit.
4862 txgbevf_dev_tx_init(struct rte_eth_dev *dev)
4864 struct txgbe_hw *hw;
4865 struct txgbe_tx_queue *txq;
4869 PMD_INIT_FUNC_TRACE();
4870 hw = TXGBE_DEV_HW(dev);
4872 /* Setup the Base and Length of the Tx Descriptor Rings */
4873 for (i = 0; i < dev->data->nb_tx_queues; i++) {
4874 txq = dev->data->tx_queues[i];
4875 bus_addr = txq->tx_ring_phys_addr;
4876 wr32(hw, TXGBE_TXBAL(i),
4877 (uint32_t)(bus_addr & BIT_MASK32));
4878 wr32(hw, TXGBE_TXBAH(i),
4879 (uint32_t)(bus_addr >> 32));
4880 wr32m(hw, TXGBE_TXCFG(i), TXGBE_TXCFG_BUFLEN_MASK,
4881 TXGBE_TXCFG_BUFLEN(txq->nb_tx_desc));
4882 /* Setup the HW Tx Head and TX Tail descriptor pointers */
4883 wr32(hw, TXGBE_TXRP(i), 0);
4884 wr32(hw, TXGBE_TXWP(i), 0);
4889 txgbe_rss_conf_init(struct txgbe_rte_flow_rss_conf *out,
4890 const struct rte_flow_action_rss *in)
4892 if (in->key_len > RTE_DIM(out->key) ||
4893 in->queue_num > RTE_DIM(out->queue))
4895 out->conf = (struct rte_flow_action_rss){
4899 .key_len = in->key_len,
4900 .queue_num = in->queue_num,
4901 .key = memcpy(out->key, in->key, in->key_len),
4902 .queue = memcpy(out->queue, in->queue,
4903 sizeof(*in->queue) * in->queue_num),
4909 txgbe_action_rss_same(const struct rte_flow_action_rss *comp,
4910 const struct rte_flow_action_rss *with)
4912 return (comp->func == with->func &&
4913 comp->level == with->level &&
4914 comp->types == with->types &&
4915 comp->key_len == with->key_len &&
4916 comp->queue_num == with->queue_num &&
4917 !memcmp(comp->key, with->key, with->key_len) &&
4918 !memcmp(comp->queue, with->queue,
4919 sizeof(*with->queue) * with->queue_num));
4923 txgbe_config_rss_filter(struct rte_eth_dev *dev,
4924 struct txgbe_rte_flow_rss_conf *conf, bool add)
4926 struct txgbe_hw *hw;
4930 struct rte_eth_rss_conf rss_conf = {
4931 .rss_key = conf->conf.key_len ?
4932 (void *)(uintptr_t)conf->conf.key : NULL,
4933 .rss_key_len = conf->conf.key_len,
4934 .rss_hf = conf->conf.types,
4936 struct txgbe_filter_info *filter_info = TXGBE_DEV_FILTER(dev);
4938 PMD_INIT_FUNC_TRACE();
4939 hw = TXGBE_DEV_HW(dev);
4942 if (txgbe_action_rss_same(&filter_info->rss_info.conf,
4944 txgbe_rss_disable(dev);
4945 memset(&filter_info->rss_info, 0,
4946 sizeof(struct txgbe_rte_flow_rss_conf));
4952 if (filter_info->rss_info.conf.queue_num)
4954 /* Fill in redirection table
4955 * The byte-swap is needed because NIC registers are in
4956 * little-endian order.
4959 for (i = 0, j = 0; i < ETH_RSS_RETA_SIZE_128; i++, j++) {
4960 if (j == conf->conf.queue_num)
4962 reta = (reta >> 8) | LS32(conf->conf.queue[j], 24, 0xFF);
4964 wr32a(hw, TXGBE_REG_RSSTBL, i >> 2, reta);
4967 /* Configure the RSS key and the RSS protocols used to compute
4968 * the RSS hash of input packets.
4970 if ((rss_conf.rss_hf & TXGBE_RSS_OFFLOAD_ALL) == 0) {
4971 txgbe_rss_disable(dev);
4974 if (rss_conf.rss_key == NULL)
4975 rss_conf.rss_key = rss_intel_key; /* Default hash key */
4976 txgbe_dev_rss_hash_update(dev, &rss_conf);
4978 if (txgbe_rss_conf_init(&filter_info->rss_info, &conf->conf))