1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2016 Intel Corporation.
3 * Copyright 2014 6WIND S.A.
17 #include <rte_byteorder.h>
18 #include <rte_common.h>
19 #include <rte_cycles.h>
21 #include <rte_debug.h>
22 #include <rte_interrupts.h>
24 #include <rte_memory.h>
25 #include <rte_memzone.h>
26 #include <rte_launch.h>
28 #include <rte_per_lcore.h>
29 #include <rte_lcore.h>
30 #include <rte_atomic.h>
31 #include <rte_branch_prediction.h>
32 #include <rte_mempool.h>
33 #include <rte_malloc.h>
35 #include <rte_ether.h>
36 #include <ethdev_driver.h>
37 #include <rte_security_driver.h>
38 #include <rte_prefetch.h>
42 #include <rte_string_fns.h>
43 #include <rte_errno.h>
48 #include "ixgbe_logs.h"
49 #include "base/ixgbe_api.h"
50 #include "base/ixgbe_vf.h"
51 #include "ixgbe_ethdev.h"
52 #include "base/ixgbe_dcb.h"
53 #include "base/ixgbe_common.h"
54 #include "ixgbe_rxtx.h"
56 #ifdef RTE_LIBRTE_IEEE1588
57 #define IXGBE_TX_IEEE1588_TMST RTE_MBUF_F_TX_IEEE1588_TMST
59 #define IXGBE_TX_IEEE1588_TMST 0
61 /* Bit Mask to indicate what bits required for building TX context */
62 #define IXGBE_TX_OFFLOAD_MASK (RTE_MBUF_F_TX_OUTER_IPV6 | \
63 RTE_MBUF_F_TX_OUTER_IPV4 | \
64 RTE_MBUF_F_TX_IPV6 | \
65 RTE_MBUF_F_TX_IPV4 | \
66 RTE_MBUF_F_TX_VLAN | \
67 RTE_MBUF_F_TX_IP_CKSUM | \
68 RTE_MBUF_F_TX_L4_MASK | \
69 RTE_MBUF_F_TX_TCP_SEG | \
70 RTE_MBUF_F_TX_MACSEC | \
71 RTE_MBUF_F_TX_OUTER_IP_CKSUM | \
72 RTE_MBUF_F_TX_SEC_OFFLOAD | \
73 IXGBE_TX_IEEE1588_TMST)
75 #define IXGBE_TX_OFFLOAD_NOTSUP_MASK \
76 (RTE_MBUF_F_TX_OFFLOAD_MASK ^ IXGBE_TX_OFFLOAD_MASK)
79 #define RTE_PMD_USE_PREFETCH
82 #ifdef RTE_PMD_USE_PREFETCH
84 * Prefetch a cache line into all cache levels.
86 #define rte_ixgbe_prefetch(p) rte_prefetch0(p)
88 #define rte_ixgbe_prefetch(p) do {} while (0)
91 /*********************************************************************
95 **********************************************************************/
98 * Check for descriptors with their DD bit set and free mbufs.
99 * Return the total number of buffers freed.
101 static __rte_always_inline int
102 ixgbe_tx_free_bufs(struct ixgbe_tx_queue *txq)
104 struct ixgbe_tx_entry *txep;
107 struct rte_mbuf *m, *free[RTE_IXGBE_TX_MAX_FREE_BUF_SZ];
109 /* check DD bit on threshold descriptor */
110 status = txq->tx_ring[txq->tx_next_dd].wb.status;
111 if (!(status & rte_cpu_to_le_32(IXGBE_ADVTXD_STAT_DD)))
115 * first buffer to free from S/W ring is at index
116 * tx_next_dd - (tx_rs_thresh-1)
118 txep = &(txq->sw_ring[txq->tx_next_dd - (txq->tx_rs_thresh - 1)]);
120 for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) {
121 /* free buffers one at a time */
122 m = rte_pktmbuf_prefree_seg(txep->mbuf);
125 if (unlikely(m == NULL))
128 if (nb_free >= RTE_IXGBE_TX_MAX_FREE_BUF_SZ ||
129 (nb_free > 0 && m->pool != free[0]->pool)) {
130 rte_mempool_put_bulk(free[0]->pool,
131 (void **)free, nb_free);
139 rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free);
141 /* buffers were freed, update counters */
142 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh);
143 txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh);
144 if (txq->tx_next_dd >= txq->nb_tx_desc)
145 txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
147 return txq->tx_rs_thresh;
150 /* Populate 4 descriptors with data from 4 mbufs */
152 tx4(volatile union ixgbe_adv_tx_desc *txdp, struct rte_mbuf **pkts)
154 uint64_t buf_dma_addr;
158 for (i = 0; i < 4; ++i, ++txdp, ++pkts) {
159 buf_dma_addr = rte_mbuf_data_iova(*pkts);
160 pkt_len = (*pkts)->data_len;
162 /* write data to descriptor */
163 txdp->read.buffer_addr = rte_cpu_to_le_64(buf_dma_addr);
165 txdp->read.cmd_type_len =
166 rte_cpu_to_le_32((uint32_t)DCMD_DTYP_FLAGS | pkt_len);
168 txdp->read.olinfo_status =
169 rte_cpu_to_le_32(pkt_len << IXGBE_ADVTXD_PAYLEN_SHIFT);
171 rte_prefetch0(&(*pkts)->pool);
175 /* Populate 1 descriptor with data from 1 mbuf */
177 tx1(volatile union ixgbe_adv_tx_desc *txdp, struct rte_mbuf **pkts)
179 uint64_t buf_dma_addr;
182 buf_dma_addr = rte_mbuf_data_iova(*pkts);
183 pkt_len = (*pkts)->data_len;
185 /* write data to descriptor */
186 txdp->read.buffer_addr = rte_cpu_to_le_64(buf_dma_addr);
187 txdp->read.cmd_type_len =
188 rte_cpu_to_le_32((uint32_t)DCMD_DTYP_FLAGS | pkt_len);
189 txdp->read.olinfo_status =
190 rte_cpu_to_le_32(pkt_len << IXGBE_ADVTXD_PAYLEN_SHIFT);
191 rte_prefetch0(&(*pkts)->pool);
195 * Fill H/W descriptor ring with mbuf data.
196 * Copy mbuf pointers to the S/W ring.
199 ixgbe_tx_fill_hw_ring(struct ixgbe_tx_queue *txq, struct rte_mbuf **pkts,
202 volatile union ixgbe_adv_tx_desc *txdp = &(txq->tx_ring[txq->tx_tail]);
203 struct ixgbe_tx_entry *txep = &(txq->sw_ring[txq->tx_tail]);
204 const int N_PER_LOOP = 4;
205 const int N_PER_LOOP_MASK = N_PER_LOOP-1;
206 int mainpart, leftover;
210 * Process most of the packets in chunks of N pkts. Any
211 * leftover packets will get processed one at a time.
213 mainpart = (nb_pkts & ((uint32_t) ~N_PER_LOOP_MASK));
214 leftover = (nb_pkts & ((uint32_t) N_PER_LOOP_MASK));
215 for (i = 0; i < mainpart; i += N_PER_LOOP) {
216 /* Copy N mbuf pointers to the S/W ring */
217 for (j = 0; j < N_PER_LOOP; ++j) {
218 (txep + i + j)->mbuf = *(pkts + i + j);
220 tx4(txdp + i, pkts + i);
223 if (unlikely(leftover > 0)) {
224 for (i = 0; i < leftover; ++i) {
225 (txep + mainpart + i)->mbuf = *(pkts + mainpart + i);
226 tx1(txdp + mainpart + i, pkts + mainpart + i);
231 static inline uint16_t
232 tx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
235 struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue;
236 volatile union ixgbe_adv_tx_desc *tx_r = txq->tx_ring;
240 * Begin scanning the H/W ring for done descriptors when the
241 * number of available descriptors drops below tx_free_thresh. For
242 * each done descriptor, free the associated buffer.
244 if (txq->nb_tx_free < txq->tx_free_thresh)
245 ixgbe_tx_free_bufs(txq);
247 /* Only use descriptors that are available */
248 nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts);
249 if (unlikely(nb_pkts == 0))
252 /* Use exactly nb_pkts descriptors */
253 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts);
256 * At this point, we know there are enough descriptors in the
257 * ring to transmit all the packets. This assumes that each
258 * mbuf contains a single segment, and that no new offloads
259 * are expected, which would require a new context descriptor.
263 * See if we're going to wrap-around. If so, handle the top
264 * of the descriptor ring first, then do the bottom. If not,
265 * the processing looks just like the "bottom" part anyway...
267 if ((txq->tx_tail + nb_pkts) > txq->nb_tx_desc) {
268 n = (uint16_t)(txq->nb_tx_desc - txq->tx_tail);
269 ixgbe_tx_fill_hw_ring(txq, tx_pkts, n);
272 * We know that the last descriptor in the ring will need to
273 * have its RS bit set because tx_rs_thresh has to be
274 * a divisor of the ring size
276 tx_r[txq->tx_next_rs].read.cmd_type_len |=
277 rte_cpu_to_le_32(IXGBE_ADVTXD_DCMD_RS);
278 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
283 /* Fill H/W descriptor ring with mbuf data */
284 ixgbe_tx_fill_hw_ring(txq, tx_pkts + n, (uint16_t)(nb_pkts - n));
285 txq->tx_tail = (uint16_t)(txq->tx_tail + (nb_pkts - n));
288 * Determine if RS bit should be set
289 * This is what we actually want:
290 * if ((txq->tx_tail - 1) >= txq->tx_next_rs)
291 * but instead of subtracting 1 and doing >=, we can just do
292 * greater than without subtracting.
294 if (txq->tx_tail > txq->tx_next_rs) {
295 tx_r[txq->tx_next_rs].read.cmd_type_len |=
296 rte_cpu_to_le_32(IXGBE_ADVTXD_DCMD_RS);
297 txq->tx_next_rs = (uint16_t)(txq->tx_next_rs +
299 if (txq->tx_next_rs >= txq->nb_tx_desc)
300 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
304 * Check for wrap-around. This would only happen if we used
305 * up to the last descriptor in the ring, no more, no less.
307 if (txq->tx_tail >= txq->nb_tx_desc)
310 /* update tail pointer */
312 IXGBE_PCI_REG_WC_WRITE_RELAXED(txq->tdt_reg_addr, txq->tx_tail);
318 ixgbe_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts,
323 /* Try to transmit at least chunks of TX_MAX_BURST pkts */
324 if (likely(nb_pkts <= RTE_PMD_IXGBE_TX_MAX_BURST))
325 return tx_xmit_pkts(tx_queue, tx_pkts, nb_pkts);
327 /* transmit more than the max burst, in chunks of TX_MAX_BURST */
332 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_IXGBE_TX_MAX_BURST);
333 ret = tx_xmit_pkts(tx_queue, &(tx_pkts[nb_tx]), n);
334 nb_tx = (uint16_t)(nb_tx + ret);
335 nb_pkts = (uint16_t)(nb_pkts - ret);
344 ixgbe_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
348 struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue;
353 num = (uint16_t)RTE_MIN(nb_pkts, txq->tx_rs_thresh);
354 ret = ixgbe_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
366 ixgbe_set_xmit_ctx(struct ixgbe_tx_queue *txq,
367 volatile struct ixgbe_adv_tx_context_desc *ctx_txd,
368 uint64_t ol_flags, union ixgbe_tx_offload tx_offload,
369 __rte_unused uint64_t *mdata)
371 uint32_t type_tucmd_mlhl;
372 uint32_t mss_l4len_idx = 0;
374 uint32_t vlan_macip_lens;
375 union ixgbe_tx_offload tx_offload_mask;
376 uint32_t seqnum_seed = 0;
378 ctx_idx = txq->ctx_curr;
379 tx_offload_mask.data[0] = 0;
380 tx_offload_mask.data[1] = 0;
383 /* Specify which HW CTX to upload. */
384 mss_l4len_idx |= (ctx_idx << IXGBE_ADVTXD_IDX_SHIFT);
386 if (ol_flags & RTE_MBUF_F_TX_VLAN)
387 tx_offload_mask.vlan_tci |= ~0;
389 /* check if TCP segmentation required for this packet */
390 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {
391 /* implies IP cksum in IPv4 */
392 if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM)
393 type_tucmd_mlhl = IXGBE_ADVTXD_TUCMD_IPV4 |
394 IXGBE_ADVTXD_TUCMD_L4T_TCP |
395 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
397 type_tucmd_mlhl = IXGBE_ADVTXD_TUCMD_IPV6 |
398 IXGBE_ADVTXD_TUCMD_L4T_TCP |
399 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
401 tx_offload_mask.l2_len |= ~0;
402 tx_offload_mask.l3_len |= ~0;
403 tx_offload_mask.l4_len |= ~0;
404 tx_offload_mask.tso_segsz |= ~0;
405 mss_l4len_idx |= tx_offload.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT;
406 mss_l4len_idx |= tx_offload.l4_len << IXGBE_ADVTXD_L4LEN_SHIFT;
407 } else { /* no TSO, check if hardware checksum is needed */
408 if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM) {
409 type_tucmd_mlhl = IXGBE_ADVTXD_TUCMD_IPV4;
410 tx_offload_mask.l2_len |= ~0;
411 tx_offload_mask.l3_len |= ~0;
414 switch (ol_flags & RTE_MBUF_F_TX_L4_MASK) {
415 case RTE_MBUF_F_TX_UDP_CKSUM:
416 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP |
417 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
418 mss_l4len_idx |= sizeof(struct rte_udp_hdr)
419 << IXGBE_ADVTXD_L4LEN_SHIFT;
420 tx_offload_mask.l2_len |= ~0;
421 tx_offload_mask.l3_len |= ~0;
423 case RTE_MBUF_F_TX_TCP_CKSUM:
424 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP |
425 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
426 mss_l4len_idx |= sizeof(struct rte_tcp_hdr)
427 << IXGBE_ADVTXD_L4LEN_SHIFT;
428 tx_offload_mask.l2_len |= ~0;
429 tx_offload_mask.l3_len |= ~0;
431 case RTE_MBUF_F_TX_SCTP_CKSUM:
432 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP |
433 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
434 mss_l4len_idx |= sizeof(struct rte_sctp_hdr)
435 << IXGBE_ADVTXD_L4LEN_SHIFT;
436 tx_offload_mask.l2_len |= ~0;
437 tx_offload_mask.l3_len |= ~0;
440 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_RSV |
441 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
446 if (ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) {
447 tx_offload_mask.outer_l2_len |= ~0;
448 tx_offload_mask.outer_l3_len |= ~0;
449 tx_offload_mask.l2_len |= ~0;
450 seqnum_seed |= tx_offload.outer_l3_len
451 << IXGBE_ADVTXD_OUTER_IPLEN;
452 seqnum_seed |= tx_offload.l2_len
453 << IXGBE_ADVTXD_TUNNEL_LEN;
455 #ifdef RTE_LIB_SECURITY
456 if (ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD) {
457 union ixgbe_crypto_tx_desc_md *md =
458 (union ixgbe_crypto_tx_desc_md *)mdata;
460 (IXGBE_ADVTXD_IPSEC_SA_INDEX_MASK & md->sa_idx);
461 type_tucmd_mlhl |= md->enc ?
462 (IXGBE_ADVTXD_TUCMD_IPSEC_TYPE_ESP |
463 IXGBE_ADVTXD_TUCMD_IPSEC_ENCRYPT_EN) : 0;
465 (md->pad_len & IXGBE_ADVTXD_IPSEC_ESP_LEN_MASK);
466 tx_offload_mask.sa_idx |= ~0;
467 tx_offload_mask.sec_pad_len |= ~0;
471 txq->ctx_cache[ctx_idx].flags = ol_flags;
472 txq->ctx_cache[ctx_idx].tx_offload.data[0] =
473 tx_offload_mask.data[0] & tx_offload.data[0];
474 txq->ctx_cache[ctx_idx].tx_offload.data[1] =
475 tx_offload_mask.data[1] & tx_offload.data[1];
476 txq->ctx_cache[ctx_idx].tx_offload_mask = tx_offload_mask;
478 ctx_txd->type_tucmd_mlhl = rte_cpu_to_le_32(type_tucmd_mlhl);
479 vlan_macip_lens = tx_offload.l3_len;
480 if (ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM)
481 vlan_macip_lens |= (tx_offload.outer_l2_len <<
482 IXGBE_ADVTXD_MACLEN_SHIFT);
484 vlan_macip_lens |= (tx_offload.l2_len <<
485 IXGBE_ADVTXD_MACLEN_SHIFT);
486 vlan_macip_lens |= ((uint32_t)tx_offload.vlan_tci << IXGBE_ADVTXD_VLAN_SHIFT);
487 ctx_txd->vlan_macip_lens = rte_cpu_to_le_32(vlan_macip_lens);
488 ctx_txd->mss_l4len_idx = rte_cpu_to_le_32(mss_l4len_idx);
489 ctx_txd->seqnum_seed = seqnum_seed;
493 * Check which hardware context can be used. Use the existing match
494 * or create a new context descriptor.
496 static inline uint32_t
497 what_advctx_update(struct ixgbe_tx_queue *txq, uint64_t flags,
498 union ixgbe_tx_offload tx_offload)
500 /* If match with the current used context */
501 if (likely((txq->ctx_cache[txq->ctx_curr].flags == flags) &&
502 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] ==
503 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0]
504 & tx_offload.data[0])) &&
505 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] ==
506 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1]
507 & tx_offload.data[1]))))
508 return txq->ctx_curr;
510 /* What if match with the next context */
512 if (likely((txq->ctx_cache[txq->ctx_curr].flags == flags) &&
513 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] ==
514 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0]
515 & tx_offload.data[0])) &&
516 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] ==
517 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1]
518 & tx_offload.data[1]))))
519 return txq->ctx_curr;
521 /* Mismatch, use the previous context */
522 return IXGBE_CTX_NUM;
525 static inline uint32_t
526 tx_desc_cksum_flags_to_olinfo(uint64_t ol_flags)
530 if ((ol_flags & RTE_MBUF_F_TX_L4_MASK) != RTE_MBUF_F_TX_L4_NO_CKSUM)
531 tmp |= IXGBE_ADVTXD_POPTS_TXSM;
532 if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM)
533 tmp |= IXGBE_ADVTXD_POPTS_IXSM;
534 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG)
535 tmp |= IXGBE_ADVTXD_POPTS_TXSM;
539 static inline uint32_t
540 tx_desc_ol_flags_to_cmdtype(uint64_t ol_flags)
542 uint32_t cmdtype = 0;
544 if (ol_flags & RTE_MBUF_F_TX_VLAN)
545 cmdtype |= IXGBE_ADVTXD_DCMD_VLE;
546 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG)
547 cmdtype |= IXGBE_ADVTXD_DCMD_TSE;
548 if (ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM)
549 cmdtype |= (1 << IXGBE_ADVTXD_OUTERIPCS_SHIFT);
550 if (ol_flags & RTE_MBUF_F_TX_MACSEC)
551 cmdtype |= IXGBE_ADVTXD_MAC_LINKSEC;
555 /* Default RS bit threshold values */
556 #ifndef DEFAULT_TX_RS_THRESH
557 #define DEFAULT_TX_RS_THRESH 32
559 #ifndef DEFAULT_TX_FREE_THRESH
560 #define DEFAULT_TX_FREE_THRESH 32
563 /* Reset transmit descriptors after they have been used */
565 ixgbe_xmit_cleanup(struct ixgbe_tx_queue *txq)
567 struct ixgbe_tx_entry *sw_ring = txq->sw_ring;
568 volatile union ixgbe_adv_tx_desc *txr = txq->tx_ring;
569 uint16_t last_desc_cleaned = txq->last_desc_cleaned;
570 uint16_t nb_tx_desc = txq->nb_tx_desc;
571 uint16_t desc_to_clean_to;
572 uint16_t nb_tx_to_clean;
575 /* Determine the last descriptor needing to be cleaned */
576 desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->tx_rs_thresh);
577 if (desc_to_clean_to >= nb_tx_desc)
578 desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc);
580 /* Check to make sure the last descriptor to clean is done */
581 desc_to_clean_to = sw_ring[desc_to_clean_to].last_id;
582 status = txr[desc_to_clean_to].wb.status;
583 if (!(status & rte_cpu_to_le_32(IXGBE_TXD_STAT_DD))) {
585 "TX descriptor %4u is not done"
586 "(port=%d queue=%d)",
588 txq->port_id, txq->queue_id);
589 /* Failed to clean any descriptors, better luck next time */
593 /* Figure out how many descriptors will be cleaned */
594 if (last_desc_cleaned > desc_to_clean_to)
595 nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) +
598 nb_tx_to_clean = (uint16_t)(desc_to_clean_to -
602 "Cleaning %4u TX descriptors: %4u to %4u "
603 "(port=%d queue=%d)",
604 nb_tx_to_clean, last_desc_cleaned, desc_to_clean_to,
605 txq->port_id, txq->queue_id);
608 * The last descriptor to clean is done, so that means all the
609 * descriptors from the last descriptor that was cleaned
610 * up to the last descriptor with the RS bit set
611 * are done. Only reset the threshold descriptor.
613 txr[desc_to_clean_to].wb.status = 0;
615 /* Update the txq to reflect the last descriptor that was cleaned */
616 txq->last_desc_cleaned = desc_to_clean_to;
617 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + nb_tx_to_clean);
624 ixgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
627 struct ixgbe_tx_queue *txq;
628 struct ixgbe_tx_entry *sw_ring;
629 struct ixgbe_tx_entry *txe, *txn;
630 volatile union ixgbe_adv_tx_desc *txr;
631 volatile union ixgbe_adv_tx_desc *txd, *txp;
632 struct rte_mbuf *tx_pkt;
633 struct rte_mbuf *m_seg;
634 uint64_t buf_dma_addr;
635 uint32_t olinfo_status;
636 uint32_t cmd_type_len;
647 union ixgbe_tx_offload tx_offload;
648 #ifdef RTE_LIB_SECURITY
652 tx_offload.data[0] = 0;
653 tx_offload.data[1] = 0;
655 sw_ring = txq->sw_ring;
657 tx_id = txq->tx_tail;
658 txe = &sw_ring[tx_id];
661 /* Determine if the descriptor ring needs to be cleaned. */
662 if (txq->nb_tx_free < txq->tx_free_thresh)
663 ixgbe_xmit_cleanup(txq);
665 rte_prefetch0(&txe->mbuf->pool);
668 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
671 pkt_len = tx_pkt->pkt_len;
674 * Determine how many (if any) context descriptors
675 * are needed for offload functionality.
677 ol_flags = tx_pkt->ol_flags;
678 #ifdef RTE_LIB_SECURITY
679 use_ipsec = txq->using_ipsec && (ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD);
682 /* If hardware offload required */
683 tx_ol_req = ol_flags & IXGBE_TX_OFFLOAD_MASK;
685 tx_offload.l2_len = tx_pkt->l2_len;
686 tx_offload.l3_len = tx_pkt->l3_len;
687 tx_offload.l4_len = tx_pkt->l4_len;
688 tx_offload.vlan_tci = tx_pkt->vlan_tci;
689 tx_offload.tso_segsz = tx_pkt->tso_segsz;
690 tx_offload.outer_l2_len = tx_pkt->outer_l2_len;
691 tx_offload.outer_l3_len = tx_pkt->outer_l3_len;
692 #ifdef RTE_LIB_SECURITY
694 union ixgbe_crypto_tx_desc_md *ipsec_mdata =
695 (union ixgbe_crypto_tx_desc_md *)
696 rte_security_dynfield(tx_pkt);
697 tx_offload.sa_idx = ipsec_mdata->sa_idx;
698 tx_offload.sec_pad_len = ipsec_mdata->pad_len;
702 /* If new context need be built or reuse the exist ctx. */
703 ctx = what_advctx_update(txq, tx_ol_req,
705 /* Only allocate context descriptor if required*/
706 new_ctx = (ctx == IXGBE_CTX_NUM);
711 * Keep track of how many descriptors are used this loop
712 * This will always be the number of segments + the number of
713 * Context descriptors required to transmit the packet
715 nb_used = (uint16_t)(tx_pkt->nb_segs + new_ctx);
718 nb_used + txq->nb_tx_used >= txq->tx_rs_thresh)
719 /* set RS on the previous packet in the burst */
720 txp->read.cmd_type_len |=
721 rte_cpu_to_le_32(IXGBE_TXD_CMD_RS);
724 * The number of descriptors that must be allocated for a
725 * packet is the number of segments of that packet, plus 1
726 * Context Descriptor for the hardware offload, if any.
727 * Determine the last TX descriptor to allocate in the TX ring
728 * for the packet, starting from the current position (tx_id)
731 tx_last = (uint16_t) (tx_id + nb_used - 1);
734 if (tx_last >= txq->nb_tx_desc)
735 tx_last = (uint16_t) (tx_last - txq->nb_tx_desc);
737 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u pktlen=%u"
738 " tx_first=%u tx_last=%u",
739 (unsigned) txq->port_id,
740 (unsigned) txq->queue_id,
746 * Make sure there are enough TX descriptors available to
747 * transmit the entire packet.
748 * nb_used better be less than or equal to txq->tx_rs_thresh
750 if (nb_used > txq->nb_tx_free) {
752 "Not enough free TX descriptors "
753 "nb_used=%4u nb_free=%4u "
754 "(port=%d queue=%d)",
755 nb_used, txq->nb_tx_free,
756 txq->port_id, txq->queue_id);
758 if (ixgbe_xmit_cleanup(txq) != 0) {
759 /* Could not clean any descriptors */
765 /* nb_used better be <= txq->tx_rs_thresh */
766 if (unlikely(nb_used > txq->tx_rs_thresh)) {
768 "The number of descriptors needed to "
769 "transmit the packet exceeds the "
770 "RS bit threshold. This will impact "
772 "nb_used=%4u nb_free=%4u "
774 "(port=%d queue=%d)",
775 nb_used, txq->nb_tx_free,
777 txq->port_id, txq->queue_id);
779 * Loop here until there are enough TX
780 * descriptors or until the ring cannot be
783 while (nb_used > txq->nb_tx_free) {
784 if (ixgbe_xmit_cleanup(txq) != 0) {
786 * Could not clean any
798 * By now there are enough free TX descriptors to transmit
803 * Set common flags of all TX Data Descriptors.
805 * The following bits must be set in all Data Descriptors:
806 * - IXGBE_ADVTXD_DTYP_DATA
807 * - IXGBE_ADVTXD_DCMD_DEXT
809 * The following bits must be set in the first Data Descriptor
810 * and are ignored in the other ones:
811 * - IXGBE_ADVTXD_DCMD_IFCS
812 * - IXGBE_ADVTXD_MAC_1588
813 * - IXGBE_ADVTXD_DCMD_VLE
815 * The following bits must only be set in the last Data
817 * - IXGBE_TXD_CMD_EOP
819 * The following bits can be set in any Data Descriptor, but
820 * are only set in the last Data Descriptor:
823 cmd_type_len = IXGBE_ADVTXD_DTYP_DATA |
824 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT;
826 #ifdef RTE_LIBRTE_IEEE1588
827 if (ol_flags & RTE_MBUF_F_TX_IEEE1588_TMST)
828 cmd_type_len |= IXGBE_ADVTXD_MAC_1588;
834 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {
835 /* when TSO is on, paylen in descriptor is the
836 * not the packet len but the tcp payload len */
837 pkt_len -= (tx_offload.l2_len +
838 tx_offload.l3_len + tx_offload.l4_len);
842 * Setup the TX Advanced Context Descriptor if required
845 volatile struct ixgbe_adv_tx_context_desc *
848 ctx_txd = (volatile struct
849 ixgbe_adv_tx_context_desc *)
852 txn = &sw_ring[txe->next_id];
853 rte_prefetch0(&txn->mbuf->pool);
855 if (txe->mbuf != NULL) {
856 rte_pktmbuf_free_seg(txe->mbuf);
860 ixgbe_set_xmit_ctx(txq, ctx_txd, tx_ol_req,
862 rte_security_dynfield(tx_pkt));
864 txe->last_id = tx_last;
865 tx_id = txe->next_id;
870 * Setup the TX Advanced Data Descriptor,
871 * This path will go through
872 * whatever new/reuse the context descriptor
874 cmd_type_len |= tx_desc_ol_flags_to_cmdtype(ol_flags);
875 olinfo_status |= tx_desc_cksum_flags_to_olinfo(ol_flags);
876 olinfo_status |= ctx << IXGBE_ADVTXD_IDX_SHIFT;
879 olinfo_status |= (pkt_len << IXGBE_ADVTXD_PAYLEN_SHIFT);
880 #ifdef RTE_LIB_SECURITY
882 olinfo_status |= IXGBE_ADVTXD_POPTS_IPSEC;
888 txn = &sw_ring[txe->next_id];
889 rte_prefetch0(&txn->mbuf->pool);
891 if (txe->mbuf != NULL)
892 rte_pktmbuf_free_seg(txe->mbuf);
896 * Set up Transmit Data Descriptor.
898 slen = m_seg->data_len;
899 buf_dma_addr = rte_mbuf_data_iova(m_seg);
900 txd->read.buffer_addr =
901 rte_cpu_to_le_64(buf_dma_addr);
902 txd->read.cmd_type_len =
903 rte_cpu_to_le_32(cmd_type_len | slen);
904 txd->read.olinfo_status =
905 rte_cpu_to_le_32(olinfo_status);
906 txe->last_id = tx_last;
907 tx_id = txe->next_id;
910 } while (m_seg != NULL);
913 * The last packet data descriptor needs End Of Packet (EOP)
915 cmd_type_len |= IXGBE_TXD_CMD_EOP;
916 txq->nb_tx_used = (uint16_t)(txq->nb_tx_used + nb_used);
917 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_used);
919 /* Set RS bit only on threshold packets' last descriptor */
920 if (txq->nb_tx_used >= txq->tx_rs_thresh) {
922 "Setting RS bit on TXD id="
923 "%4u (port=%d queue=%d)",
924 tx_last, txq->port_id, txq->queue_id);
926 cmd_type_len |= IXGBE_TXD_CMD_RS;
928 /* Update txq RS bit counters */
934 txd->read.cmd_type_len |= rte_cpu_to_le_32(cmd_type_len);
938 /* set RS on last packet in the burst */
940 txp->read.cmd_type_len |= rte_cpu_to_le_32(IXGBE_TXD_CMD_RS);
945 * Set the Transmit Descriptor Tail (TDT)
947 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u",
948 (unsigned) txq->port_id, (unsigned) txq->queue_id,
949 (unsigned) tx_id, (unsigned) nb_tx);
950 IXGBE_PCI_REG_WC_WRITE_RELAXED(txq->tdt_reg_addr, tx_id);
951 txq->tx_tail = tx_id;
956 /*********************************************************************
960 **********************************************************************/
962 ixgbe_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
967 struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue;
969 for (i = 0; i < nb_pkts; i++) {
971 ol_flags = m->ol_flags;
974 * Check if packet meets requirements for number of segments
976 * NOTE: for ixgbe it's always (40 - WTHRESH) for both TSO and
980 if (m->nb_segs > IXGBE_TX_MAX_SEG - txq->wthresh) {
985 if (ol_flags & IXGBE_TX_OFFLOAD_NOTSUP_MASK) {
990 /* check the size of packet */
991 if (m->pkt_len < IXGBE_TX_MIN_PKT_LEN) {
996 #ifdef RTE_ETHDEV_DEBUG_TX
997 ret = rte_validate_tx_offload(m);
1003 ret = rte_net_intel_cksum_prepare(m);
1013 /*********************************************************************
1017 **********************************************************************/
1019 #define IXGBE_PACKET_TYPE_ETHER 0X00
1020 #define IXGBE_PACKET_TYPE_IPV4 0X01
1021 #define IXGBE_PACKET_TYPE_IPV4_TCP 0X11
1022 #define IXGBE_PACKET_TYPE_IPV4_UDP 0X21
1023 #define IXGBE_PACKET_TYPE_IPV4_SCTP 0X41
1024 #define IXGBE_PACKET_TYPE_IPV4_EXT 0X03
1025 #define IXGBE_PACKET_TYPE_IPV4_EXT_TCP 0X13
1026 #define IXGBE_PACKET_TYPE_IPV4_EXT_UDP 0X23
1027 #define IXGBE_PACKET_TYPE_IPV4_EXT_SCTP 0X43
1028 #define IXGBE_PACKET_TYPE_IPV6 0X04
1029 #define IXGBE_PACKET_TYPE_IPV6_TCP 0X14
1030 #define IXGBE_PACKET_TYPE_IPV6_UDP 0X24
1031 #define IXGBE_PACKET_TYPE_IPV6_SCTP 0X44
1032 #define IXGBE_PACKET_TYPE_IPV6_EXT 0X0C
1033 #define IXGBE_PACKET_TYPE_IPV6_EXT_TCP 0X1C
1034 #define IXGBE_PACKET_TYPE_IPV6_EXT_UDP 0X2C
1035 #define IXGBE_PACKET_TYPE_IPV6_EXT_SCTP 0X4C
1036 #define IXGBE_PACKET_TYPE_IPV4_IPV6 0X05
1037 #define IXGBE_PACKET_TYPE_IPV4_IPV6_TCP 0X15
1038 #define IXGBE_PACKET_TYPE_IPV4_IPV6_UDP 0X25
1039 #define IXGBE_PACKET_TYPE_IPV4_IPV6_SCTP 0X45
1040 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6 0X07
1041 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_TCP 0X17
1042 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_UDP 0X27
1043 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_SCTP 0X47
1044 #define IXGBE_PACKET_TYPE_IPV4_IPV6_EXT 0X0D
1045 #define IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_TCP 0X1D
1046 #define IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_UDP 0X2D
1047 #define IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_SCTP 0X4D
1048 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT 0X0F
1049 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_TCP 0X1F
1050 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_UDP 0X2F
1051 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_SCTP 0X4F
1053 #define IXGBE_PACKET_TYPE_NVGRE 0X00
1054 #define IXGBE_PACKET_TYPE_NVGRE_IPV4 0X01
1055 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_TCP 0X11
1056 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_UDP 0X21
1057 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_SCTP 0X41
1058 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT 0X03
1059 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_TCP 0X13
1060 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_UDP 0X23
1061 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_SCTP 0X43
1062 #define IXGBE_PACKET_TYPE_NVGRE_IPV6 0X04
1063 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_TCP 0X14
1064 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_UDP 0X24
1065 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_SCTP 0X44
1066 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT 0X0C
1067 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_TCP 0X1C
1068 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_UDP 0X2C
1069 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_SCTP 0X4C
1070 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6 0X05
1071 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_TCP 0X15
1072 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_UDP 0X25
1073 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT 0X0D
1074 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_TCP 0X1D
1075 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_UDP 0X2D
1077 #define IXGBE_PACKET_TYPE_VXLAN 0X80
1078 #define IXGBE_PACKET_TYPE_VXLAN_IPV4 0X81
1079 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_TCP 0x91
1080 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_UDP 0xA1
1081 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_SCTP 0xC1
1082 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT 0x83
1083 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_TCP 0X93
1084 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_UDP 0XA3
1085 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_SCTP 0XC3
1086 #define IXGBE_PACKET_TYPE_VXLAN_IPV6 0X84
1087 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_TCP 0X94
1088 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_UDP 0XA4
1089 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_SCTP 0XC4
1090 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT 0X8C
1091 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_TCP 0X9C
1092 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_UDP 0XAC
1093 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_SCTP 0XCC
1094 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6 0X85
1095 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_TCP 0X95
1096 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_UDP 0XA5
1097 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT 0X8D
1098 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_TCP 0X9D
1099 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_UDP 0XAD
1102 * Use 2 different table for normal packet and tunnel packet
1103 * to save the space.
1106 ptype_table[IXGBE_PACKET_TYPE_MAX] __rte_cache_aligned = {
1107 [IXGBE_PACKET_TYPE_ETHER] = RTE_PTYPE_L2_ETHER,
1108 [IXGBE_PACKET_TYPE_IPV4] = RTE_PTYPE_L2_ETHER |
1110 [IXGBE_PACKET_TYPE_IPV4_TCP] = RTE_PTYPE_L2_ETHER |
1111 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP,
1112 [IXGBE_PACKET_TYPE_IPV4_UDP] = RTE_PTYPE_L2_ETHER |
1113 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP,
1114 [IXGBE_PACKET_TYPE_IPV4_SCTP] = RTE_PTYPE_L2_ETHER |
1115 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_SCTP,
1116 [IXGBE_PACKET_TYPE_IPV4_EXT] = RTE_PTYPE_L2_ETHER |
1117 RTE_PTYPE_L3_IPV4_EXT,
1118 [IXGBE_PACKET_TYPE_IPV4_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1119 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_TCP,
1120 [IXGBE_PACKET_TYPE_IPV4_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1121 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_UDP,
1122 [IXGBE_PACKET_TYPE_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1123 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_SCTP,
1124 [IXGBE_PACKET_TYPE_IPV6] = RTE_PTYPE_L2_ETHER |
1126 [IXGBE_PACKET_TYPE_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1127 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP,
1128 [IXGBE_PACKET_TYPE_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1129 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP,
1130 [IXGBE_PACKET_TYPE_IPV6_SCTP] = RTE_PTYPE_L2_ETHER |
1131 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_SCTP,
1132 [IXGBE_PACKET_TYPE_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1133 RTE_PTYPE_L3_IPV6_EXT,
1134 [IXGBE_PACKET_TYPE_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1135 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_TCP,
1136 [IXGBE_PACKET_TYPE_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1137 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_UDP,
1138 [IXGBE_PACKET_TYPE_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1139 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_SCTP,
1140 [IXGBE_PACKET_TYPE_IPV4_IPV6] = RTE_PTYPE_L2_ETHER |
1141 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1142 RTE_PTYPE_INNER_L3_IPV6,
1143 [IXGBE_PACKET_TYPE_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1144 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1145 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP,
1146 [IXGBE_PACKET_TYPE_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1147 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1148 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP,
1149 [IXGBE_PACKET_TYPE_IPV4_IPV6_SCTP] = RTE_PTYPE_L2_ETHER |
1150 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1151 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_SCTP,
1152 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6] = RTE_PTYPE_L2_ETHER |
1153 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1154 RTE_PTYPE_INNER_L3_IPV6,
1155 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1156 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1157 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP,
1158 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1159 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1160 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP,
1161 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_SCTP] = RTE_PTYPE_L2_ETHER |
1162 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1163 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_SCTP,
1164 [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1165 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1166 RTE_PTYPE_INNER_L3_IPV6_EXT,
1167 [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1168 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1169 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP,
1170 [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1171 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1172 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP,
1173 [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1174 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1175 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_SCTP,
1176 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1177 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1178 RTE_PTYPE_INNER_L3_IPV6_EXT,
1179 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1180 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1181 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP,
1182 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1183 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1184 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP,
1185 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_SCTP] =
1186 RTE_PTYPE_L2_ETHER |
1187 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1188 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_SCTP,
1192 ptype_table_tn[IXGBE_PACKET_TYPE_TN_MAX] __rte_cache_aligned = {
1193 [IXGBE_PACKET_TYPE_NVGRE] = RTE_PTYPE_L2_ETHER |
1194 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1195 RTE_PTYPE_INNER_L2_ETHER,
1196 [IXGBE_PACKET_TYPE_NVGRE_IPV4] = RTE_PTYPE_L2_ETHER |
1197 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1198 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1199 [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT] = RTE_PTYPE_L2_ETHER |
1200 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1201 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT,
1202 [IXGBE_PACKET_TYPE_NVGRE_IPV6] = RTE_PTYPE_L2_ETHER |
1203 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1204 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6,
1205 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6] = RTE_PTYPE_L2_ETHER |
1206 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1207 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1208 [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1209 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1210 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT,
1211 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1212 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1213 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1214 [IXGBE_PACKET_TYPE_NVGRE_IPV4_TCP] = RTE_PTYPE_L2_ETHER |
1215 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1216 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4 |
1217 RTE_PTYPE_INNER_L4_TCP,
1218 [IXGBE_PACKET_TYPE_NVGRE_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1219 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1220 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6 |
1221 RTE_PTYPE_INNER_L4_TCP,
1222 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1223 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1224 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1225 [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1226 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1227 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT |
1228 RTE_PTYPE_INNER_L4_TCP,
1229 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_TCP] =
1230 RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
1231 RTE_PTYPE_TUNNEL_GRE | RTE_PTYPE_INNER_L2_ETHER |
1232 RTE_PTYPE_INNER_L3_IPV4,
1233 [IXGBE_PACKET_TYPE_NVGRE_IPV4_UDP] = RTE_PTYPE_L2_ETHER |
1234 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1235 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4 |
1236 RTE_PTYPE_INNER_L4_UDP,
1237 [IXGBE_PACKET_TYPE_NVGRE_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1238 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1239 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6 |
1240 RTE_PTYPE_INNER_L4_UDP,
1241 [IXGBE_PACKET_TYPE_NVGRE_IPV6_SCTP] = RTE_PTYPE_L2_ETHER |
1242 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1243 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6 |
1244 RTE_PTYPE_INNER_L4_SCTP,
1245 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1246 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1247 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1248 [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1249 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1250 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT |
1251 RTE_PTYPE_INNER_L4_UDP,
1252 [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1253 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1254 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT |
1255 RTE_PTYPE_INNER_L4_SCTP,
1256 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_UDP] =
1257 RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
1258 RTE_PTYPE_TUNNEL_GRE | RTE_PTYPE_INNER_L2_ETHER |
1259 RTE_PTYPE_INNER_L3_IPV4,
1260 [IXGBE_PACKET_TYPE_NVGRE_IPV4_SCTP] = RTE_PTYPE_L2_ETHER |
1261 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1262 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4 |
1263 RTE_PTYPE_INNER_L4_SCTP,
1264 [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1265 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1266 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT |
1267 RTE_PTYPE_INNER_L4_SCTP,
1268 [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1269 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1270 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT |
1271 RTE_PTYPE_INNER_L4_TCP,
1272 [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1273 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1274 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT |
1275 RTE_PTYPE_INNER_L4_UDP,
1277 [IXGBE_PACKET_TYPE_VXLAN] = RTE_PTYPE_L2_ETHER |
1278 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1279 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER,
1280 [IXGBE_PACKET_TYPE_VXLAN_IPV4] = RTE_PTYPE_L2_ETHER |
1281 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1282 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1283 RTE_PTYPE_INNER_L3_IPV4,
1284 [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT] = RTE_PTYPE_L2_ETHER |
1285 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1286 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1287 RTE_PTYPE_INNER_L3_IPV4_EXT,
1288 [IXGBE_PACKET_TYPE_VXLAN_IPV6] = RTE_PTYPE_L2_ETHER |
1289 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1290 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1291 RTE_PTYPE_INNER_L3_IPV6,
1292 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6] = RTE_PTYPE_L2_ETHER |
1293 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1294 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1295 RTE_PTYPE_INNER_L3_IPV4,
1296 [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1297 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1298 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1299 RTE_PTYPE_INNER_L3_IPV6_EXT,
1300 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1301 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1302 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1303 RTE_PTYPE_INNER_L3_IPV4,
1304 [IXGBE_PACKET_TYPE_VXLAN_IPV4_TCP] = RTE_PTYPE_L2_ETHER |
1305 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1306 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1307 RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L4_TCP,
1308 [IXGBE_PACKET_TYPE_VXLAN_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1309 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1310 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1311 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP,
1312 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1313 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1314 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1315 RTE_PTYPE_INNER_L3_IPV4,
1316 [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1317 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1318 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1319 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP,
1320 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_TCP] =
1321 RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
1322 RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_VXLAN |
1323 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1324 [IXGBE_PACKET_TYPE_VXLAN_IPV4_UDP] = RTE_PTYPE_L2_ETHER |
1325 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1326 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1327 RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L4_UDP,
1328 [IXGBE_PACKET_TYPE_VXLAN_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1329 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1330 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1331 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP,
1332 [IXGBE_PACKET_TYPE_VXLAN_IPV6_SCTP] = RTE_PTYPE_L2_ETHER |
1333 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1334 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1335 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_SCTP,
1336 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1337 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1338 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1339 RTE_PTYPE_INNER_L3_IPV4,
1340 [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1341 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1342 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1343 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP,
1344 [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1345 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1346 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1347 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_SCTP,
1348 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_UDP] =
1349 RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
1350 RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_VXLAN |
1351 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1352 [IXGBE_PACKET_TYPE_VXLAN_IPV4_SCTP] = RTE_PTYPE_L2_ETHER |
1353 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1354 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1355 RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L4_SCTP,
1356 [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1357 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1358 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1359 RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_SCTP,
1360 [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1361 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1362 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1363 RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_TCP,
1364 [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1365 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1366 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1367 RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_UDP,
1371 ixgbe_monitor_callback(const uint64_t value,
1372 const uint64_t arg[RTE_POWER_MONITOR_OPAQUE_SZ] __rte_unused)
1374 const uint64_t m = rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD);
1376 * we expect the DD bit to be set to 1 if this descriptor was already
1379 return (value & m) == m ? -1 : 0;
1383 ixgbe_get_monitor_addr(void *rx_queue, struct rte_power_monitor_cond *pmc)
1385 volatile union ixgbe_adv_rx_desc *rxdp;
1386 struct ixgbe_rx_queue *rxq = rx_queue;
1389 desc = rxq->rx_tail;
1390 rxdp = &rxq->rx_ring[desc];
1391 /* watch for changes in status bit */
1392 pmc->addr = &rxdp->wb.upper.status_error;
1394 /* comparison callback */
1395 pmc->fn = ixgbe_monitor_callback;
1397 /* the registers are 32-bit */
1398 pmc->size = sizeof(uint32_t);
1403 /* @note: fix ixgbe_dev_supported_ptypes_get() if any change here. */
1404 static inline uint32_t
1405 ixgbe_rxd_pkt_info_to_pkt_type(uint32_t pkt_info, uint16_t ptype_mask)
1408 if (unlikely(pkt_info & IXGBE_RXDADV_PKTTYPE_ETQF))
1409 return RTE_PTYPE_UNKNOWN;
1411 pkt_info = (pkt_info >> IXGBE_PACKET_TYPE_SHIFT) & ptype_mask;
1413 /* For tunnel packet */
1414 if (pkt_info & IXGBE_PACKET_TYPE_TUNNEL_BIT) {
1415 /* Remove the tunnel bit to save the space. */
1416 pkt_info &= IXGBE_PACKET_TYPE_MASK_TUNNEL;
1417 return ptype_table_tn[pkt_info];
1421 * For x550, if it's not tunnel,
1422 * tunnel type bit should be set to 0.
1423 * Reuse 82599's mask.
1425 pkt_info &= IXGBE_PACKET_TYPE_MASK_82599;
1427 return ptype_table[pkt_info];
1430 static inline uint64_t
1431 ixgbe_rxd_pkt_info_to_pkt_flags(uint16_t pkt_info)
1433 static uint64_t ip_rss_types_map[16] __rte_cache_aligned = {
1434 0, RTE_MBUF_F_RX_RSS_HASH, RTE_MBUF_F_RX_RSS_HASH, RTE_MBUF_F_RX_RSS_HASH,
1435 0, RTE_MBUF_F_RX_RSS_HASH, 0, RTE_MBUF_F_RX_RSS_HASH,
1436 RTE_MBUF_F_RX_RSS_HASH, 0, 0, 0,
1437 0, 0, 0, RTE_MBUF_F_RX_FDIR,
1439 #ifdef RTE_LIBRTE_IEEE1588
1440 static uint64_t ip_pkt_etqf_map[8] = {
1441 0, 0, 0, RTE_MBUF_F_RX_IEEE1588_PTP,
1445 if (likely(pkt_info & IXGBE_RXDADV_PKTTYPE_ETQF))
1446 return ip_pkt_etqf_map[(pkt_info >> 4) & 0X07] |
1447 ip_rss_types_map[pkt_info & 0XF];
1449 return ip_rss_types_map[pkt_info & 0XF];
1451 return ip_rss_types_map[pkt_info & 0XF];
1455 static inline uint64_t
1456 rx_desc_status_to_pkt_flags(uint32_t rx_status, uint64_t vlan_flags)
1461 * Check if VLAN present only.
1462 * Do not check whether L3/L4 rx checksum done by NIC or not,
1463 * That can be found from rte_eth_rxmode.offloads flag
1465 pkt_flags = (rx_status & IXGBE_RXD_STAT_VP) ? vlan_flags : 0;
1467 #ifdef RTE_LIBRTE_IEEE1588
1468 if (rx_status & IXGBE_RXD_STAT_TMST)
1469 pkt_flags = pkt_flags | RTE_MBUF_F_RX_IEEE1588_TMST;
1474 static inline uint64_t
1475 rx_desc_error_to_pkt_flags(uint32_t rx_status, uint16_t pkt_info,
1476 uint8_t rx_udp_csum_zero_err)
1481 * Bit 31: IPE, IPv4 checksum error
1482 * Bit 30: L4I, L4I integrity error
1484 static uint64_t error_to_pkt_flags_map[4] = {
1485 RTE_MBUF_F_RX_IP_CKSUM_GOOD | RTE_MBUF_F_RX_L4_CKSUM_GOOD,
1486 RTE_MBUF_F_RX_IP_CKSUM_GOOD | RTE_MBUF_F_RX_L4_CKSUM_BAD,
1487 RTE_MBUF_F_RX_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_GOOD,
1488 RTE_MBUF_F_RX_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD
1490 pkt_flags = error_to_pkt_flags_map[(rx_status >>
1491 IXGBE_RXDADV_ERR_CKSUM_BIT) & IXGBE_RXDADV_ERR_CKSUM_MSK];
1493 /* Mask out the bad UDP checksum error if the hardware has UDP zero
1494 * checksum error issue, so that the software application will then
1495 * have to recompute the checksum itself if needed.
1497 if ((rx_status & IXGBE_RXDADV_ERR_TCPE) &&
1498 (pkt_info & IXGBE_RXDADV_PKTTYPE_UDP) &&
1499 rx_udp_csum_zero_err)
1500 pkt_flags &= ~RTE_MBUF_F_RX_L4_CKSUM_BAD;
1502 if ((rx_status & IXGBE_RXD_STAT_OUTERIPCS) &&
1503 (rx_status & IXGBE_RXDADV_ERR_OUTERIPER)) {
1504 pkt_flags |= RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD;
1507 #ifdef RTE_LIB_SECURITY
1508 if (rx_status & IXGBE_RXD_STAT_SECP) {
1509 pkt_flags |= RTE_MBUF_F_RX_SEC_OFFLOAD;
1510 if (rx_status & IXGBE_RXDADV_LNKSEC_ERROR_BAD_SIG)
1511 pkt_flags |= RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED;
1519 * LOOK_AHEAD defines how many desc statuses to check beyond the
1520 * current descriptor.
1521 * It must be a pound define for optimal performance.
1522 * Do not change the value of LOOK_AHEAD, as the ixgbe_rx_scan_hw_ring
1523 * function only works with LOOK_AHEAD=8.
1525 #define LOOK_AHEAD 8
1526 #if (LOOK_AHEAD != 8)
1527 #error "PMD IXGBE: LOOK_AHEAD must be 8\n"
1530 ixgbe_rx_scan_hw_ring(struct ixgbe_rx_queue *rxq)
1532 volatile union ixgbe_adv_rx_desc *rxdp;
1533 struct ixgbe_rx_entry *rxep;
1534 struct rte_mbuf *mb;
1538 uint32_t s[LOOK_AHEAD];
1539 uint32_t pkt_info[LOOK_AHEAD];
1540 int i, j, nb_rx = 0;
1542 uint64_t vlan_flags = rxq->vlan_flags;
1544 /* get references to current descriptor and S/W ring entry */
1545 rxdp = &rxq->rx_ring[rxq->rx_tail];
1546 rxep = &rxq->sw_ring[rxq->rx_tail];
1548 status = rxdp->wb.upper.status_error;
1549 /* check to make sure there is at least 1 packet to receive */
1550 if (!(status & rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD)))
1554 * Scan LOOK_AHEAD descriptors at a time to determine which descriptors
1555 * reference packets that are ready to be received.
1557 for (i = 0; i < RTE_PMD_IXGBE_RX_MAX_BURST;
1558 i += LOOK_AHEAD, rxdp += LOOK_AHEAD, rxep += LOOK_AHEAD) {
1559 /* Read desc statuses backwards to avoid race condition */
1560 for (j = 0; j < LOOK_AHEAD; j++)
1561 s[j] = rte_le_to_cpu_32(rxdp[j].wb.upper.status_error);
1565 /* Compute how many status bits were set */
1566 for (nb_dd = 0; nb_dd < LOOK_AHEAD &&
1567 (s[nb_dd] & IXGBE_RXDADV_STAT_DD); nb_dd++)
1570 for (j = 0; j < nb_dd; j++)
1571 pkt_info[j] = rte_le_to_cpu_32(rxdp[j].wb.lower.
1576 /* Translate descriptor info to mbuf format */
1577 for (j = 0; j < nb_dd; ++j) {
1579 pkt_len = rte_le_to_cpu_16(rxdp[j].wb.upper.length) -
1581 mb->data_len = pkt_len;
1582 mb->pkt_len = pkt_len;
1583 mb->vlan_tci = rte_le_to_cpu_16(rxdp[j].wb.upper.vlan);
1585 /* convert descriptor fields to rte mbuf flags */
1586 pkt_flags = rx_desc_status_to_pkt_flags(s[j],
1588 pkt_flags |= rx_desc_error_to_pkt_flags(s[j],
1589 (uint16_t)pkt_info[j],
1590 rxq->rx_udp_csum_zero_err);
1591 pkt_flags |= ixgbe_rxd_pkt_info_to_pkt_flags
1592 ((uint16_t)pkt_info[j]);
1593 mb->ol_flags = pkt_flags;
1595 ixgbe_rxd_pkt_info_to_pkt_type
1596 (pkt_info[j], rxq->pkt_type_mask);
1598 if (likely(pkt_flags & RTE_MBUF_F_RX_RSS_HASH))
1599 mb->hash.rss = rte_le_to_cpu_32(
1600 rxdp[j].wb.lower.hi_dword.rss);
1601 else if (pkt_flags & RTE_MBUF_F_RX_FDIR) {
1602 mb->hash.fdir.hash = rte_le_to_cpu_16(
1603 rxdp[j].wb.lower.hi_dword.csum_ip.csum) &
1604 IXGBE_ATR_HASH_MASK;
1605 mb->hash.fdir.id = rte_le_to_cpu_16(
1606 rxdp[j].wb.lower.hi_dword.csum_ip.ip_id);
1610 /* Move mbuf pointers from the S/W ring to the stage */
1611 for (j = 0; j < LOOK_AHEAD; ++j) {
1612 rxq->rx_stage[i + j] = rxep[j].mbuf;
1615 /* stop if all requested packets could not be received */
1616 if (nb_dd != LOOK_AHEAD)
1620 /* clear software ring entries so we can cleanup correctly */
1621 for (i = 0; i < nb_rx; ++i) {
1622 rxq->sw_ring[rxq->rx_tail + i].mbuf = NULL;
1630 ixgbe_rx_alloc_bufs(struct ixgbe_rx_queue *rxq, bool reset_mbuf)
1632 volatile union ixgbe_adv_rx_desc *rxdp;
1633 struct ixgbe_rx_entry *rxep;
1634 struct rte_mbuf *mb;
1639 /* allocate buffers in bulk directly into the S/W ring */
1640 alloc_idx = rxq->rx_free_trigger - (rxq->rx_free_thresh - 1);
1641 rxep = &rxq->sw_ring[alloc_idx];
1642 diag = rte_mempool_get_bulk(rxq->mb_pool, (void *)rxep,
1643 rxq->rx_free_thresh);
1644 if (unlikely(diag != 0))
1647 rxdp = &rxq->rx_ring[alloc_idx];
1648 for (i = 0; i < rxq->rx_free_thresh; ++i) {
1649 /* populate the static rte mbuf fields */
1652 mb->port = rxq->port_id;
1655 rte_mbuf_refcnt_set(mb, 1);
1656 mb->data_off = RTE_PKTMBUF_HEADROOM;
1658 /* populate the descriptors */
1659 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mb));
1660 rxdp[i].read.hdr_addr = 0;
1661 rxdp[i].read.pkt_addr = dma_addr;
1664 /* update state of internal queue structure */
1665 rxq->rx_free_trigger = rxq->rx_free_trigger + rxq->rx_free_thresh;
1666 if (rxq->rx_free_trigger >= rxq->nb_rx_desc)
1667 rxq->rx_free_trigger = rxq->rx_free_thresh - 1;
1673 static inline uint16_t
1674 ixgbe_rx_fill_from_stage(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
1677 struct rte_mbuf **stage = &rxq->rx_stage[rxq->rx_next_avail];
1680 /* how many packets are ready to return? */
1681 nb_pkts = (uint16_t)RTE_MIN(nb_pkts, rxq->rx_nb_avail);
1683 /* copy mbuf pointers to the application's packet list */
1684 for (i = 0; i < nb_pkts; ++i)
1685 rx_pkts[i] = stage[i];
1687 /* update internal queue state */
1688 rxq->rx_nb_avail = (uint16_t)(rxq->rx_nb_avail - nb_pkts);
1689 rxq->rx_next_avail = (uint16_t)(rxq->rx_next_avail + nb_pkts);
1694 static inline uint16_t
1695 rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
1698 struct ixgbe_rx_queue *rxq = (struct ixgbe_rx_queue *)rx_queue;
1701 /* Any previously recv'd pkts will be returned from the Rx stage */
1702 if (rxq->rx_nb_avail)
1703 return ixgbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
1705 /* Scan the H/W ring for packets to receive */
1706 nb_rx = (uint16_t)ixgbe_rx_scan_hw_ring(rxq);
1708 /* update internal queue state */
1709 rxq->rx_next_avail = 0;
1710 rxq->rx_nb_avail = nb_rx;
1711 rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_rx);
1713 /* if required, allocate new buffers to replenish descriptors */
1714 if (rxq->rx_tail > rxq->rx_free_trigger) {
1715 uint16_t cur_free_trigger = rxq->rx_free_trigger;
1717 if (ixgbe_rx_alloc_bufs(rxq, true) != 0) {
1720 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u "
1721 "queue_id=%u", (unsigned) rxq->port_id,
1722 (unsigned) rxq->queue_id);
1724 rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed +=
1725 rxq->rx_free_thresh;
1728 * Need to rewind any previous receives if we cannot
1729 * allocate new buffers to replenish the old ones.
1731 rxq->rx_nb_avail = 0;
1732 rxq->rx_tail = (uint16_t)(rxq->rx_tail - nb_rx);
1733 for (i = 0, j = rxq->rx_tail; i < nb_rx; ++i, ++j)
1734 rxq->sw_ring[j].mbuf = rxq->rx_stage[i];
1739 /* update tail pointer */
1741 IXGBE_PCI_REG_WC_WRITE_RELAXED(rxq->rdt_reg_addr,
1745 if (rxq->rx_tail >= rxq->nb_rx_desc)
1748 /* received any packets this loop? */
1749 if (rxq->rx_nb_avail)
1750 return ixgbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
1755 /* split requests into chunks of size RTE_PMD_IXGBE_RX_MAX_BURST */
1757 ixgbe_recv_pkts_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
1762 if (unlikely(nb_pkts == 0))
1765 if (likely(nb_pkts <= RTE_PMD_IXGBE_RX_MAX_BURST))
1766 return rx_recv_pkts(rx_queue, rx_pkts, nb_pkts);
1768 /* request is relatively large, chunk it up */
1773 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_IXGBE_RX_MAX_BURST);
1774 ret = rx_recv_pkts(rx_queue, &rx_pkts[nb_rx], n);
1775 nb_rx = (uint16_t)(nb_rx + ret);
1776 nb_pkts = (uint16_t)(nb_pkts - ret);
1785 ixgbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
1788 struct ixgbe_rx_queue *rxq;
1789 volatile union ixgbe_adv_rx_desc *rx_ring;
1790 volatile union ixgbe_adv_rx_desc *rxdp;
1791 struct ixgbe_rx_entry *sw_ring;
1792 struct ixgbe_rx_entry *rxe;
1793 struct rte_mbuf *rxm;
1794 struct rte_mbuf *nmb;
1795 union ixgbe_adv_rx_desc rxd;
1804 uint64_t vlan_flags;
1809 rx_id = rxq->rx_tail;
1810 rx_ring = rxq->rx_ring;
1811 sw_ring = rxq->sw_ring;
1812 vlan_flags = rxq->vlan_flags;
1813 while (nb_rx < nb_pkts) {
1815 * The order of operations here is important as the DD status
1816 * bit must not be read after any other descriptor fields.
1817 * rx_ring and rxdp are pointing to volatile data so the order
1818 * of accesses cannot be reordered by the compiler. If they were
1819 * not volatile, they could be reordered which could lead to
1820 * using invalid descriptor fields when read from rxd.
1822 rxdp = &rx_ring[rx_id];
1823 staterr = rxdp->wb.upper.status_error;
1824 if (!(staterr & rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD)))
1831 * If the IXGBE_RXDADV_STAT_EOP flag is not set, the RX packet
1832 * is likely to be invalid and to be dropped by the various
1833 * validation checks performed by the network stack.
1835 * Allocate a new mbuf to replenish the RX ring descriptor.
1836 * If the allocation fails:
1837 * - arrange for that RX descriptor to be the first one
1838 * being parsed the next time the receive function is
1839 * invoked [on the same queue].
1841 * - Stop parsing the RX ring and return immediately.
1843 * This policy do not drop the packet received in the RX
1844 * descriptor for which the allocation of a new mbuf failed.
1845 * Thus, it allows that packet to be later retrieved if
1846 * mbuf have been freed in the mean time.
1847 * As a side effect, holding RX descriptors instead of
1848 * systematically giving them back to the NIC may lead to
1849 * RX ring exhaustion situations.
1850 * However, the NIC can gracefully prevent such situations
1851 * to happen by sending specific "back-pressure" flow control
1852 * frames to its peer(s).
1854 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u "
1855 "ext_err_stat=0x%08x pkt_len=%u",
1856 (unsigned) rxq->port_id, (unsigned) rxq->queue_id,
1857 (unsigned) rx_id, (unsigned) staterr,
1858 (unsigned) rte_le_to_cpu_16(rxd.wb.upper.length));
1860 nmb = rte_mbuf_raw_alloc(rxq->mb_pool);
1862 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u "
1863 "queue_id=%u", (unsigned) rxq->port_id,
1864 (unsigned) rxq->queue_id);
1865 rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed++;
1870 rxe = &sw_ring[rx_id];
1872 if (rx_id == rxq->nb_rx_desc)
1875 /* Prefetch next mbuf while processing current one. */
1876 rte_ixgbe_prefetch(sw_ring[rx_id].mbuf);
1879 * When next RX descriptor is on a cache-line boundary,
1880 * prefetch the next 4 RX descriptors and the next 8 pointers
1883 if ((rx_id & 0x3) == 0) {
1884 rte_ixgbe_prefetch(&rx_ring[rx_id]);
1885 rte_ixgbe_prefetch(&sw_ring[rx_id]);
1891 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
1892 rxdp->read.hdr_addr = 0;
1893 rxdp->read.pkt_addr = dma_addr;
1896 * Initialize the returned mbuf.
1897 * 1) setup generic mbuf fields:
1898 * - number of segments,
1901 * - RX port identifier.
1902 * 2) integrate hardware offload data, if any:
1903 * - RSS flag & hash,
1904 * - IP checksum flag,
1905 * - VLAN TCI, if any,
1908 pkt_len = (uint16_t) (rte_le_to_cpu_16(rxd.wb.upper.length) -
1910 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1911 rte_packet_prefetch((char *)rxm->buf_addr + rxm->data_off);
1914 rxm->pkt_len = pkt_len;
1915 rxm->data_len = pkt_len;
1916 rxm->port = rxq->port_id;
1918 pkt_info = rte_le_to_cpu_32(rxd.wb.lower.lo_dword.data);
1919 /* Only valid if RTE_MBUF_F_RX_VLAN set in pkt_flags */
1920 rxm->vlan_tci = rte_le_to_cpu_16(rxd.wb.upper.vlan);
1922 pkt_flags = rx_desc_status_to_pkt_flags(staterr, vlan_flags);
1923 pkt_flags = pkt_flags |
1924 rx_desc_error_to_pkt_flags(staterr, (uint16_t)pkt_info,
1925 rxq->rx_udp_csum_zero_err);
1926 pkt_flags = pkt_flags |
1927 ixgbe_rxd_pkt_info_to_pkt_flags((uint16_t)pkt_info);
1928 rxm->ol_flags = pkt_flags;
1930 ixgbe_rxd_pkt_info_to_pkt_type(pkt_info,
1931 rxq->pkt_type_mask);
1933 if (likely(pkt_flags & RTE_MBUF_F_RX_RSS_HASH))
1934 rxm->hash.rss = rte_le_to_cpu_32(
1935 rxd.wb.lower.hi_dword.rss);
1936 else if (pkt_flags & RTE_MBUF_F_RX_FDIR) {
1937 rxm->hash.fdir.hash = rte_le_to_cpu_16(
1938 rxd.wb.lower.hi_dword.csum_ip.csum) &
1939 IXGBE_ATR_HASH_MASK;
1940 rxm->hash.fdir.id = rte_le_to_cpu_16(
1941 rxd.wb.lower.hi_dword.csum_ip.ip_id);
1944 * Store the mbuf address into the next entry of the array
1945 * of returned packets.
1947 rx_pkts[nb_rx++] = rxm;
1949 rxq->rx_tail = rx_id;
1952 * If the number of free RX descriptors is greater than the RX free
1953 * threshold of the queue, advance the Receive Descriptor Tail (RDT)
1955 * Update the RDT with the value of the last processed RX descriptor
1956 * minus 1, to guarantee that the RDT register is never equal to the
1957 * RDH register, which creates a "full" ring situation from the
1958 * hardware point of view...
1960 nb_hold = (uint16_t) (nb_hold + rxq->nb_rx_hold);
1961 if (nb_hold > rxq->rx_free_thresh) {
1962 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u "
1963 "nb_hold=%u nb_rx=%u",
1964 (unsigned) rxq->port_id, (unsigned) rxq->queue_id,
1965 (unsigned) rx_id, (unsigned) nb_hold,
1967 rx_id = (uint16_t) ((rx_id == 0) ?
1968 (rxq->nb_rx_desc - 1) : (rx_id - 1));
1969 IXGBE_PCI_REG_WC_WRITE(rxq->rdt_reg_addr, rx_id);
1972 rxq->nb_rx_hold = nb_hold;
1977 * Detect an RSC descriptor.
1979 static inline uint32_t
1980 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
1982 return (rte_le_to_cpu_32(rx->wb.lower.lo_dword.data) &
1983 IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
1987 * ixgbe_fill_cluster_head_buf - fill the first mbuf of the returned packet
1989 * Fill the following info in the HEAD buffer of the Rx cluster:
1990 * - RX port identifier
1991 * - hardware offload data, if any:
1993 * - IP checksum flag
1994 * - VLAN TCI, if any
1996 * @head HEAD of the packet cluster
1997 * @desc HW descriptor to get data from
1998 * @rxq Pointer to the Rx queue
2001 ixgbe_fill_cluster_head_buf(
2002 struct rte_mbuf *head,
2003 union ixgbe_adv_rx_desc *desc,
2004 struct ixgbe_rx_queue *rxq,
2010 head->port = rxq->port_id;
2012 /* The vlan_tci field is only valid when RTE_MBUF_F_RX_VLAN is
2013 * set in the pkt_flags field.
2015 head->vlan_tci = rte_le_to_cpu_16(desc->wb.upper.vlan);
2016 pkt_info = rte_le_to_cpu_32(desc->wb.lower.lo_dword.data);
2017 pkt_flags = rx_desc_status_to_pkt_flags(staterr, rxq->vlan_flags);
2018 pkt_flags |= rx_desc_error_to_pkt_flags(staterr, (uint16_t)pkt_info,
2019 rxq->rx_udp_csum_zero_err);
2020 pkt_flags |= ixgbe_rxd_pkt_info_to_pkt_flags((uint16_t)pkt_info);
2021 head->ol_flags = pkt_flags;
2023 ixgbe_rxd_pkt_info_to_pkt_type(pkt_info, rxq->pkt_type_mask);
2025 if (likely(pkt_flags & RTE_MBUF_F_RX_RSS_HASH))
2026 head->hash.rss = rte_le_to_cpu_32(desc->wb.lower.hi_dword.rss);
2027 else if (pkt_flags & RTE_MBUF_F_RX_FDIR) {
2028 head->hash.fdir.hash =
2029 rte_le_to_cpu_16(desc->wb.lower.hi_dword.csum_ip.csum)
2030 & IXGBE_ATR_HASH_MASK;
2031 head->hash.fdir.id =
2032 rte_le_to_cpu_16(desc->wb.lower.hi_dword.csum_ip.ip_id);
2037 * ixgbe_recv_pkts_lro - receive handler for and LRO case.
2039 * @rx_queue Rx queue handle
2040 * @rx_pkts table of received packets
2041 * @nb_pkts size of rx_pkts table
2042 * @bulk_alloc if TRUE bulk allocation is used for a HW ring refilling
2044 * Handles the Rx HW ring completions when RSC feature is configured. Uses an
2045 * additional ring of ixgbe_rsc_entry's that will hold the relevant RSC info.
2047 * We use the same logic as in Linux and in FreeBSD ixgbe drivers:
2048 * 1) When non-EOP RSC completion arrives:
2049 * a) Update the HEAD of the current RSC aggregation cluster with the new
2050 * segment's data length.
2051 * b) Set the "next" pointer of the current segment to point to the segment
2052 * at the NEXTP index.
2053 * c) Pass the HEAD of RSC aggregation cluster on to the next NEXTP entry
2054 * in the sw_rsc_ring.
2055 * 2) When EOP arrives we just update the cluster's total length and offload
2056 * flags and deliver the cluster up to the upper layers. In our case - put it
2057 * in the rx_pkts table.
2059 * Returns the number of received packets/clusters (according to the "bulk
2060 * receive" interface).
2062 static inline uint16_t
2063 ixgbe_recv_pkts_lro(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts,
2066 struct ixgbe_rx_queue *rxq = rx_queue;
2067 volatile union ixgbe_adv_rx_desc *rx_ring = rxq->rx_ring;
2068 struct ixgbe_rx_entry *sw_ring = rxq->sw_ring;
2069 struct ixgbe_scattered_rx_entry *sw_sc_ring = rxq->sw_sc_ring;
2070 uint16_t rx_id = rxq->rx_tail;
2072 uint16_t nb_hold = rxq->nb_rx_hold;
2073 uint16_t prev_id = rxq->rx_tail;
2075 while (nb_rx < nb_pkts) {
2077 struct ixgbe_rx_entry *rxe;
2078 struct ixgbe_scattered_rx_entry *sc_entry;
2079 struct ixgbe_scattered_rx_entry *next_sc_entry = NULL;
2080 struct ixgbe_rx_entry *next_rxe = NULL;
2081 struct rte_mbuf *first_seg;
2082 struct rte_mbuf *rxm;
2083 struct rte_mbuf *nmb = NULL;
2084 union ixgbe_adv_rx_desc rxd;
2087 volatile union ixgbe_adv_rx_desc *rxdp;
2092 * The code in this whole file uses the volatile pointer to
2093 * ensure the read ordering of the status and the rest of the
2094 * descriptor fields (on the compiler level only!!!). This is so
2095 * UGLY - why not to just use the compiler barrier instead? DPDK
2096 * even has the rte_compiler_barrier() for that.
2098 * But most importantly this is just wrong because this doesn't
2099 * ensure memory ordering in a general case at all. For
2100 * instance, DPDK is supposed to work on Power CPUs where
2101 * compiler barrier may just not be enough!
2103 * I tried to write only this function properly to have a
2104 * starting point (as a part of an LRO/RSC series) but the
2105 * compiler cursed at me when I tried to cast away the
2106 * "volatile" from rx_ring (yes, it's volatile too!!!). So, I'm
2107 * keeping it the way it is for now.
2109 * The code in this file is broken in so many other places and
2110 * will just not work on a big endian CPU anyway therefore the
2111 * lines below will have to be revisited together with the rest
2115 * - Get rid of "volatile" and let the compiler do its job.
2116 * - Use the proper memory barrier (rte_rmb()) to ensure the
2117 * memory ordering below.
2119 rxdp = &rx_ring[rx_id];
2120 staterr = rte_le_to_cpu_32(rxdp->wb.upper.status_error);
2122 if (!(staterr & IXGBE_RXDADV_STAT_DD))
2127 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u "
2128 "staterr=0x%x data_len=%u",
2129 rxq->port_id, rxq->queue_id, rx_id, staterr,
2130 rte_le_to_cpu_16(rxd.wb.upper.length));
2133 nmb = rte_mbuf_raw_alloc(rxq->mb_pool);
2135 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed "
2136 "port_id=%u queue_id=%u",
2137 rxq->port_id, rxq->queue_id);
2139 rte_eth_devices[rxq->port_id].data->
2140 rx_mbuf_alloc_failed++;
2143 } else if (nb_hold > rxq->rx_free_thresh) {
2144 uint16_t next_rdt = rxq->rx_free_trigger;
2146 if (!ixgbe_rx_alloc_bufs(rxq, false)) {
2148 IXGBE_PCI_REG_WC_WRITE_RELAXED(
2151 nb_hold -= rxq->rx_free_thresh;
2153 PMD_RX_LOG(DEBUG, "RX bulk alloc failed "
2154 "port_id=%u queue_id=%u",
2155 rxq->port_id, rxq->queue_id);
2157 rte_eth_devices[rxq->port_id].data->
2158 rx_mbuf_alloc_failed++;
2164 rxe = &sw_ring[rx_id];
2165 eop = staterr & IXGBE_RXDADV_STAT_EOP;
2167 next_id = rx_id + 1;
2168 if (next_id == rxq->nb_rx_desc)
2171 /* Prefetch next mbuf while processing current one. */
2172 rte_ixgbe_prefetch(sw_ring[next_id].mbuf);
2175 * When next RX descriptor is on a cache-line boundary,
2176 * prefetch the next 4 RX descriptors and the next 4 pointers
2179 if ((next_id & 0x3) == 0) {
2180 rte_ixgbe_prefetch(&rx_ring[next_id]);
2181 rte_ixgbe_prefetch(&sw_ring[next_id]);
2188 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
2190 * Update RX descriptor with the physical address of the
2191 * new data buffer of the new allocated mbuf.
2195 rxm->data_off = RTE_PKTMBUF_HEADROOM;
2196 rxdp->read.hdr_addr = 0;
2197 rxdp->read.pkt_addr = dma;
2202 * Set data length & data buffer address of mbuf.
2204 data_len = rte_le_to_cpu_16(rxd.wb.upper.length);
2205 rxm->data_len = data_len;
2210 * Get next descriptor index:
2211 * - For RSC it's in the NEXTP field.
2212 * - For a scattered packet - it's just a following
2215 if (ixgbe_rsc_count(&rxd))
2217 (staterr & IXGBE_RXDADV_NEXTP_MASK) >>
2218 IXGBE_RXDADV_NEXTP_SHIFT;
2222 next_sc_entry = &sw_sc_ring[nextp_id];
2223 next_rxe = &sw_ring[nextp_id];
2224 rte_ixgbe_prefetch(next_rxe);
2227 sc_entry = &sw_sc_ring[rx_id];
2228 first_seg = sc_entry->fbuf;
2229 sc_entry->fbuf = NULL;
2232 * If this is the first buffer of the received packet,
2233 * set the pointer to the first mbuf of the packet and
2234 * initialize its context.
2235 * Otherwise, update the total length and the number of segments
2236 * of the current scattered packet, and update the pointer to
2237 * the last mbuf of the current packet.
2239 if (first_seg == NULL) {
2241 first_seg->pkt_len = data_len;
2242 first_seg->nb_segs = 1;
2244 first_seg->pkt_len += data_len;
2245 first_seg->nb_segs++;
2252 * If this is not the last buffer of the received packet, update
2253 * the pointer to the first mbuf at the NEXTP entry in the
2254 * sw_sc_ring and continue to parse the RX ring.
2256 if (!eop && next_rxe) {
2257 rxm->next = next_rxe->mbuf;
2258 next_sc_entry->fbuf = first_seg;
2262 /* Initialize the first mbuf of the returned packet */
2263 ixgbe_fill_cluster_head_buf(first_seg, &rxd, rxq, staterr);
2266 * Deal with the case, when HW CRC srip is disabled.
2267 * That can't happen when LRO is enabled, but still could
2268 * happen for scattered RX mode.
2270 first_seg->pkt_len -= rxq->crc_len;
2271 if (unlikely(rxm->data_len <= rxq->crc_len)) {
2272 struct rte_mbuf *lp;
2274 for (lp = first_seg; lp->next != rxm; lp = lp->next)
2277 first_seg->nb_segs--;
2278 lp->data_len -= rxq->crc_len - rxm->data_len;
2280 rte_pktmbuf_free_seg(rxm);
2282 rxm->data_len -= rxq->crc_len;
2284 /* Prefetch data of first segment, if configured to do so. */
2285 rte_packet_prefetch((char *)first_seg->buf_addr +
2286 first_seg->data_off);
2289 * Store the mbuf address into the next entry of the array
2290 * of returned packets.
2292 rx_pkts[nb_rx++] = first_seg;
2296 * Record index of the next RX descriptor to probe.
2298 rxq->rx_tail = rx_id;
2301 * If the number of free RX descriptors is greater than the RX free
2302 * threshold of the queue, advance the Receive Descriptor Tail (RDT)
2304 * Update the RDT with the value of the last processed RX descriptor
2305 * minus 1, to guarantee that the RDT register is never equal to the
2306 * RDH register, which creates a "full" ring situation from the
2307 * hardware point of view...
2309 if (!bulk_alloc && nb_hold > rxq->rx_free_thresh) {
2310 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u "
2311 "nb_hold=%u nb_rx=%u",
2312 rxq->port_id, rxq->queue_id, rx_id, nb_hold, nb_rx);
2315 IXGBE_PCI_REG_WC_WRITE_RELAXED(rxq->rdt_reg_addr, prev_id);
2319 rxq->nb_rx_hold = nb_hold;
2324 ixgbe_recv_pkts_lro_single_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
2327 return ixgbe_recv_pkts_lro(rx_queue, rx_pkts, nb_pkts, false);
2331 ixgbe_recv_pkts_lro_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
2334 return ixgbe_recv_pkts_lro(rx_queue, rx_pkts, nb_pkts, true);
2337 /*********************************************************************
2339 * Queue management functions
2341 **********************************************************************/
2343 static void __rte_cold
2344 ixgbe_tx_queue_release_mbufs(struct ixgbe_tx_queue *txq)
2348 if (txq->sw_ring != NULL) {
2349 for (i = 0; i < txq->nb_tx_desc; i++) {
2350 if (txq->sw_ring[i].mbuf != NULL) {
2351 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
2352 txq->sw_ring[i].mbuf = NULL;
2359 ixgbe_tx_done_cleanup_full(struct ixgbe_tx_queue *txq, uint32_t free_cnt)
2361 struct ixgbe_tx_entry *swr_ring = txq->sw_ring;
2362 uint16_t i, tx_last, tx_id;
2363 uint16_t nb_tx_free_last;
2364 uint16_t nb_tx_to_clean;
2367 /* Start free mbuf from the next of tx_tail */
2368 tx_last = txq->tx_tail;
2369 tx_id = swr_ring[tx_last].next_id;
2371 if (txq->nb_tx_free == 0 && ixgbe_xmit_cleanup(txq))
2374 nb_tx_to_clean = txq->nb_tx_free;
2375 nb_tx_free_last = txq->nb_tx_free;
2377 free_cnt = txq->nb_tx_desc;
2379 /* Loop through swr_ring to count the amount of
2380 * freeable mubfs and packets.
2382 for (pkt_cnt = 0; pkt_cnt < free_cnt; ) {
2383 for (i = 0; i < nb_tx_to_clean &&
2384 pkt_cnt < free_cnt &&
2385 tx_id != tx_last; i++) {
2386 if (swr_ring[tx_id].mbuf != NULL) {
2387 rte_pktmbuf_free_seg(swr_ring[tx_id].mbuf);
2388 swr_ring[tx_id].mbuf = NULL;
2391 * last segment in the packet,
2392 * increment packet count
2394 pkt_cnt += (swr_ring[tx_id].last_id == tx_id);
2397 tx_id = swr_ring[tx_id].next_id;
2400 if (txq->tx_rs_thresh > txq->nb_tx_desc -
2401 txq->nb_tx_free || tx_id == tx_last)
2404 if (pkt_cnt < free_cnt) {
2405 if (ixgbe_xmit_cleanup(txq))
2408 nb_tx_to_clean = txq->nb_tx_free - nb_tx_free_last;
2409 nb_tx_free_last = txq->nb_tx_free;
2413 return (int)pkt_cnt;
2417 ixgbe_tx_done_cleanup_simple(struct ixgbe_tx_queue *txq,
2422 if (free_cnt == 0 || free_cnt > txq->nb_tx_desc)
2423 free_cnt = txq->nb_tx_desc;
2425 cnt = free_cnt - free_cnt % txq->tx_rs_thresh;
2427 for (i = 0; i < cnt; i += n) {
2428 if (txq->nb_tx_desc - txq->nb_tx_free < txq->tx_rs_thresh)
2431 n = ixgbe_tx_free_bufs(txq);
2441 ixgbe_tx_done_cleanup_vec(struct ixgbe_tx_queue *txq __rte_unused,
2442 uint32_t free_cnt __rte_unused)
2448 ixgbe_dev_tx_done_cleanup(void *tx_queue, uint32_t free_cnt)
2450 struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue;
2451 if (txq->offloads == 0 &&
2452 #ifdef RTE_LIB_SECURITY
2453 !(txq->using_ipsec) &&
2455 txq->tx_rs_thresh >= RTE_PMD_IXGBE_TX_MAX_BURST) {
2456 if (txq->tx_rs_thresh <= RTE_IXGBE_TX_MAX_FREE_BUF_SZ &&
2457 rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128 &&
2458 (rte_eal_process_type() != RTE_PROC_PRIMARY ||
2459 txq->sw_ring_v != NULL)) {
2460 return ixgbe_tx_done_cleanup_vec(txq, free_cnt);
2462 return ixgbe_tx_done_cleanup_simple(txq, free_cnt);
2466 return ixgbe_tx_done_cleanup_full(txq, free_cnt);
2469 static void __rte_cold
2470 ixgbe_tx_free_swring(struct ixgbe_tx_queue *txq)
2473 txq->sw_ring != NULL)
2474 rte_free(txq->sw_ring);
2477 static void __rte_cold
2478 ixgbe_tx_queue_release(struct ixgbe_tx_queue *txq)
2480 if (txq != NULL && txq->ops != NULL) {
2481 txq->ops->release_mbufs(txq);
2482 txq->ops->free_swring(txq);
2483 rte_memzone_free(txq->mz);
2489 ixgbe_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
2491 ixgbe_tx_queue_release(dev->data->tx_queues[qid]);
2494 /* (Re)set dynamic ixgbe_tx_queue fields to defaults */
2495 static void __rte_cold
2496 ixgbe_reset_tx_queue(struct ixgbe_tx_queue *txq)
2498 static const union ixgbe_adv_tx_desc zeroed_desc = {{0}};
2499 struct ixgbe_tx_entry *txe = txq->sw_ring;
2502 /* Zero out HW ring memory */
2503 for (i = 0; i < txq->nb_tx_desc; i++) {
2504 txq->tx_ring[i] = zeroed_desc;
2507 /* Initialize SW ring entries */
2508 prev = (uint16_t) (txq->nb_tx_desc - 1);
2509 for (i = 0; i < txq->nb_tx_desc; i++) {
2510 volatile union ixgbe_adv_tx_desc *txd = &txq->tx_ring[i];
2512 txd->wb.status = rte_cpu_to_le_32(IXGBE_TXD_STAT_DD);
2515 txe[prev].next_id = i;
2519 txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
2520 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
2523 txq->nb_tx_used = 0;
2525 * Always allow 1 descriptor to be un-allocated to avoid
2526 * a H/W race condition
2528 txq->last_desc_cleaned = (uint16_t)(txq->nb_tx_desc - 1);
2529 txq->nb_tx_free = (uint16_t)(txq->nb_tx_desc - 1);
2531 memset((void *)&txq->ctx_cache, 0,
2532 IXGBE_CTX_NUM * sizeof(struct ixgbe_advctx_info));
2535 static const struct ixgbe_txq_ops def_txq_ops = {
2536 .release_mbufs = ixgbe_tx_queue_release_mbufs,
2537 .free_swring = ixgbe_tx_free_swring,
2538 .reset = ixgbe_reset_tx_queue,
2541 /* Takes an ethdev and a queue and sets up the tx function to be used based on
2542 * the queue parameters. Used in tx_queue_setup by primary process and then
2543 * in dev_init by secondary process when attaching to an existing ethdev.
2546 ixgbe_set_tx_function(struct rte_eth_dev *dev, struct ixgbe_tx_queue *txq)
2548 /* Use a simple Tx queue (no offloads, no multi segs) if possible */
2549 if ((txq->offloads == 0) &&
2550 #ifdef RTE_LIB_SECURITY
2551 !(txq->using_ipsec) &&
2553 (txq->tx_rs_thresh >= RTE_PMD_IXGBE_TX_MAX_BURST)) {
2554 PMD_INIT_LOG(DEBUG, "Using simple tx code path");
2555 dev->tx_pkt_prepare = NULL;
2556 if (txq->tx_rs_thresh <= RTE_IXGBE_TX_MAX_FREE_BUF_SZ &&
2557 rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128 &&
2558 (rte_eal_process_type() != RTE_PROC_PRIMARY ||
2559 ixgbe_txq_vec_setup(txq) == 0)) {
2560 PMD_INIT_LOG(DEBUG, "Vector tx enabled.");
2561 dev->tx_pkt_burst = ixgbe_xmit_pkts_vec;
2563 dev->tx_pkt_burst = ixgbe_xmit_pkts_simple;
2565 PMD_INIT_LOG(DEBUG, "Using full-featured tx code path");
2567 " - offloads = 0x%" PRIx64,
2570 " - tx_rs_thresh = %lu " "[RTE_PMD_IXGBE_TX_MAX_BURST=%lu]",
2571 (unsigned long)txq->tx_rs_thresh,
2572 (unsigned long)RTE_PMD_IXGBE_TX_MAX_BURST);
2573 dev->tx_pkt_burst = ixgbe_xmit_pkts;
2574 dev->tx_pkt_prepare = ixgbe_prep_pkts;
2579 ixgbe_get_tx_queue_offloads(struct rte_eth_dev *dev)
2587 ixgbe_get_tx_port_offloads(struct rte_eth_dev *dev)
2589 uint64_t tx_offload_capa;
2590 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2593 RTE_ETH_TX_OFFLOAD_VLAN_INSERT |
2594 RTE_ETH_TX_OFFLOAD_IPV4_CKSUM |
2595 RTE_ETH_TX_OFFLOAD_UDP_CKSUM |
2596 RTE_ETH_TX_OFFLOAD_TCP_CKSUM |
2597 RTE_ETH_TX_OFFLOAD_SCTP_CKSUM |
2598 RTE_ETH_TX_OFFLOAD_TCP_TSO |
2599 RTE_ETH_TX_OFFLOAD_MULTI_SEGS;
2601 if (hw->mac.type == ixgbe_mac_82599EB ||
2602 hw->mac.type == ixgbe_mac_X540)
2603 tx_offload_capa |= RTE_ETH_TX_OFFLOAD_MACSEC_INSERT;
2605 if (hw->mac.type == ixgbe_mac_X550 ||
2606 hw->mac.type == ixgbe_mac_X550EM_x ||
2607 hw->mac.type == ixgbe_mac_X550EM_a)
2608 tx_offload_capa |= RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM;
2610 #ifdef RTE_LIB_SECURITY
2611 if (dev->security_ctx)
2612 tx_offload_capa |= RTE_ETH_TX_OFFLOAD_SECURITY;
2614 return tx_offload_capa;
2618 ixgbe_dev_tx_queue_setup(struct rte_eth_dev *dev,
2621 unsigned int socket_id,
2622 const struct rte_eth_txconf *tx_conf)
2624 const struct rte_memzone *tz;
2625 struct ixgbe_tx_queue *txq;
2626 struct ixgbe_hw *hw;
2627 uint16_t tx_rs_thresh, tx_free_thresh;
2630 PMD_INIT_FUNC_TRACE();
2631 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2633 offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads;
2636 * Validate number of transmit descriptors.
2637 * It must not exceed hardware maximum, and must be multiple
2640 if (nb_desc % IXGBE_TXD_ALIGN != 0 ||
2641 (nb_desc > IXGBE_MAX_RING_DESC) ||
2642 (nb_desc < IXGBE_MIN_RING_DESC)) {
2647 * The following two parameters control the setting of the RS bit on
2648 * transmit descriptors.
2649 * TX descriptors will have their RS bit set after txq->tx_rs_thresh
2650 * descriptors have been used.
2651 * The TX descriptor ring will be cleaned after txq->tx_free_thresh
2652 * descriptors are used or if the number of descriptors required
2653 * to transmit a packet is greater than the number of free TX
2655 * The following constraints must be satisfied:
2656 * tx_rs_thresh must be greater than 0.
2657 * tx_rs_thresh must be less than the size of the ring minus 2.
2658 * tx_rs_thresh must be less than or equal to tx_free_thresh.
2659 * tx_rs_thresh must be a divisor of the ring size.
2660 * tx_free_thresh must be greater than 0.
2661 * tx_free_thresh must be less than the size of the ring minus 3.
2662 * tx_free_thresh + tx_rs_thresh must not exceed nb_desc.
2663 * One descriptor in the TX ring is used as a sentinel to avoid a
2664 * H/W race condition, hence the maximum threshold constraints.
2665 * When set to zero use default values.
2667 tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ?
2668 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH);
2669 /* force tx_rs_thresh to adapt an aggressive tx_free_thresh */
2670 tx_rs_thresh = (DEFAULT_TX_RS_THRESH + tx_free_thresh > nb_desc) ?
2671 nb_desc - tx_free_thresh : DEFAULT_TX_RS_THRESH;
2672 if (tx_conf->tx_rs_thresh > 0)
2673 tx_rs_thresh = tx_conf->tx_rs_thresh;
2674 if (tx_rs_thresh + tx_free_thresh > nb_desc) {
2675 PMD_INIT_LOG(ERR, "tx_rs_thresh + tx_free_thresh must not "
2676 "exceed nb_desc. (tx_rs_thresh=%u "
2677 "tx_free_thresh=%u nb_desc=%u port = %d queue=%d)",
2678 (unsigned int)tx_rs_thresh,
2679 (unsigned int)tx_free_thresh,
2680 (unsigned int)nb_desc,
2681 (int)dev->data->port_id,
2685 if (tx_rs_thresh >= (nb_desc - 2)) {
2686 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than the number "
2687 "of TX descriptors minus 2. (tx_rs_thresh=%u "
2688 "port=%d queue=%d)", (unsigned int)tx_rs_thresh,
2689 (int)dev->data->port_id, (int)queue_idx);
2692 if (tx_rs_thresh > DEFAULT_TX_RS_THRESH) {
2693 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less or equal than %u. "
2694 "(tx_rs_thresh=%u port=%d queue=%d)",
2695 DEFAULT_TX_RS_THRESH, (unsigned int)tx_rs_thresh,
2696 (int)dev->data->port_id, (int)queue_idx);
2699 if (tx_free_thresh >= (nb_desc - 3)) {
2700 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than the "
2701 "tx_free_thresh must be less than the number of "
2702 "TX descriptors minus 3. (tx_free_thresh=%u "
2703 "port=%d queue=%d)",
2704 (unsigned int)tx_free_thresh,
2705 (int)dev->data->port_id, (int)queue_idx);
2708 if (tx_rs_thresh > tx_free_thresh) {
2709 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than or equal to "
2710 "tx_free_thresh. (tx_free_thresh=%u "
2711 "tx_rs_thresh=%u port=%d queue=%d)",
2712 (unsigned int)tx_free_thresh,
2713 (unsigned int)tx_rs_thresh,
2714 (int)dev->data->port_id,
2718 if ((nb_desc % tx_rs_thresh) != 0) {
2719 PMD_INIT_LOG(ERR, "tx_rs_thresh must be a divisor of the "
2720 "number of TX descriptors. (tx_rs_thresh=%u "
2721 "port=%d queue=%d)", (unsigned int)tx_rs_thresh,
2722 (int)dev->data->port_id, (int)queue_idx);
2727 * If rs_bit_thresh is greater than 1, then TX WTHRESH should be
2728 * set to 0. If WTHRESH is greater than zero, the RS bit is ignored
2729 * by the NIC and all descriptors are written back after the NIC
2730 * accumulates WTHRESH descriptors.
2732 if ((tx_rs_thresh > 1) && (tx_conf->tx_thresh.wthresh != 0)) {
2733 PMD_INIT_LOG(ERR, "TX WTHRESH must be set to 0 if "
2734 "tx_rs_thresh is greater than 1. (tx_rs_thresh=%u "
2735 "port=%d queue=%d)", (unsigned int)tx_rs_thresh,
2736 (int)dev->data->port_id, (int)queue_idx);
2740 /* Free memory prior to re-allocation if needed... */
2741 if (dev->data->tx_queues[queue_idx] != NULL) {
2742 ixgbe_tx_queue_release(dev->data->tx_queues[queue_idx]);
2743 dev->data->tx_queues[queue_idx] = NULL;
2746 /* First allocate the tx queue data structure */
2747 txq = rte_zmalloc_socket("ethdev TX queue", sizeof(struct ixgbe_tx_queue),
2748 RTE_CACHE_LINE_SIZE, socket_id);
2753 * Allocate TX ring hardware descriptors. A memzone large enough to
2754 * handle the maximum ring size is allocated in order to allow for
2755 * resizing in later calls to the queue setup function.
2757 tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx,
2758 sizeof(union ixgbe_adv_tx_desc) * IXGBE_MAX_RING_DESC,
2759 IXGBE_ALIGN, socket_id);
2761 ixgbe_tx_queue_release(txq);
2766 txq->nb_tx_desc = nb_desc;
2767 txq->tx_rs_thresh = tx_rs_thresh;
2768 txq->tx_free_thresh = tx_free_thresh;
2769 txq->pthresh = tx_conf->tx_thresh.pthresh;
2770 txq->hthresh = tx_conf->tx_thresh.hthresh;
2771 txq->wthresh = tx_conf->tx_thresh.wthresh;
2772 txq->queue_id = queue_idx;
2773 txq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ?
2774 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx);
2775 txq->port_id = dev->data->port_id;
2776 txq->offloads = offloads;
2777 txq->ops = &def_txq_ops;
2778 txq->tx_deferred_start = tx_conf->tx_deferred_start;
2779 #ifdef RTE_LIB_SECURITY
2780 txq->using_ipsec = !!(dev->data->dev_conf.txmode.offloads &
2781 RTE_ETH_TX_OFFLOAD_SECURITY);
2785 * Modification to set VFTDT for virtual function if vf is detected
2787 if (hw->mac.type == ixgbe_mac_82599_vf ||
2788 hw->mac.type == ixgbe_mac_X540_vf ||
2789 hw->mac.type == ixgbe_mac_X550_vf ||
2790 hw->mac.type == ixgbe_mac_X550EM_x_vf ||
2791 hw->mac.type == ixgbe_mac_X550EM_a_vf)
2792 txq->tdt_reg_addr = IXGBE_PCI_REG_ADDR(hw, IXGBE_VFTDT(queue_idx));
2794 txq->tdt_reg_addr = IXGBE_PCI_REG_ADDR(hw, IXGBE_TDT(txq->reg_idx));
2796 txq->tx_ring_phys_addr = tz->iova;
2797 txq->tx_ring = (union ixgbe_adv_tx_desc *) tz->addr;
2799 /* Allocate software ring */
2800 txq->sw_ring = rte_zmalloc_socket("txq->sw_ring",
2801 sizeof(struct ixgbe_tx_entry) * nb_desc,
2802 RTE_CACHE_LINE_SIZE, socket_id);
2803 if (txq->sw_ring == NULL) {
2804 ixgbe_tx_queue_release(txq);
2807 PMD_INIT_LOG(DEBUG, "sw_ring=%p hw_ring=%p dma_addr=0x%"PRIx64,
2808 txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr);
2810 /* set up vector or scalar TX function as appropriate */
2811 ixgbe_set_tx_function(dev, txq);
2813 txq->ops->reset(txq);
2815 dev->data->tx_queues[queue_idx] = txq;
2822 * ixgbe_free_sc_cluster - free the not-yet-completed scattered cluster
2824 * The "next" pointer of the last segment of (not-yet-completed) RSC clusters
2825 * in the sw_rsc_ring is not set to NULL but rather points to the next
2826 * mbuf of this RSC aggregation (that has not been completed yet and still
2827 * resides on the HW ring). So, instead of calling for rte_pktmbuf_free() we
2828 * will just free first "nb_segs" segments of the cluster explicitly by calling
2829 * an rte_pktmbuf_free_seg().
2831 * @m scattered cluster head
2833 static void __rte_cold
2834 ixgbe_free_sc_cluster(struct rte_mbuf *m)
2836 uint16_t i, nb_segs = m->nb_segs;
2837 struct rte_mbuf *next_seg;
2839 for (i = 0; i < nb_segs; i++) {
2841 rte_pktmbuf_free_seg(m);
2846 static void __rte_cold
2847 ixgbe_rx_queue_release_mbufs(struct ixgbe_rx_queue *rxq)
2851 /* SSE Vector driver has a different way of releasing mbufs. */
2852 if (rxq->rx_using_sse) {
2853 ixgbe_rx_queue_release_mbufs_vec(rxq);
2857 if (rxq->sw_ring != NULL) {
2858 for (i = 0; i < rxq->nb_rx_desc; i++) {
2859 if (rxq->sw_ring[i].mbuf != NULL) {
2860 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
2861 rxq->sw_ring[i].mbuf = NULL;
2864 if (rxq->rx_nb_avail) {
2865 for (i = 0; i < rxq->rx_nb_avail; ++i) {
2866 struct rte_mbuf *mb;
2868 mb = rxq->rx_stage[rxq->rx_next_avail + i];
2869 rte_pktmbuf_free_seg(mb);
2871 rxq->rx_nb_avail = 0;
2875 if (rxq->sw_sc_ring)
2876 for (i = 0; i < rxq->nb_rx_desc; i++)
2877 if (rxq->sw_sc_ring[i].fbuf) {
2878 ixgbe_free_sc_cluster(rxq->sw_sc_ring[i].fbuf);
2879 rxq->sw_sc_ring[i].fbuf = NULL;
2883 static void __rte_cold
2884 ixgbe_rx_queue_release(struct ixgbe_rx_queue *rxq)
2887 ixgbe_rx_queue_release_mbufs(rxq);
2888 rte_free(rxq->sw_ring);
2889 rte_free(rxq->sw_sc_ring);
2890 rte_memzone_free(rxq->mz);
2896 ixgbe_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
2898 ixgbe_rx_queue_release(dev->data->rx_queues[qid]);
2902 * Check if Rx Burst Bulk Alloc function can be used.
2904 * 0: the preconditions are satisfied and the bulk allocation function
2906 * -EINVAL: the preconditions are NOT satisfied and the default Rx burst
2907 * function must be used.
2909 static inline int __rte_cold
2910 check_rx_burst_bulk_alloc_preconditions(struct ixgbe_rx_queue *rxq)
2915 * Make sure the following pre-conditions are satisfied:
2916 * rxq->rx_free_thresh >= RTE_PMD_IXGBE_RX_MAX_BURST
2917 * rxq->rx_free_thresh < rxq->nb_rx_desc
2918 * (rxq->nb_rx_desc % rxq->rx_free_thresh) == 0
2919 * Scattered packets are not supported. This should be checked
2920 * outside of this function.
2922 if (!(rxq->rx_free_thresh >= RTE_PMD_IXGBE_RX_MAX_BURST)) {
2923 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
2924 "rxq->rx_free_thresh=%d, "
2925 "RTE_PMD_IXGBE_RX_MAX_BURST=%d",
2926 rxq->rx_free_thresh, RTE_PMD_IXGBE_RX_MAX_BURST);
2928 } else if (!(rxq->rx_free_thresh < rxq->nb_rx_desc)) {
2929 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
2930 "rxq->rx_free_thresh=%d, "
2931 "rxq->nb_rx_desc=%d",
2932 rxq->rx_free_thresh, rxq->nb_rx_desc);
2934 } else if (!((rxq->nb_rx_desc % rxq->rx_free_thresh) == 0)) {
2935 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
2936 "rxq->nb_rx_desc=%d, "
2937 "rxq->rx_free_thresh=%d",
2938 rxq->nb_rx_desc, rxq->rx_free_thresh);
2945 /* Reset dynamic ixgbe_rx_queue fields back to defaults */
2946 static void __rte_cold
2947 ixgbe_reset_rx_queue(struct ixgbe_adapter *adapter, struct ixgbe_rx_queue *rxq)
2949 static const union ixgbe_adv_rx_desc zeroed_desc = {{0}};
2951 uint16_t len = rxq->nb_rx_desc;
2954 * By default, the Rx queue setup function allocates enough memory for
2955 * IXGBE_MAX_RING_DESC. The Rx Burst bulk allocation function requires
2956 * extra memory at the end of the descriptor ring to be zero'd out.
2958 if (adapter->rx_bulk_alloc_allowed)
2959 /* zero out extra memory */
2960 len += RTE_PMD_IXGBE_RX_MAX_BURST;
2963 * Zero out HW ring memory. Zero out extra memory at the end of
2964 * the H/W ring so look-ahead logic in Rx Burst bulk alloc function
2965 * reads extra memory as zeros.
2967 for (i = 0; i < len; i++) {
2968 rxq->rx_ring[i] = zeroed_desc;
2972 * initialize extra software ring entries. Space for these extra
2973 * entries is always allocated
2975 memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf));
2976 for (i = rxq->nb_rx_desc; i < len; ++i) {
2977 rxq->sw_ring[i].mbuf = &rxq->fake_mbuf;
2980 rxq->rx_nb_avail = 0;
2981 rxq->rx_next_avail = 0;
2982 rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1);
2984 rxq->nb_rx_hold = 0;
2986 rte_pktmbuf_free(rxq->pkt_first_seg);
2988 rxq->pkt_first_seg = NULL;
2989 rxq->pkt_last_seg = NULL;
2991 #if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64)
2992 rxq->rxrearm_start = 0;
2993 rxq->rxrearm_nb = 0;
2998 ixgbe_is_vf(struct rte_eth_dev *dev)
3000 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3002 switch (hw->mac.type) {
3003 case ixgbe_mac_82599_vf:
3004 case ixgbe_mac_X540_vf:
3005 case ixgbe_mac_X550_vf:
3006 case ixgbe_mac_X550EM_x_vf:
3007 case ixgbe_mac_X550EM_a_vf:
3015 ixgbe_get_rx_queue_offloads(struct rte_eth_dev *dev)
3017 uint64_t offloads = 0;
3018 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3020 if (hw->mac.type != ixgbe_mac_82598EB)
3021 offloads |= RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
3027 ixgbe_get_rx_port_offloads(struct rte_eth_dev *dev)
3030 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3032 offloads = RTE_ETH_RX_OFFLOAD_IPV4_CKSUM |
3033 RTE_ETH_RX_OFFLOAD_UDP_CKSUM |
3034 RTE_ETH_RX_OFFLOAD_TCP_CKSUM |
3035 RTE_ETH_RX_OFFLOAD_KEEP_CRC |
3036 RTE_ETH_RX_OFFLOAD_VLAN_FILTER |
3037 RTE_ETH_RX_OFFLOAD_SCATTER |
3038 RTE_ETH_RX_OFFLOAD_RSS_HASH;
3040 if (hw->mac.type == ixgbe_mac_82598EB)
3041 offloads |= RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
3043 if (ixgbe_is_vf(dev) == 0)
3044 offloads |= RTE_ETH_RX_OFFLOAD_VLAN_EXTEND;
3047 * RSC is only supported by 82599 and x540 PF devices in a non-SR-IOV
3050 if ((hw->mac.type == ixgbe_mac_82599EB ||
3051 hw->mac.type == ixgbe_mac_X540 ||
3052 hw->mac.type == ixgbe_mac_X550) &&
3053 !RTE_ETH_DEV_SRIOV(dev).active)
3054 offloads |= RTE_ETH_RX_OFFLOAD_TCP_LRO;
3056 if (hw->mac.type == ixgbe_mac_82599EB ||
3057 hw->mac.type == ixgbe_mac_X540)
3058 offloads |= RTE_ETH_RX_OFFLOAD_MACSEC_STRIP;
3060 if (hw->mac.type == ixgbe_mac_X550 ||
3061 hw->mac.type == ixgbe_mac_X550EM_x ||
3062 hw->mac.type == ixgbe_mac_X550EM_a)
3063 offloads |= RTE_ETH_RX_OFFLOAD_OUTER_IPV4_CKSUM;
3065 #ifdef RTE_LIB_SECURITY
3066 if (dev->security_ctx)
3067 offloads |= RTE_ETH_RX_OFFLOAD_SECURITY;
3074 ixgbe_dev_rx_queue_setup(struct rte_eth_dev *dev,
3077 unsigned int socket_id,
3078 const struct rte_eth_rxconf *rx_conf,
3079 struct rte_mempool *mp)
3081 const struct rte_memzone *rz;
3082 struct ixgbe_rx_queue *rxq;
3083 struct ixgbe_hw *hw;
3085 struct ixgbe_adapter *adapter = dev->data->dev_private;
3088 PMD_INIT_FUNC_TRACE();
3089 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3091 offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads;
3094 * Validate number of receive descriptors.
3095 * It must not exceed hardware maximum, and must be multiple
3098 if (nb_desc % IXGBE_RXD_ALIGN != 0 ||
3099 (nb_desc > IXGBE_MAX_RING_DESC) ||
3100 (nb_desc < IXGBE_MIN_RING_DESC)) {
3104 /* Free memory prior to re-allocation if needed... */
3105 if (dev->data->rx_queues[queue_idx] != NULL) {
3106 ixgbe_rx_queue_release(dev->data->rx_queues[queue_idx]);
3107 dev->data->rx_queues[queue_idx] = NULL;
3110 /* First allocate the rx queue data structure */
3111 rxq = rte_zmalloc_socket("ethdev RX queue", sizeof(struct ixgbe_rx_queue),
3112 RTE_CACHE_LINE_SIZE, socket_id);
3116 rxq->nb_rx_desc = nb_desc;
3117 rxq->rx_free_thresh = rx_conf->rx_free_thresh;
3118 rxq->queue_id = queue_idx;
3119 rxq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ?
3120 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx);
3121 rxq->port_id = dev->data->port_id;
3122 if (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC)
3123 rxq->crc_len = RTE_ETHER_CRC_LEN;
3126 rxq->drop_en = rx_conf->rx_drop_en;
3127 rxq->rx_deferred_start = rx_conf->rx_deferred_start;
3128 rxq->offloads = offloads;
3131 * The packet type in RX descriptor is different for different NICs.
3132 * Some bits are used for x550 but reserved for other NICS.
3133 * So set different masks for different NICs.
3135 if (hw->mac.type == ixgbe_mac_X550 ||
3136 hw->mac.type == ixgbe_mac_X550EM_x ||
3137 hw->mac.type == ixgbe_mac_X550EM_a ||
3138 hw->mac.type == ixgbe_mac_X550_vf ||
3139 hw->mac.type == ixgbe_mac_X550EM_x_vf ||
3140 hw->mac.type == ixgbe_mac_X550EM_a_vf)
3141 rxq->pkt_type_mask = IXGBE_PACKET_TYPE_MASK_X550;
3143 rxq->pkt_type_mask = IXGBE_PACKET_TYPE_MASK_82599;
3146 * 82599 errata, UDP frames with a 0 checksum can be marked as checksum
3149 if (hw->mac.type == ixgbe_mac_82599EB)
3150 rxq->rx_udp_csum_zero_err = 1;
3153 * Allocate RX ring hardware descriptors. A memzone large enough to
3154 * handle the maximum ring size is allocated in order to allow for
3155 * resizing in later calls to the queue setup function.
3157 rz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx,
3158 RX_RING_SZ, IXGBE_ALIGN, socket_id);
3160 ixgbe_rx_queue_release(rxq);
3166 * Zero init all the descriptors in the ring.
3168 memset(rz->addr, 0, RX_RING_SZ);
3171 * Modified to setup VFRDT for Virtual Function
3173 if (hw->mac.type == ixgbe_mac_82599_vf ||
3174 hw->mac.type == ixgbe_mac_X540_vf ||
3175 hw->mac.type == ixgbe_mac_X550_vf ||
3176 hw->mac.type == ixgbe_mac_X550EM_x_vf ||
3177 hw->mac.type == ixgbe_mac_X550EM_a_vf) {
3179 IXGBE_PCI_REG_ADDR(hw, IXGBE_VFRDT(queue_idx));
3181 IXGBE_PCI_REG_ADDR(hw, IXGBE_VFRDH(queue_idx));
3184 IXGBE_PCI_REG_ADDR(hw, IXGBE_RDT(rxq->reg_idx));
3186 IXGBE_PCI_REG_ADDR(hw, IXGBE_RDH(rxq->reg_idx));
3189 rxq->rx_ring_phys_addr = rz->iova;
3190 rxq->rx_ring = (union ixgbe_adv_rx_desc *) rz->addr;
3193 * Certain constraints must be met in order to use the bulk buffer
3194 * allocation Rx burst function. If any of Rx queues doesn't meet them
3195 * the feature should be disabled for the whole port.
3197 if (check_rx_burst_bulk_alloc_preconditions(rxq)) {
3198 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Rx Bulk Alloc "
3199 "preconditions - canceling the feature for "
3200 "the whole port[%d]",
3201 rxq->queue_id, rxq->port_id);
3202 adapter->rx_bulk_alloc_allowed = false;
3206 * Allocate software ring. Allow for space at the end of the
3207 * S/W ring to make sure look-ahead logic in bulk alloc Rx burst
3208 * function does not access an invalid memory region.
3211 if (adapter->rx_bulk_alloc_allowed)
3212 len += RTE_PMD_IXGBE_RX_MAX_BURST;
3214 rxq->sw_ring = rte_zmalloc_socket("rxq->sw_ring",
3215 sizeof(struct ixgbe_rx_entry) * len,
3216 RTE_CACHE_LINE_SIZE, socket_id);
3217 if (!rxq->sw_ring) {
3218 ixgbe_rx_queue_release(rxq);
3223 * Always allocate even if it's not going to be needed in order to
3224 * simplify the code.
3226 * This ring is used in LRO and Scattered Rx cases and Scattered Rx may
3227 * be requested in ixgbe_dev_rx_init(), which is called later from
3231 rte_zmalloc_socket("rxq->sw_sc_ring",
3232 sizeof(struct ixgbe_scattered_rx_entry) * len,
3233 RTE_CACHE_LINE_SIZE, socket_id);
3234 if (!rxq->sw_sc_ring) {
3235 ixgbe_rx_queue_release(rxq);
3239 PMD_INIT_LOG(DEBUG, "sw_ring=%p sw_sc_ring=%p hw_ring=%p "
3240 "dma_addr=0x%"PRIx64,
3241 rxq->sw_ring, rxq->sw_sc_ring, rxq->rx_ring,
3242 rxq->rx_ring_phys_addr);
3244 if (!rte_is_power_of_2(nb_desc)) {
3245 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
3246 "preconditions - canceling the feature for "
3247 "the whole port[%d]",
3248 rxq->queue_id, rxq->port_id);
3249 adapter->rx_vec_allowed = false;
3251 ixgbe_rxq_vec_setup(rxq);
3253 dev->data->rx_queues[queue_idx] = rxq;
3255 ixgbe_reset_rx_queue(adapter, rxq);
3261 ixgbe_dev_rx_queue_count(void *rx_queue)
3263 #define IXGBE_RXQ_SCAN_INTERVAL 4
3264 volatile union ixgbe_adv_rx_desc *rxdp;
3265 struct ixgbe_rx_queue *rxq;
3269 rxdp = &(rxq->rx_ring[rxq->rx_tail]);
3271 while ((desc < rxq->nb_rx_desc) &&
3272 (rxdp->wb.upper.status_error &
3273 rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD))) {
3274 desc += IXGBE_RXQ_SCAN_INTERVAL;
3275 rxdp += IXGBE_RXQ_SCAN_INTERVAL;
3276 if (rxq->rx_tail + desc >= rxq->nb_rx_desc)
3277 rxdp = &(rxq->rx_ring[rxq->rx_tail +
3278 desc - rxq->nb_rx_desc]);
3285 ixgbe_dev_rx_descriptor_status(void *rx_queue, uint16_t offset)
3287 struct ixgbe_rx_queue *rxq = rx_queue;
3288 volatile uint32_t *status;
3289 uint32_t nb_hold, desc;
3291 if (unlikely(offset >= rxq->nb_rx_desc))
3294 #if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64)
3295 if (rxq->rx_using_sse)
3296 nb_hold = rxq->rxrearm_nb;
3299 nb_hold = rxq->nb_rx_hold;
3300 if (offset >= rxq->nb_rx_desc - nb_hold)
3301 return RTE_ETH_RX_DESC_UNAVAIL;
3303 desc = rxq->rx_tail + offset;
3304 if (desc >= rxq->nb_rx_desc)
3305 desc -= rxq->nb_rx_desc;
3307 status = &rxq->rx_ring[desc].wb.upper.status_error;
3308 if (*status & rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD))
3309 return RTE_ETH_RX_DESC_DONE;
3311 return RTE_ETH_RX_DESC_AVAIL;
3315 ixgbe_dev_tx_descriptor_status(void *tx_queue, uint16_t offset)
3317 struct ixgbe_tx_queue *txq = tx_queue;
3318 volatile uint32_t *status;
3321 if (unlikely(offset >= txq->nb_tx_desc))
3324 desc = txq->tx_tail + offset;
3325 /* go to next desc that has the RS bit */
3326 desc = ((desc + txq->tx_rs_thresh - 1) / txq->tx_rs_thresh) *
3328 if (desc >= txq->nb_tx_desc) {
3329 desc -= txq->nb_tx_desc;
3330 if (desc >= txq->nb_tx_desc)
3331 desc -= txq->nb_tx_desc;
3334 status = &txq->tx_ring[desc].wb.status;
3335 if (*status & rte_cpu_to_le_32(IXGBE_ADVTXD_STAT_DD))
3336 return RTE_ETH_TX_DESC_DONE;
3338 return RTE_ETH_TX_DESC_FULL;
3342 * Set up link loopback for X540/X550 mode Tx->Rx.
3344 static inline void __rte_cold
3345 ixgbe_setup_loopback_link_x540_x550(struct ixgbe_hw *hw, bool enable)
3348 PMD_INIT_FUNC_TRACE();
3350 u16 autoneg_reg = IXGBE_MII_AUTONEG_REG;
3352 hw->phy.ops.read_reg(hw, IXGBE_MDIO_AUTO_NEG_CONTROL,
3353 IXGBE_MDIO_AUTO_NEG_DEV_TYPE, &autoneg_reg);
3354 macc = IXGBE_READ_REG(hw, IXGBE_MACC);
3357 /* datasheet 15.2.1: disable AUTONEG (PHY Bit 7.0.C) */
3358 autoneg_reg |= IXGBE_MII_AUTONEG_ENABLE;
3359 /* datasheet 15.2.1: MACC.FLU = 1 (force link up) */
3360 macc |= IXGBE_MACC_FLU;
3362 autoneg_reg &= ~IXGBE_MII_AUTONEG_ENABLE;
3363 macc &= ~IXGBE_MACC_FLU;
3366 hw->phy.ops.write_reg(hw, IXGBE_MDIO_AUTO_NEG_CONTROL,
3367 IXGBE_MDIO_AUTO_NEG_DEV_TYPE, autoneg_reg);
3369 IXGBE_WRITE_REG(hw, IXGBE_MACC, macc);
3373 ixgbe_dev_clear_queues(struct rte_eth_dev *dev)
3376 struct ixgbe_adapter *adapter = dev->data->dev_private;
3377 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3379 PMD_INIT_FUNC_TRACE();
3381 for (i = 0; i < dev->data->nb_tx_queues; i++) {
3382 struct ixgbe_tx_queue *txq = dev->data->tx_queues[i];
3385 txq->ops->release_mbufs(txq);
3386 txq->ops->reset(txq);
3390 for (i = 0; i < dev->data->nb_rx_queues; i++) {
3391 struct ixgbe_rx_queue *rxq = dev->data->rx_queues[i];
3394 ixgbe_rx_queue_release_mbufs(rxq);
3395 ixgbe_reset_rx_queue(adapter, rxq);
3398 /* If loopback mode was enabled, reconfigure the link accordingly */
3399 if (dev->data->dev_conf.lpbk_mode != 0) {
3400 if (hw->mac.type == ixgbe_mac_X540 ||
3401 hw->mac.type == ixgbe_mac_X550 ||
3402 hw->mac.type == ixgbe_mac_X550EM_x ||
3403 hw->mac.type == ixgbe_mac_X550EM_a)
3404 ixgbe_setup_loopback_link_x540_x550(hw, false);
3409 ixgbe_dev_free_queues(struct rte_eth_dev *dev)
3413 PMD_INIT_FUNC_TRACE();
3415 for (i = 0; i < dev->data->nb_rx_queues; i++) {
3416 ixgbe_dev_rx_queue_release(dev, i);
3417 dev->data->rx_queues[i] = NULL;
3419 dev->data->nb_rx_queues = 0;
3421 for (i = 0; i < dev->data->nb_tx_queues; i++) {
3422 ixgbe_dev_tx_queue_release(dev, i);
3423 dev->data->tx_queues[i] = NULL;
3425 dev->data->nb_tx_queues = 0;
3428 /*********************************************************************
3430 * Device RX/TX init functions
3432 **********************************************************************/
3435 * Receive Side Scaling (RSS)
3436 * See section 7.1.2.8 in the following document:
3437 * "Intel 82599 10 GbE Controller Datasheet" - Revision 2.1 October 2009
3440 * The source and destination IP addresses of the IP header and the source
3441 * and destination ports of TCP/UDP headers, if any, of received packets are
3442 * hashed against a configurable random key to compute a 32-bit RSS hash result.
3443 * The seven (7) LSBs of the 32-bit hash result are used as an index into a
3444 * 128-entry redirection table (RETA). Each entry of the RETA provides a 3-bit
3445 * RSS output index which is used as the RX queue index where to store the
3447 * The following output is supplied in the RX write-back descriptor:
3448 * - 32-bit result of the Microsoft RSS hash function,
3449 * - 4-bit RSS type field.
3453 * RSS random key supplied in section 7.1.2.8.3 of the Intel 82599 datasheet.
3454 * Used as the default key.
3456 static uint8_t rss_intel_key[40] = {
3457 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
3458 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
3459 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
3460 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
3461 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
3465 ixgbe_rss_disable(struct rte_eth_dev *dev)
3467 struct ixgbe_hw *hw;
3471 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3472 mrqc_reg = ixgbe_mrqc_reg_get(hw->mac.type);
3473 mrqc = IXGBE_READ_REG(hw, mrqc_reg);
3474 mrqc &= ~IXGBE_MRQC_RSSEN;
3475 IXGBE_WRITE_REG(hw, mrqc_reg, mrqc);
3479 ixgbe_hw_rss_hash_set(struct ixgbe_hw *hw, struct rte_eth_rss_conf *rss_conf)
3489 mrqc_reg = ixgbe_mrqc_reg_get(hw->mac.type);
3490 rssrk_reg = ixgbe_rssrk_reg_get(hw->mac.type, 0);
3492 hash_key = rss_conf->rss_key;
3493 if (hash_key != NULL) {
3494 /* Fill in RSS hash key */
3495 for (i = 0; i < 10; i++) {
3496 rss_key = hash_key[(i * 4)];
3497 rss_key |= hash_key[(i * 4) + 1] << 8;
3498 rss_key |= hash_key[(i * 4) + 2] << 16;
3499 rss_key |= hash_key[(i * 4) + 3] << 24;
3500 IXGBE_WRITE_REG_ARRAY(hw, rssrk_reg, i, rss_key);
3504 /* Set configured hashing protocols in MRQC register */
3505 rss_hf = rss_conf->rss_hf;
3506 mrqc = IXGBE_MRQC_RSSEN; /* Enable RSS */
3507 if (rss_hf & RTE_ETH_RSS_IPV4)
3508 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4;
3509 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_TCP)
3510 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP;
3511 if (rss_hf & RTE_ETH_RSS_IPV6)
3512 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6;
3513 if (rss_hf & RTE_ETH_RSS_IPV6_EX)
3514 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX;
3515 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV6_TCP)
3516 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP;
3517 if (rss_hf & RTE_ETH_RSS_IPV6_TCP_EX)
3518 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP;
3519 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_UDP)
3520 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP;
3521 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV6_UDP)
3522 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP;
3523 if (rss_hf & RTE_ETH_RSS_IPV6_UDP_EX)
3524 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
3525 IXGBE_WRITE_REG(hw, mrqc_reg, mrqc);
3529 ixgbe_dev_rss_hash_update(struct rte_eth_dev *dev,
3530 struct rte_eth_rss_conf *rss_conf)
3532 struct ixgbe_hw *hw;
3537 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3539 if (!ixgbe_rss_update_sp(hw->mac.type)) {
3540 PMD_DRV_LOG(ERR, "RSS hash update is not supported on this "
3544 mrqc_reg = ixgbe_mrqc_reg_get(hw->mac.type);
3547 * Excerpt from section 7.1.2.8 Receive-Side Scaling (RSS):
3548 * "RSS enabling cannot be done dynamically while it must be
3549 * preceded by a software reset"
3550 * Before changing anything, first check that the update RSS operation
3551 * does not attempt to disable RSS, if RSS was enabled at
3552 * initialization time, or does not attempt to enable RSS, if RSS was
3553 * disabled at initialization time.
3555 rss_hf = rss_conf->rss_hf & IXGBE_RSS_OFFLOAD_ALL;
3556 mrqc = IXGBE_READ_REG(hw, mrqc_reg);
3557 if (!(mrqc & IXGBE_MRQC_RSSEN)) { /* RSS disabled */
3558 if (rss_hf != 0) /* Enable RSS */
3560 return 0; /* Nothing to do */
3563 if (rss_hf == 0) /* Disable RSS */
3565 ixgbe_hw_rss_hash_set(hw, rss_conf);
3570 ixgbe_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
3571 struct rte_eth_rss_conf *rss_conf)
3573 struct ixgbe_hw *hw;
3582 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3583 mrqc_reg = ixgbe_mrqc_reg_get(hw->mac.type);
3584 rssrk_reg = ixgbe_rssrk_reg_get(hw->mac.type, 0);
3585 hash_key = rss_conf->rss_key;
3586 if (hash_key != NULL) {
3587 /* Return RSS hash key */
3588 for (i = 0; i < 10; i++) {
3589 rss_key = IXGBE_READ_REG_ARRAY(hw, rssrk_reg, i);
3590 hash_key[(i * 4)] = rss_key & 0x000000FF;
3591 hash_key[(i * 4) + 1] = (rss_key >> 8) & 0x000000FF;
3592 hash_key[(i * 4) + 2] = (rss_key >> 16) & 0x000000FF;
3593 hash_key[(i * 4) + 3] = (rss_key >> 24) & 0x000000FF;
3597 /* Get RSS functions configured in MRQC register */
3598 mrqc = IXGBE_READ_REG(hw, mrqc_reg);
3599 if ((mrqc & IXGBE_MRQC_RSSEN) == 0) { /* RSS is disabled */
3600 rss_conf->rss_hf = 0;
3604 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV4)
3605 rss_hf |= RTE_ETH_RSS_IPV4;
3606 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV4_TCP)
3607 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_TCP;
3608 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6)
3609 rss_hf |= RTE_ETH_RSS_IPV6;
3610 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_EX)
3611 rss_hf |= RTE_ETH_RSS_IPV6_EX;
3612 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_TCP)
3613 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_TCP;
3614 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP)
3615 rss_hf |= RTE_ETH_RSS_IPV6_TCP_EX;
3616 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV4_UDP)
3617 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_UDP;
3618 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_UDP)
3619 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_UDP;
3620 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP)
3621 rss_hf |= RTE_ETH_RSS_IPV6_UDP_EX;
3622 rss_conf->rss_hf = rss_hf;
3627 ixgbe_rss_configure(struct rte_eth_dev *dev)
3629 struct rte_eth_rss_conf rss_conf;
3630 struct ixgbe_adapter *adapter;
3631 struct ixgbe_hw *hw;
3635 uint16_t sp_reta_size;
3638 PMD_INIT_FUNC_TRACE();
3639 adapter = dev->data->dev_private;
3640 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3642 sp_reta_size = ixgbe_reta_size_get(hw->mac.type);
3645 * Fill in redirection table
3646 * The byte-swap is needed because NIC registers are in
3647 * little-endian order.
3649 if (adapter->rss_reta_updated == 0) {
3651 for (i = 0, j = 0; i < sp_reta_size; i++, j++) {
3652 reta_reg = ixgbe_reta_reg_get(hw->mac.type, i);
3654 if (j == dev->data->nb_rx_queues)
3656 reta = (reta << 8) | j;
3658 IXGBE_WRITE_REG(hw, reta_reg,
3664 * Configure the RSS key and the RSS protocols used to compute
3665 * the RSS hash of input packets.
3667 rss_conf = dev->data->dev_conf.rx_adv_conf.rss_conf;
3668 if ((rss_conf.rss_hf & IXGBE_RSS_OFFLOAD_ALL) == 0) {
3669 ixgbe_rss_disable(dev);
3672 if (rss_conf.rss_key == NULL)
3673 rss_conf.rss_key = rss_intel_key; /* Default hash key */
3674 ixgbe_hw_rss_hash_set(hw, &rss_conf);
3677 #define NUM_VFTA_REGISTERS 128
3678 #define NIC_RX_BUFFER_SIZE 0x200
3679 #define X550_RX_BUFFER_SIZE 0x180
3682 ixgbe_vmdq_dcb_configure(struct rte_eth_dev *dev)
3684 struct rte_eth_vmdq_dcb_conf *cfg;
3685 struct ixgbe_hw *hw;
3686 enum rte_eth_nb_pools num_pools;
3687 uint32_t mrqc, vt_ctl, queue_mapping, vlanctrl;
3689 uint8_t nb_tcs; /* number of traffic classes */
3692 PMD_INIT_FUNC_TRACE();
3693 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3694 cfg = &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf;
3695 num_pools = cfg->nb_queue_pools;
3696 /* Check we have a valid number of pools */
3697 if (num_pools != RTE_ETH_16_POOLS && num_pools != RTE_ETH_32_POOLS) {
3698 ixgbe_rss_disable(dev);
3701 /* 16 pools -> 8 traffic classes, 32 pools -> 4 traffic classes */
3702 nb_tcs = (uint8_t)(RTE_ETH_VMDQ_DCB_NUM_QUEUES / (int)num_pools);
3706 * split rx buffer up into sections, each for 1 traffic class
3708 switch (hw->mac.type) {
3709 case ixgbe_mac_X550:
3710 case ixgbe_mac_X550EM_x:
3711 case ixgbe_mac_X550EM_a:
3712 pbsize = (uint16_t)(X550_RX_BUFFER_SIZE / nb_tcs);
3715 pbsize = (uint16_t)(NIC_RX_BUFFER_SIZE / nb_tcs);
3718 for (i = 0; i < nb_tcs; i++) {
3719 uint32_t rxpbsize = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(i));
3721 rxpbsize &= (~(0x3FF << IXGBE_RXPBSIZE_SHIFT));
3722 /* clear 10 bits. */
3723 rxpbsize |= (pbsize << IXGBE_RXPBSIZE_SHIFT); /* set value */
3724 IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), rxpbsize);
3726 /* zero alloc all unused TCs */
3727 for (i = nb_tcs; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
3728 uint32_t rxpbsize = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(i));
3730 rxpbsize &= (~(0x3FF << IXGBE_RXPBSIZE_SHIFT));
3731 /* clear 10 bits. */
3732 IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), rxpbsize);
3735 /* MRQC: enable vmdq and dcb */
3736 mrqc = (num_pools == RTE_ETH_16_POOLS) ?
3737 IXGBE_MRQC_VMDQRT8TCEN : IXGBE_MRQC_VMDQRT4TCEN;
3738 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
3740 /* PFVTCTL: turn on virtualisation and set the default pool */
3741 vt_ctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
3742 if (cfg->enable_default_pool) {
3743 vt_ctl |= (cfg->default_pool << IXGBE_VT_CTL_POOL_SHIFT);
3745 vt_ctl |= IXGBE_VT_CTL_DIS_DEFPL;
3748 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vt_ctl);
3750 /* RTRUP2TC: mapping user priorities to traffic classes (TCs) */
3752 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++)
3754 * mapping is done with 3 bits per priority,
3755 * so shift by i*3 each time
3757 queue_mapping |= ((cfg->dcb_tc[i] & 0x07) << (i * 3));
3759 IXGBE_WRITE_REG(hw, IXGBE_RTRUP2TC, queue_mapping);
3761 /* RTRPCS: DCB related */
3762 IXGBE_WRITE_REG(hw, IXGBE_RTRPCS, IXGBE_RMCS_RRM);
3764 /* VLNCTRL: enable vlan filtering and allow all vlan tags through */
3765 vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
3766 vlanctrl |= IXGBE_VLNCTRL_VFE; /* enable vlan filters */
3767 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl);
3769 /* VFTA - enable all vlan filters */
3770 for (i = 0; i < NUM_VFTA_REGISTERS; i++) {
3771 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), 0xFFFFFFFF);
3774 /* VFRE: pool enabling for receive - 16 or 32 */
3775 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0),
3776 num_pools == RTE_ETH_16_POOLS ? 0xFFFF : 0xFFFFFFFF);
3779 * MPSAR - allow pools to read specific mac addresses
3780 * In this case, all pools should be able to read from mac addr 0
3782 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(0), 0xFFFFFFFF);
3783 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(0), 0xFFFFFFFF);
3785 /* PFVLVF, PFVLVFB: set up filters for vlan tags as configured */
3786 for (i = 0; i < cfg->nb_pool_maps; i++) {
3787 /* set vlan id in VF register and set the valid bit */
3788 IXGBE_WRITE_REG(hw, IXGBE_VLVF(i), (IXGBE_VLVF_VIEN |
3789 (cfg->pool_map[i].vlan_id & 0xFFF)));
3791 * Put the allowed pools in VFB reg. As we only have 16 or 32
3792 * pools, we only need to use the first half of the register
3795 IXGBE_WRITE_REG(hw, IXGBE_VLVFB(i*2), cfg->pool_map[i].pools);
3800 * ixgbe_dcb_config_tx_hw_config - Configure general DCB TX parameters
3801 * @dev: pointer to eth_dev structure
3802 * @dcb_config: pointer to ixgbe_dcb_config structure
3805 ixgbe_dcb_tx_hw_config(struct rte_eth_dev *dev,
3806 struct ixgbe_dcb_config *dcb_config)
3809 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3811 PMD_INIT_FUNC_TRACE();
3812 if (hw->mac.type != ixgbe_mac_82598EB) {
3813 /* Disable the Tx desc arbiter so that MTQC can be changed */
3814 reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3815 reg |= IXGBE_RTTDCS_ARBDIS;
3816 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg);
3818 /* Enable DCB for Tx with 8 TCs */
3819 if (dcb_config->num_tcs.pg_tcs == 8) {
3820 reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;
3822 reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
3824 if (dcb_config->vt_mode)
3825 reg |= IXGBE_MTQC_VT_ENA;
3826 IXGBE_WRITE_REG(hw, IXGBE_MTQC, reg);
3828 /* Enable the Tx desc arbiter */
3829 reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3830 reg &= ~IXGBE_RTTDCS_ARBDIS;
3831 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg);
3833 /* Enable Security TX Buffer IFG for DCB */
3834 reg = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG);
3835 reg |= IXGBE_SECTX_DCB;
3836 IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, reg);
3841 * ixgbe_vmdq_dcb_hw_tx_config - Configure general VMDQ+DCB TX parameters
3842 * @dev: pointer to rte_eth_dev structure
3843 * @dcb_config: pointer to ixgbe_dcb_config structure
3846 ixgbe_vmdq_dcb_hw_tx_config(struct rte_eth_dev *dev,
3847 struct ixgbe_dcb_config *dcb_config)
3849 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3850 &dev->data->dev_conf.tx_adv_conf.vmdq_dcb_tx_conf;
3851 struct ixgbe_hw *hw =
3852 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3854 PMD_INIT_FUNC_TRACE();
3855 if (hw->mac.type != ixgbe_mac_82598EB)
3856 /*PF VF Transmit Enable*/
3857 IXGBE_WRITE_REG(hw, IXGBE_VFTE(0),
3858 vmdq_tx_conf->nb_queue_pools == RTE_ETH_16_POOLS ? 0xFFFF : 0xFFFFFFFF);
3860 /*Configure general DCB TX parameters*/
3861 ixgbe_dcb_tx_hw_config(dev, dcb_config);
3865 ixgbe_vmdq_dcb_rx_config(struct rte_eth_dev *dev,
3866 struct ixgbe_dcb_config *dcb_config)
3868 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3869 &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf;
3870 struct ixgbe_dcb_tc_config *tc;
3873 /* convert rte_eth_conf.rx_adv_conf to struct ixgbe_dcb_config */
3874 if (vmdq_rx_conf->nb_queue_pools == RTE_ETH_16_POOLS) {
3875 dcb_config->num_tcs.pg_tcs = RTE_ETH_8_TCS;
3876 dcb_config->num_tcs.pfc_tcs = RTE_ETH_8_TCS;
3878 dcb_config->num_tcs.pg_tcs = RTE_ETH_4_TCS;
3879 dcb_config->num_tcs.pfc_tcs = RTE_ETH_4_TCS;
3882 /* Initialize User Priority to Traffic Class mapping */
3883 for (j = 0; j < IXGBE_DCB_MAX_TRAFFIC_CLASS; j++) {
3884 tc = &dcb_config->tc_config[j];
3885 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0;
3888 /* User Priority to Traffic Class mapping */
3889 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
3890 j = vmdq_rx_conf->dcb_tc[i];
3891 tc = &dcb_config->tc_config[j];
3892 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap |=
3898 ixgbe_dcb_vt_tx_config(struct rte_eth_dev *dev,
3899 struct ixgbe_dcb_config *dcb_config)
3901 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3902 &dev->data->dev_conf.tx_adv_conf.vmdq_dcb_tx_conf;
3903 struct ixgbe_dcb_tc_config *tc;
3906 /* convert rte_eth_conf.rx_adv_conf to struct ixgbe_dcb_config */
3907 if (vmdq_tx_conf->nb_queue_pools == RTE_ETH_16_POOLS) {
3908 dcb_config->num_tcs.pg_tcs = RTE_ETH_8_TCS;
3909 dcb_config->num_tcs.pfc_tcs = RTE_ETH_8_TCS;
3911 dcb_config->num_tcs.pg_tcs = RTE_ETH_4_TCS;
3912 dcb_config->num_tcs.pfc_tcs = RTE_ETH_4_TCS;
3915 /* Initialize User Priority to Traffic Class mapping */
3916 for (j = 0; j < IXGBE_DCB_MAX_TRAFFIC_CLASS; j++) {
3917 tc = &dcb_config->tc_config[j];
3918 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0;
3921 /* User Priority to Traffic Class mapping */
3922 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
3923 j = vmdq_tx_conf->dcb_tc[i];
3924 tc = &dcb_config->tc_config[j];
3925 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap |=
3931 ixgbe_dcb_rx_config(struct rte_eth_dev *dev,
3932 struct ixgbe_dcb_config *dcb_config)
3934 struct rte_eth_dcb_rx_conf *rx_conf =
3935 &dev->data->dev_conf.rx_adv_conf.dcb_rx_conf;
3936 struct ixgbe_dcb_tc_config *tc;
3939 dcb_config->num_tcs.pg_tcs = (uint8_t)rx_conf->nb_tcs;
3940 dcb_config->num_tcs.pfc_tcs = (uint8_t)rx_conf->nb_tcs;
3942 /* Initialize User Priority to Traffic Class mapping */
3943 for (j = 0; j < IXGBE_DCB_MAX_TRAFFIC_CLASS; j++) {
3944 tc = &dcb_config->tc_config[j];
3945 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0;
3948 /* User Priority to Traffic Class mapping */
3949 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
3950 j = rx_conf->dcb_tc[i];
3951 tc = &dcb_config->tc_config[j];
3952 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap |=
3958 ixgbe_dcb_tx_config(struct rte_eth_dev *dev,
3959 struct ixgbe_dcb_config *dcb_config)
3961 struct rte_eth_dcb_tx_conf *tx_conf =
3962 &dev->data->dev_conf.tx_adv_conf.dcb_tx_conf;
3963 struct ixgbe_dcb_tc_config *tc;
3966 dcb_config->num_tcs.pg_tcs = (uint8_t)tx_conf->nb_tcs;
3967 dcb_config->num_tcs.pfc_tcs = (uint8_t)tx_conf->nb_tcs;
3969 /* Initialize User Priority to Traffic Class mapping */
3970 for (j = 0; j < IXGBE_DCB_MAX_TRAFFIC_CLASS; j++) {
3971 tc = &dcb_config->tc_config[j];
3972 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0;
3975 /* User Priority to Traffic Class mapping */
3976 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
3977 j = tx_conf->dcb_tc[i];
3978 tc = &dcb_config->tc_config[j];
3979 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap |=
3985 * ixgbe_dcb_rx_hw_config - Configure general DCB RX HW parameters
3986 * @dev: pointer to eth_dev structure
3987 * @dcb_config: pointer to ixgbe_dcb_config structure
3990 ixgbe_dcb_rx_hw_config(struct rte_eth_dev *dev,
3991 struct ixgbe_dcb_config *dcb_config)
3997 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3999 PMD_INIT_FUNC_TRACE();
4001 * Disable the arbiter before changing parameters
4002 * (always enable recycle mode; WSP)
4004 reg = IXGBE_RTRPCS_RRM | IXGBE_RTRPCS_RAC | IXGBE_RTRPCS_ARBDIS;
4005 IXGBE_WRITE_REG(hw, IXGBE_RTRPCS, reg);
4007 if (hw->mac.type != ixgbe_mac_82598EB) {
4008 reg = IXGBE_READ_REG(hw, IXGBE_MRQC);
4009 if (dcb_config->num_tcs.pg_tcs == 4) {
4010 if (dcb_config->vt_mode)
4011 reg = (reg & ~IXGBE_MRQC_MRQE_MASK) |
4012 IXGBE_MRQC_VMDQRT4TCEN;
4014 /* no matter the mode is DCB or DCB_RSS, just
4015 * set the MRQE to RSSXTCEN. RSS is controlled
4018 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, 0);
4019 reg = (reg & ~IXGBE_MRQC_MRQE_MASK) |
4020 IXGBE_MRQC_RTRSS4TCEN;
4023 if (dcb_config->num_tcs.pg_tcs == 8) {
4024 if (dcb_config->vt_mode)
4025 reg = (reg & ~IXGBE_MRQC_MRQE_MASK) |
4026 IXGBE_MRQC_VMDQRT8TCEN;
4028 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, 0);
4029 reg = (reg & ~IXGBE_MRQC_MRQE_MASK) |
4030 IXGBE_MRQC_RTRSS8TCEN;
4034 IXGBE_WRITE_REG(hw, IXGBE_MRQC, reg);
4036 if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
4037 /* Disable drop for all queues in VMDQ mode*/
4038 for (q = 0; q < IXGBE_MAX_RX_QUEUE_NUM; q++)
4039 IXGBE_WRITE_REG(hw, IXGBE_QDE,
4041 (q << IXGBE_QDE_IDX_SHIFT)));
4043 /* Enable drop for all queues in SRIOV mode */
4044 for (q = 0; q < IXGBE_MAX_RX_QUEUE_NUM; q++)
4045 IXGBE_WRITE_REG(hw, IXGBE_QDE,
4047 (q << IXGBE_QDE_IDX_SHIFT) |
4052 /* VLNCTRL: enable vlan filtering and allow all vlan tags through */
4053 vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4054 vlanctrl |= IXGBE_VLNCTRL_VFE; /* enable vlan filters */
4055 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl);
4057 /* VFTA - enable all vlan filters */
4058 for (i = 0; i < NUM_VFTA_REGISTERS; i++) {
4059 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), 0xFFFFFFFF);
4063 * Configure Rx packet plane (recycle mode; WSP) and
4066 reg = IXGBE_RTRPCS_RRM | IXGBE_RTRPCS_RAC;
4067 IXGBE_WRITE_REG(hw, IXGBE_RTRPCS, reg);
4071 ixgbe_dcb_hw_arbite_rx_config(struct ixgbe_hw *hw, uint16_t *refill,
4072 uint16_t *max, uint8_t *bwg_id, uint8_t *tsa, uint8_t *map)
4074 switch (hw->mac.type) {
4075 case ixgbe_mac_82598EB:
4076 ixgbe_dcb_config_rx_arbiter_82598(hw, refill, max, tsa);
4078 case ixgbe_mac_82599EB:
4079 case ixgbe_mac_X540:
4080 case ixgbe_mac_X550:
4081 case ixgbe_mac_X550EM_x:
4082 case ixgbe_mac_X550EM_a:
4083 ixgbe_dcb_config_rx_arbiter_82599(hw, refill, max, bwg_id,
4092 ixgbe_dcb_hw_arbite_tx_config(struct ixgbe_hw *hw, uint16_t *refill, uint16_t *max,
4093 uint8_t *bwg_id, uint8_t *tsa, uint8_t *map)
4095 switch (hw->mac.type) {
4096 case ixgbe_mac_82598EB:
4097 ixgbe_dcb_config_tx_desc_arbiter_82598(hw, refill, max, bwg_id, tsa);
4098 ixgbe_dcb_config_tx_data_arbiter_82598(hw, refill, max, bwg_id, tsa);
4100 case ixgbe_mac_82599EB:
4101 case ixgbe_mac_X540:
4102 case ixgbe_mac_X550:
4103 case ixgbe_mac_X550EM_x:
4104 case ixgbe_mac_X550EM_a:
4105 ixgbe_dcb_config_tx_desc_arbiter_82599(hw, refill, max, bwg_id, tsa);
4106 ixgbe_dcb_config_tx_data_arbiter_82599(hw, refill, max, bwg_id, tsa, map);
4113 #define DCB_RX_CONFIG 1
4114 #define DCB_TX_CONFIG 1
4115 #define DCB_TX_PB 1024
4117 * ixgbe_dcb_hw_configure - Enable DCB and configure
4118 * general DCB in VT mode and non-VT mode parameters
4119 * @dev: pointer to rte_eth_dev structure
4120 * @dcb_config: pointer to ixgbe_dcb_config structure
4123 ixgbe_dcb_hw_configure(struct rte_eth_dev *dev,
4124 struct ixgbe_dcb_config *dcb_config)
4127 uint8_t i, pfc_en, nb_tcs;
4128 uint16_t pbsize, rx_buffer_size;
4129 uint8_t config_dcb_rx = 0;
4130 uint8_t config_dcb_tx = 0;
4131 uint8_t tsa[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0};
4132 uint8_t bwgid[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0};
4133 uint16_t refill[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0};
4134 uint16_t max[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0};
4135 uint8_t map[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0};
4136 struct ixgbe_dcb_tc_config *tc;
4137 uint32_t max_frame = dev->data->mtu + RTE_ETHER_HDR_LEN +
4139 struct ixgbe_hw *hw =
4140 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4141 struct ixgbe_bw_conf *bw_conf =
4142 IXGBE_DEV_PRIVATE_TO_BW_CONF(dev->data->dev_private);
4144 switch (dev->data->dev_conf.rxmode.mq_mode) {
4145 case RTE_ETH_MQ_RX_VMDQ_DCB:
4146 dcb_config->vt_mode = true;
4147 if (hw->mac.type != ixgbe_mac_82598EB) {
4148 config_dcb_rx = DCB_RX_CONFIG;
4150 *get dcb and VT rx configuration parameters
4153 ixgbe_vmdq_dcb_rx_config(dev, dcb_config);
4154 /*Configure general VMDQ and DCB RX parameters*/
4155 ixgbe_vmdq_dcb_configure(dev);
4158 case RTE_ETH_MQ_RX_DCB:
4159 case RTE_ETH_MQ_RX_DCB_RSS:
4160 dcb_config->vt_mode = false;
4161 config_dcb_rx = DCB_RX_CONFIG;
4162 /* Get dcb TX configuration parameters from rte_eth_conf */
4163 ixgbe_dcb_rx_config(dev, dcb_config);
4164 /*Configure general DCB RX parameters*/
4165 ixgbe_dcb_rx_hw_config(dev, dcb_config);
4168 PMD_INIT_LOG(ERR, "Incorrect DCB RX mode configuration");
4171 switch (dev->data->dev_conf.txmode.mq_mode) {
4172 case RTE_ETH_MQ_TX_VMDQ_DCB:
4173 dcb_config->vt_mode = true;
4174 config_dcb_tx = DCB_TX_CONFIG;
4175 /* get DCB and VT TX configuration parameters
4178 ixgbe_dcb_vt_tx_config(dev, dcb_config);
4179 /*Configure general VMDQ and DCB TX parameters*/
4180 ixgbe_vmdq_dcb_hw_tx_config(dev, dcb_config);
4183 case RTE_ETH_MQ_TX_DCB:
4184 dcb_config->vt_mode = false;
4185 config_dcb_tx = DCB_TX_CONFIG;
4186 /*get DCB TX configuration parameters from rte_eth_conf*/
4187 ixgbe_dcb_tx_config(dev, dcb_config);
4188 /*Configure general DCB TX parameters*/
4189 ixgbe_dcb_tx_hw_config(dev, dcb_config);
4192 PMD_INIT_LOG(ERR, "Incorrect DCB TX mode configuration");
4196 nb_tcs = dcb_config->num_tcs.pfc_tcs;
4198 ixgbe_dcb_unpack_map_cee(dcb_config, IXGBE_DCB_RX_CONFIG, map);
4199 if (nb_tcs == RTE_ETH_4_TCS) {
4200 /* Avoid un-configured priority mapping to TC0 */
4202 uint8_t mask = 0xFF;
4204 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES - 4; i++)
4205 mask = (uint8_t)(mask & (~(1 << map[i])));
4206 for (i = 0; mask && (i < IXGBE_DCB_MAX_TRAFFIC_CLASS); i++) {
4207 if ((mask & 0x1) && j < RTE_ETH_DCB_NUM_USER_PRIORITIES)
4211 /* Re-configure 4 TCs BW */
4212 for (i = 0; i < nb_tcs; i++) {
4213 tc = &dcb_config->tc_config[i];
4214 if (bw_conf->tc_num != nb_tcs)
4215 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent =
4216 (uint8_t)(100 / nb_tcs);
4217 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent =
4218 (uint8_t)(100 / nb_tcs);
4220 for (; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
4221 tc = &dcb_config->tc_config[i];
4222 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent = 0;
4223 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent = 0;
4226 /* Re-configure 8 TCs BW */
4227 for (i = 0; i < nb_tcs; i++) {
4228 tc = &dcb_config->tc_config[i];
4229 if (bw_conf->tc_num != nb_tcs)
4230 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent =
4231 (uint8_t)(100 / nb_tcs + (i & 1));
4232 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent =
4233 (uint8_t)(100 / nb_tcs + (i & 1));
4237 switch (hw->mac.type) {
4238 case ixgbe_mac_X550:
4239 case ixgbe_mac_X550EM_x:
4240 case ixgbe_mac_X550EM_a:
4241 rx_buffer_size = X550_RX_BUFFER_SIZE;
4244 rx_buffer_size = NIC_RX_BUFFER_SIZE;
4248 if (config_dcb_rx) {
4249 /* Set RX buffer size */
4250 pbsize = (uint16_t)(rx_buffer_size / nb_tcs);
4251 uint32_t rxpbsize = pbsize << IXGBE_RXPBSIZE_SHIFT;
4253 for (i = 0; i < nb_tcs; i++) {
4254 IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), rxpbsize);
4256 /* zero alloc all unused TCs */
4257 for (; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++)
4258 IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), 0);
4260 if (config_dcb_tx) {
4261 /* Only support an equally distributed
4262 * Tx packet buffer strategy.
4264 uint32_t txpktsize = IXGBE_TXPBSIZE_MAX / nb_tcs;
4265 uint32_t txpbthresh = (txpktsize / DCB_TX_PB) - IXGBE_TXPKT_SIZE_MAX;
4267 for (i = 0; i < nb_tcs; i++) {
4268 IXGBE_WRITE_REG(hw, IXGBE_TXPBSIZE(i), txpktsize);
4269 IXGBE_WRITE_REG(hw, IXGBE_TXPBTHRESH(i), txpbthresh);
4271 /* Clear unused TCs, if any, to zero buffer size*/
4272 for (; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
4273 IXGBE_WRITE_REG(hw, IXGBE_TXPBSIZE(i), 0);
4274 IXGBE_WRITE_REG(hw, IXGBE_TXPBTHRESH(i), 0);
4278 /*Calculates traffic class credits*/
4279 ixgbe_dcb_calculate_tc_credits_cee(hw, dcb_config, max_frame,
4280 IXGBE_DCB_TX_CONFIG);
4281 ixgbe_dcb_calculate_tc_credits_cee(hw, dcb_config, max_frame,
4282 IXGBE_DCB_RX_CONFIG);
4284 if (config_dcb_rx) {
4285 /* Unpack CEE standard containers */
4286 ixgbe_dcb_unpack_refill_cee(dcb_config, IXGBE_DCB_RX_CONFIG, refill);
4287 ixgbe_dcb_unpack_max_cee(dcb_config, max);
4288 ixgbe_dcb_unpack_bwgid_cee(dcb_config, IXGBE_DCB_RX_CONFIG, bwgid);
4289 ixgbe_dcb_unpack_tsa_cee(dcb_config, IXGBE_DCB_RX_CONFIG, tsa);
4290 /* Configure PG(ETS) RX */
4291 ixgbe_dcb_hw_arbite_rx_config(hw, refill, max, bwgid, tsa, map);
4294 if (config_dcb_tx) {
4295 /* Unpack CEE standard containers */
4296 ixgbe_dcb_unpack_refill_cee(dcb_config, IXGBE_DCB_TX_CONFIG, refill);
4297 ixgbe_dcb_unpack_max_cee(dcb_config, max);
4298 ixgbe_dcb_unpack_bwgid_cee(dcb_config, IXGBE_DCB_TX_CONFIG, bwgid);
4299 ixgbe_dcb_unpack_tsa_cee(dcb_config, IXGBE_DCB_TX_CONFIG, tsa);
4300 /* Configure PG(ETS) TX */
4301 ixgbe_dcb_hw_arbite_tx_config(hw, refill, max, bwgid, tsa, map);
4304 /*Configure queue statistics registers*/
4305 ixgbe_dcb_config_tc_stats_82599(hw, dcb_config);
4307 /* Check if the PFC is supported */
4308 if (dev->data->dev_conf.dcb_capability_en & RTE_ETH_DCB_PFC_SUPPORT) {
4309 pbsize = (uint16_t)(rx_buffer_size / nb_tcs);
4310 for (i = 0; i < nb_tcs; i++) {
4312 * If the TC count is 8,and the default high_water is 48,
4313 * the low_water is 16 as default.
4315 hw->fc.high_water[i] = (pbsize * 3) / 4;
4316 hw->fc.low_water[i] = pbsize / 4;
4317 /* Enable pfc for this TC */
4318 tc = &dcb_config->tc_config[i];
4319 tc->pfc = ixgbe_dcb_pfc_enabled;
4321 ixgbe_dcb_unpack_pfc_cee(dcb_config, map, &pfc_en);
4322 if (dcb_config->num_tcs.pfc_tcs == RTE_ETH_4_TCS)
4324 ret = ixgbe_dcb_config_pfc(hw, pfc_en, map);
4331 * ixgbe_configure_dcb - Configure DCB Hardware
4332 * @dev: pointer to rte_eth_dev
4334 void ixgbe_configure_dcb(struct rte_eth_dev *dev)
4336 struct ixgbe_dcb_config *dcb_cfg =
4337 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private);
4338 struct rte_eth_conf *dev_conf = &(dev->data->dev_conf);
4340 PMD_INIT_FUNC_TRACE();
4342 /* check support mq_mode for DCB */
4343 if (dev_conf->rxmode.mq_mode != RTE_ETH_MQ_RX_VMDQ_DCB &&
4344 dev_conf->rxmode.mq_mode != RTE_ETH_MQ_RX_DCB &&
4345 dev_conf->rxmode.mq_mode != RTE_ETH_MQ_RX_DCB_RSS)
4348 if (dev->data->nb_rx_queues > RTE_ETH_DCB_NUM_QUEUES)
4351 /** Configure DCB hardware **/
4352 ixgbe_dcb_hw_configure(dev, dcb_cfg);
4356 * VMDq only support for 10 GbE NIC.
4359 ixgbe_vmdq_rx_hw_configure(struct rte_eth_dev *dev)
4361 struct rte_eth_vmdq_rx_conf *cfg;
4362 struct ixgbe_hw *hw;
4363 enum rte_eth_nb_pools num_pools;
4364 uint32_t mrqc, vt_ctl, vlanctrl;
4368 PMD_INIT_FUNC_TRACE();
4369 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4370 cfg = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
4371 num_pools = cfg->nb_queue_pools;
4373 ixgbe_rss_disable(dev);
4375 /* MRQC: enable vmdq */
4376 mrqc = IXGBE_MRQC_VMDQEN;
4377 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4379 /* PFVTCTL: turn on virtualisation and set the default pool */
4380 vt_ctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
4381 if (cfg->enable_default_pool)
4382 vt_ctl |= (cfg->default_pool << IXGBE_VT_CTL_POOL_SHIFT);
4384 vt_ctl |= IXGBE_VT_CTL_DIS_DEFPL;
4386 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vt_ctl);
4388 for (i = 0; i < (int)num_pools; i++) {
4389 vmolr = ixgbe_convert_vm_rx_mask_to_val(cfg->rx_mode, vmolr);
4390 IXGBE_WRITE_REG(hw, IXGBE_VMOLR(i), vmolr);
4393 /* VLNCTRL: enable vlan filtering and allow all vlan tags through */
4394 vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4395 vlanctrl |= IXGBE_VLNCTRL_VFE; /* enable vlan filters */
4396 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl);
4398 /* VFTA - enable all vlan filters */
4399 for (i = 0; i < NUM_VFTA_REGISTERS; i++)
4400 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), UINT32_MAX);
4402 /* VFRE: pool enabling for receive - 64 */
4403 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), UINT32_MAX);
4404 if (num_pools == RTE_ETH_64_POOLS)
4405 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), UINT32_MAX);
4408 * MPSAR - allow pools to read specific mac addresses
4409 * In this case, all pools should be able to read from mac addr 0
4411 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(0), UINT32_MAX);
4412 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(0), UINT32_MAX);
4414 /* PFVLVF, PFVLVFB: set up filters for vlan tags as configured */
4415 for (i = 0; i < cfg->nb_pool_maps; i++) {
4416 /* set vlan id in VF register and set the valid bit */
4417 IXGBE_WRITE_REG(hw, IXGBE_VLVF(i), (IXGBE_VLVF_VIEN |
4418 (cfg->pool_map[i].vlan_id & IXGBE_RXD_VLAN_ID_MASK)));
4420 * Put the allowed pools in VFB reg. As we only have 16 or 64
4421 * pools, we only need to use the first half of the register
4424 if (((cfg->pool_map[i].pools >> 32) & UINT32_MAX) == 0)
4425 IXGBE_WRITE_REG(hw, IXGBE_VLVFB(i * 2),
4426 (cfg->pool_map[i].pools & UINT32_MAX));
4428 IXGBE_WRITE_REG(hw, IXGBE_VLVFB((i * 2 + 1)),
4429 ((cfg->pool_map[i].pools >> 32) & UINT32_MAX));
4433 /* PFDMA Tx General Switch Control Enables VMDQ loopback */
4434 if (cfg->enable_loop_back) {
4435 IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN);
4436 for (i = 0; i < RTE_IXGBE_VMTXSW_REGISTER_COUNT; i++)
4437 IXGBE_WRITE_REG(hw, IXGBE_VMTXSW(i), UINT32_MAX);
4440 IXGBE_WRITE_FLUSH(hw);
4444 * ixgbe_dcb_config_tx_hw_config - Configure general VMDq TX parameters
4445 * @hw: pointer to hardware structure
4448 ixgbe_vmdq_tx_hw_configure(struct ixgbe_hw *hw)
4453 PMD_INIT_FUNC_TRACE();
4454 /*PF VF Transmit Enable*/
4455 IXGBE_WRITE_REG(hw, IXGBE_VFTE(0), UINT32_MAX);
4456 IXGBE_WRITE_REG(hw, IXGBE_VFTE(1), UINT32_MAX);
4458 /* Disable the Tx desc arbiter so that MTQC can be changed */
4459 reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
4460 reg |= IXGBE_RTTDCS_ARBDIS;
4461 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg);
4463 reg = IXGBE_MTQC_VT_ENA | IXGBE_MTQC_64VF;
4464 IXGBE_WRITE_REG(hw, IXGBE_MTQC, reg);
4466 /* Disable drop for all queues */
4467 for (q = 0; q < IXGBE_MAX_RX_QUEUE_NUM; q++)
4468 IXGBE_WRITE_REG(hw, IXGBE_QDE,
4469 (IXGBE_QDE_WRITE | (q << IXGBE_QDE_IDX_SHIFT)));
4471 /* Enable the Tx desc arbiter */
4472 reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
4473 reg &= ~IXGBE_RTTDCS_ARBDIS;
4474 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg);
4476 IXGBE_WRITE_FLUSH(hw);
4479 static int __rte_cold
4480 ixgbe_alloc_rx_queue_mbufs(struct ixgbe_rx_queue *rxq)
4482 struct ixgbe_rx_entry *rxe = rxq->sw_ring;
4486 /* Initialize software ring entries */
4487 for (i = 0; i < rxq->nb_rx_desc; i++) {
4488 volatile union ixgbe_adv_rx_desc *rxd;
4489 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mb_pool);
4492 PMD_INIT_LOG(ERR, "RX mbuf alloc failed queue_id=%u",
4493 (unsigned) rxq->queue_id);
4497 mbuf->data_off = RTE_PKTMBUF_HEADROOM;
4498 mbuf->port = rxq->port_id;
4501 rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf));
4502 rxd = &rxq->rx_ring[i];
4503 rxd->read.hdr_addr = 0;
4504 rxd->read.pkt_addr = dma_addr;
4512 ixgbe_config_vf_rss(struct rte_eth_dev *dev)
4514 struct ixgbe_hw *hw;
4517 ixgbe_rss_configure(dev);
4519 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4521 /* MRQC: enable VF RSS */
4522 mrqc = IXGBE_READ_REG(hw, IXGBE_MRQC);
4523 mrqc &= ~IXGBE_MRQC_MRQE_MASK;
4524 switch (RTE_ETH_DEV_SRIOV(dev).active) {
4525 case RTE_ETH_64_POOLS:
4526 mrqc |= IXGBE_MRQC_VMDQRSS64EN;
4529 case RTE_ETH_32_POOLS:
4530 mrqc |= IXGBE_MRQC_VMDQRSS32EN;
4534 PMD_INIT_LOG(ERR, "Invalid pool number in IOV mode with VMDQ RSS");
4538 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4544 ixgbe_config_vf_default(struct rte_eth_dev *dev)
4546 struct ixgbe_hw *hw =
4547 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4549 switch (RTE_ETH_DEV_SRIOV(dev).active) {
4550 case RTE_ETH_64_POOLS:
4551 IXGBE_WRITE_REG(hw, IXGBE_MRQC,
4555 case RTE_ETH_32_POOLS:
4556 IXGBE_WRITE_REG(hw, IXGBE_MRQC,
4557 IXGBE_MRQC_VMDQRT4TCEN);
4560 case RTE_ETH_16_POOLS:
4561 IXGBE_WRITE_REG(hw, IXGBE_MRQC,
4562 IXGBE_MRQC_VMDQRT8TCEN);
4566 "invalid pool number in IOV mode");
4573 ixgbe_dev_mq_rx_configure(struct rte_eth_dev *dev)
4575 struct ixgbe_hw *hw =
4576 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4578 if (hw->mac.type == ixgbe_mac_82598EB)
4581 if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
4583 * SRIOV inactive scheme
4584 * any DCB/RSS w/o VMDq multi-queue setting
4586 switch (dev->data->dev_conf.rxmode.mq_mode) {
4587 case RTE_ETH_MQ_RX_RSS:
4588 case RTE_ETH_MQ_RX_DCB_RSS:
4589 case RTE_ETH_MQ_RX_VMDQ_RSS:
4590 ixgbe_rss_configure(dev);
4593 case RTE_ETH_MQ_RX_VMDQ_DCB:
4594 ixgbe_vmdq_dcb_configure(dev);
4597 case RTE_ETH_MQ_RX_VMDQ_ONLY:
4598 ixgbe_vmdq_rx_hw_configure(dev);
4601 case RTE_ETH_MQ_RX_NONE:
4603 /* if mq_mode is none, disable rss mode.*/
4604 ixgbe_rss_disable(dev);
4608 /* SRIOV active scheme
4609 * Support RSS together with SRIOV.
4611 switch (dev->data->dev_conf.rxmode.mq_mode) {
4612 case RTE_ETH_MQ_RX_RSS:
4613 case RTE_ETH_MQ_RX_VMDQ_RSS:
4614 ixgbe_config_vf_rss(dev);
4616 case RTE_ETH_MQ_RX_VMDQ_DCB:
4617 case RTE_ETH_MQ_RX_DCB:
4618 /* In SRIOV, the configuration is the same as VMDq case */
4619 ixgbe_vmdq_dcb_configure(dev);
4621 /* DCB/RSS together with SRIOV is not supported */
4622 case RTE_ETH_MQ_RX_VMDQ_DCB_RSS:
4623 case RTE_ETH_MQ_RX_DCB_RSS:
4625 "Could not support DCB/RSS with VMDq & SRIOV");
4628 ixgbe_config_vf_default(dev);
4637 ixgbe_dev_mq_tx_configure(struct rte_eth_dev *dev)
4639 struct ixgbe_hw *hw =
4640 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4644 if (hw->mac.type == ixgbe_mac_82598EB)
4647 /* disable arbiter before setting MTQC */
4648 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
4649 rttdcs |= IXGBE_RTTDCS_ARBDIS;
4650 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
4652 if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
4654 * SRIOV inactive scheme
4655 * any DCB w/o VMDq multi-queue setting
4657 if (dev->data->dev_conf.txmode.mq_mode == RTE_ETH_MQ_TX_VMDQ_ONLY)
4658 ixgbe_vmdq_tx_hw_configure(hw);
4660 mtqc = IXGBE_MTQC_64Q_1PB;
4661 IXGBE_WRITE_REG(hw, IXGBE_MTQC, mtqc);
4664 switch (RTE_ETH_DEV_SRIOV(dev).active) {
4667 * SRIOV active scheme
4668 * FIXME if support DCB together with VMDq & SRIOV
4670 case RTE_ETH_64_POOLS:
4671 mtqc = IXGBE_MTQC_VT_ENA | IXGBE_MTQC_64VF;
4673 case RTE_ETH_32_POOLS:
4674 mtqc = IXGBE_MTQC_VT_ENA | IXGBE_MTQC_32VF;
4676 case RTE_ETH_16_POOLS:
4677 mtqc = IXGBE_MTQC_VT_ENA | IXGBE_MTQC_RT_ENA |
4681 mtqc = IXGBE_MTQC_64Q_1PB;
4682 PMD_INIT_LOG(ERR, "invalid pool number in IOV mode");
4684 IXGBE_WRITE_REG(hw, IXGBE_MTQC, mtqc);
4687 /* re-enable arbiter */
4688 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
4689 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
4695 * ixgbe_get_rscctl_maxdesc - Calculate the RSCCTL[n].MAXDESC for PF
4697 * Return the RSCCTL[n].MAXDESC for 82599 and x540 PF devices according to the
4698 * spec rev. 3.0 chapter 8.2.3.8.13.
4700 * @pool Memory pool of the Rx queue
4702 static inline uint32_t
4703 ixgbe_get_rscctl_maxdesc(struct rte_mempool *pool)
4705 struct rte_pktmbuf_pool_private *mp_priv = rte_mempool_get_priv(pool);
4707 /* MAXDESC * SRRCTL.BSIZEPKT must not exceed 64 KB minus one */
4709 RTE_IPV4_MAX_PKT_LEN /
4710 (mp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM);
4713 return IXGBE_RSCCTL_MAXDESC_16;
4714 else if (maxdesc >= 8)
4715 return IXGBE_RSCCTL_MAXDESC_8;
4716 else if (maxdesc >= 4)
4717 return IXGBE_RSCCTL_MAXDESC_4;
4719 return IXGBE_RSCCTL_MAXDESC_1;
4723 * ixgbe_set_ivar - Setup the correct IVAR register for a particular MSIX
4726 * (Taken from FreeBSD tree)
4727 * (yes this is all very magic and confusing :)
4730 * @entry the register array entry
4731 * @vector the MSIX vector for this queue
4735 ixgbe_set_ivar(struct rte_eth_dev *dev, u8 entry, u8 vector, s8 type)
4737 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4740 vector |= IXGBE_IVAR_ALLOC_VAL;
4742 switch (hw->mac.type) {
4744 case ixgbe_mac_82598EB:
4746 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
4748 entry += (type * 64);
4749 index = (entry >> 2) & 0x1F;
4750 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4751 ivar &= ~(0xFF << (8 * (entry & 0x3)));
4752 ivar |= (vector << (8 * (entry & 0x3)));
4753 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
4756 case ixgbe_mac_82599EB:
4757 case ixgbe_mac_X540:
4758 if (type == -1) { /* MISC IVAR */
4759 index = (entry & 1) * 8;
4760 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4761 ivar &= ~(0xFF << index);
4762 ivar |= (vector << index);
4763 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4764 } else { /* RX/TX IVARS */
4765 index = (16 * (entry & 1)) + (8 * type);
4766 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
4767 ivar &= ~(0xFF << index);
4768 ivar |= (vector << index);
4769 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
4780 ixgbe_set_rx_function(struct rte_eth_dev *dev)
4782 uint16_t i, rx_using_sse;
4783 struct ixgbe_adapter *adapter = dev->data->dev_private;
4786 * In order to allow Vector Rx there are a few configuration
4787 * conditions to be met and Rx Bulk Allocation should be allowed.
4789 if (ixgbe_rx_vec_dev_conf_condition_check(dev) ||
4790 !adapter->rx_bulk_alloc_allowed ||
4791 rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128) {
4792 PMD_INIT_LOG(DEBUG, "Port[%d] doesn't meet Vector Rx "
4794 dev->data->port_id);
4796 adapter->rx_vec_allowed = false;
4800 * Initialize the appropriate LRO callback.
4802 * If all queues satisfy the bulk allocation preconditions
4803 * (hw->rx_bulk_alloc_allowed is TRUE) then we may use bulk allocation.
4804 * Otherwise use a single allocation version.
4806 if (dev->data->lro) {
4807 if (adapter->rx_bulk_alloc_allowed) {
4808 PMD_INIT_LOG(DEBUG, "LRO is requested. Using a bulk "
4809 "allocation version");
4810 dev->rx_pkt_burst = ixgbe_recv_pkts_lro_bulk_alloc;
4812 PMD_INIT_LOG(DEBUG, "LRO is requested. Using a single "
4813 "allocation version");
4814 dev->rx_pkt_burst = ixgbe_recv_pkts_lro_single_alloc;
4816 } else if (dev->data->scattered_rx) {
4818 * Set the non-LRO scattered callback: there are Vector and
4819 * single allocation versions.
4821 if (adapter->rx_vec_allowed) {
4822 PMD_INIT_LOG(DEBUG, "Using Vector Scattered Rx "
4823 "callback (port=%d).",
4824 dev->data->port_id);
4826 dev->rx_pkt_burst = ixgbe_recv_scattered_pkts_vec;
4827 } else if (adapter->rx_bulk_alloc_allowed) {
4828 PMD_INIT_LOG(DEBUG, "Using a Scattered with bulk "
4829 "allocation callback (port=%d).",
4830 dev->data->port_id);
4831 dev->rx_pkt_burst = ixgbe_recv_pkts_lro_bulk_alloc;
4833 PMD_INIT_LOG(DEBUG, "Using Regular (non-vector, "
4834 "single allocation) "
4835 "Scattered Rx callback "
4837 dev->data->port_id);
4839 dev->rx_pkt_burst = ixgbe_recv_pkts_lro_single_alloc;
4842 * Below we set "simple" callbacks according to port/queues parameters.
4843 * If parameters allow we are going to choose between the following
4847 * - Single buffer allocation (the simplest one)
4849 } else if (adapter->rx_vec_allowed) {
4850 PMD_INIT_LOG(DEBUG, "Vector rx enabled, please make sure RX "
4851 "burst size no less than %d (port=%d).",
4852 RTE_IXGBE_DESCS_PER_LOOP,
4853 dev->data->port_id);
4855 dev->rx_pkt_burst = ixgbe_recv_pkts_vec;
4856 } else if (adapter->rx_bulk_alloc_allowed) {
4857 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are "
4858 "satisfied. Rx Burst Bulk Alloc function "
4859 "will be used on port=%d.",
4860 dev->data->port_id);
4862 dev->rx_pkt_burst = ixgbe_recv_pkts_bulk_alloc;
4864 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are not "
4865 "satisfied, or Scattered Rx is requested "
4867 dev->data->port_id);
4869 dev->rx_pkt_burst = ixgbe_recv_pkts;
4872 /* Propagate information about RX function choice through all queues. */
4875 (dev->rx_pkt_burst == ixgbe_recv_scattered_pkts_vec ||
4876 dev->rx_pkt_burst == ixgbe_recv_pkts_vec);
4878 for (i = 0; i < dev->data->nb_rx_queues; i++) {
4879 struct ixgbe_rx_queue *rxq = dev->data->rx_queues[i];
4881 rxq->rx_using_sse = rx_using_sse;
4882 #ifdef RTE_LIB_SECURITY
4883 rxq->using_ipsec = !!(dev->data->dev_conf.rxmode.offloads &
4884 RTE_ETH_RX_OFFLOAD_SECURITY);
4890 * ixgbe_set_rsc - configure RSC related port HW registers
4892 * Configures the port's RSC related registers according to the 4.6.7.2 chapter
4893 * of 82599 Spec (x540 configuration is virtually the same).
4897 * Returns 0 in case of success or a non-zero error code
4900 ixgbe_set_rsc(struct rte_eth_dev *dev)
4902 struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode;
4903 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4904 struct rte_eth_dev_info dev_info = { 0 };
4905 bool rsc_capable = false;
4911 dev->dev_ops->dev_infos_get(dev, &dev_info);
4912 if (dev_info.rx_offload_capa & RTE_ETH_RX_OFFLOAD_TCP_LRO)
4915 if (!rsc_capable && (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO)) {
4916 PMD_INIT_LOG(CRIT, "LRO is requested on HW that doesn't "
4921 /* RSC global configuration (chapter 4.6.7.2.1 of 82599 Spec) */
4923 if ((rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) &&
4924 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO)) {
4926 * According to chapter of 4.6.7.2.1 of the Spec Rev.
4927 * 3.0 RSC configuration requires HW CRC stripping being
4928 * enabled. If user requested both HW CRC stripping off
4929 * and RSC on - return an error.
4931 PMD_INIT_LOG(CRIT, "LRO can't be enabled when HW CRC "
4936 /* RFCTL configuration */
4937 rfctl = IXGBE_READ_REG(hw, IXGBE_RFCTL);
4938 if ((rsc_capable) && (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO))
4939 rfctl &= ~IXGBE_RFCTL_RSC_DIS;
4941 rfctl |= IXGBE_RFCTL_RSC_DIS;
4942 /* disable NFS filtering */
4943 rfctl |= IXGBE_RFCTL_NFSW_DIS | IXGBE_RFCTL_NFSR_DIS;
4944 IXGBE_WRITE_REG(hw, IXGBE_RFCTL, rfctl);
4946 /* If LRO hasn't been requested - we are done here. */
4947 if (!(rx_conf->offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO))
4950 /* Set RDRXCTL.RSCACKC bit */
4951 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
4952 rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
4953 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
4955 /* Per-queue RSC configuration (chapter 4.6.7.2.2 of 82599 Spec) */
4956 for (i = 0; i < dev->data->nb_rx_queues; i++) {
4957 struct ixgbe_rx_queue *rxq = dev->data->rx_queues[i];
4959 IXGBE_READ_REG(hw, IXGBE_SRRCTL(rxq->reg_idx));
4961 IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxq->reg_idx));
4963 IXGBE_READ_REG(hw, IXGBE_PSRTYPE(rxq->reg_idx));
4965 IXGBE_READ_REG(hw, IXGBE_EITR(rxq->reg_idx));
4968 * ixgbe PMD doesn't support header-split at the moment.
4970 * Following the 4.6.7.2.1 chapter of the 82599/x540
4971 * Spec if RSC is enabled the SRRCTL[n].BSIZEHEADER
4972 * should be configured even if header split is not
4973 * enabled. We will configure it 128 bytes following the
4974 * recommendation in the spec.
4976 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
4977 srrctl |= (128 << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) &
4978 IXGBE_SRRCTL_BSIZEHDR_MASK;
4981 * TODO: Consider setting the Receive Descriptor Minimum
4982 * Threshold Size for an RSC case. This is not an obviously
4983 * beneficiary option but the one worth considering...
4986 rscctl |= IXGBE_RSCCTL_RSCEN;
4987 rscctl |= ixgbe_get_rscctl_maxdesc(rxq->mb_pool);
4988 psrtype |= IXGBE_PSRTYPE_TCPHDR;
4991 * RSC: Set ITR interval corresponding to 2K ints/s.
4993 * Full-sized RSC aggregations for a 10Gb/s link will
4994 * arrive at about 20K aggregation/s rate.
4996 * 2K inst/s rate will make only 10% of the
4997 * aggregations to be closed due to the interrupt timer
4998 * expiration for a streaming at wire-speed case.
5000 * For a sparse streaming case this setting will yield
5001 * at most 500us latency for a single RSC aggregation.
5003 eitr &= ~IXGBE_EITR_ITR_INT_MASK;
5004 eitr |= IXGBE_EITR_INTERVAL_US(IXGBE_QUEUE_ITR_INTERVAL_DEFAULT);
5005 eitr |= IXGBE_EITR_CNT_WDIS;
5007 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxq->reg_idx), srrctl);
5008 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxq->reg_idx), rscctl);
5009 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(rxq->reg_idx), psrtype);
5010 IXGBE_WRITE_REG(hw, IXGBE_EITR(rxq->reg_idx), eitr);
5013 * RSC requires the mapping of the queue to the
5016 ixgbe_set_ivar(dev, rxq->reg_idx, i, 0);
5021 PMD_INIT_LOG(DEBUG, "enabling LRO mode");
5027 * Initializes Receive Unit.
5030 ixgbe_dev_rx_init(struct rte_eth_dev *dev)
5032 struct ixgbe_hw *hw;
5033 struct ixgbe_rx_queue *rxq;
5044 struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode;
5045 uint32_t frame_size = dev->data->mtu + IXGBE_ETH_OVERHEAD;
5048 PMD_INIT_FUNC_TRACE();
5049 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5052 * Make sure receives are disabled while setting
5053 * up the RX context (registers, descriptor rings, etc.).
5055 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
5056 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl & ~IXGBE_RXCTRL_RXEN);
5058 /* Enable receipt of broadcasted frames */
5059 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
5060 fctrl |= IXGBE_FCTRL_BAM;
5061 fctrl |= IXGBE_FCTRL_DPF;
5062 fctrl |= IXGBE_FCTRL_PMCF;
5063 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
5066 * Configure CRC stripping, if any.
5068 hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
5069 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC)
5070 hlreg0 &= ~IXGBE_HLREG0_RXCRCSTRP;
5072 hlreg0 |= IXGBE_HLREG0_RXCRCSTRP;
5075 * Configure jumbo frame support, if any.
5077 if (dev->data->mtu > RTE_ETHER_MTU) {
5078 hlreg0 |= IXGBE_HLREG0_JUMBOEN;
5079 maxfrs = IXGBE_READ_REG(hw, IXGBE_MAXFRS);
5080 maxfrs &= 0x0000FFFF;
5081 maxfrs |= (frame_size << 16);
5082 IXGBE_WRITE_REG(hw, IXGBE_MAXFRS, maxfrs);
5084 hlreg0 &= ~IXGBE_HLREG0_JUMBOEN;
5087 * If loopback mode is configured, set LPBK bit.
5089 if (dev->data->dev_conf.lpbk_mode != 0) {
5090 rc = ixgbe_check_supported_loopback_mode(dev);
5092 PMD_INIT_LOG(ERR, "Unsupported loopback mode");
5095 hlreg0 |= IXGBE_HLREG0_LPBK;
5097 hlreg0 &= ~IXGBE_HLREG0_LPBK;
5100 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);
5103 * Assume no header split and no VLAN strip support
5104 * on any Rx queue first .
5106 rx_conf->offloads &= ~RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
5107 /* Setup RX queues */
5108 for (i = 0; i < dev->data->nb_rx_queues; i++) {
5109 rxq = dev->data->rx_queues[i];
5112 * Reset crc_len in case it was changed after queue setup by a
5113 * call to configure.
5115 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC)
5116 rxq->crc_len = RTE_ETHER_CRC_LEN;
5120 /* Setup the Base and Length of the Rx Descriptor Rings */
5121 bus_addr = rxq->rx_ring_phys_addr;
5122 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(rxq->reg_idx),
5123 (uint32_t)(bus_addr & 0x00000000ffffffffULL));
5124 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(rxq->reg_idx),
5125 (uint32_t)(bus_addr >> 32));
5126 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(rxq->reg_idx),
5127 rxq->nb_rx_desc * sizeof(union ixgbe_adv_rx_desc));
5128 IXGBE_WRITE_REG(hw, IXGBE_RDH(rxq->reg_idx), 0);
5129 IXGBE_WRITE_REG(hw, IXGBE_RDT(rxq->reg_idx), 0);
5131 /* Configure the SRRCTL register */
5132 srrctl = IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
5134 /* Set if packets are dropped when no descriptors available */
5136 srrctl |= IXGBE_SRRCTL_DROP_EN;
5139 * Configure the RX buffer size in the BSIZEPACKET field of
5140 * the SRRCTL register of the queue.
5141 * The value is in 1 KB resolution. Valid values can be from
5144 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) -
5145 RTE_PKTMBUF_HEADROOM);
5146 srrctl |= ((buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) &
5147 IXGBE_SRRCTL_BSIZEPKT_MASK);
5149 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxq->reg_idx), srrctl);
5151 buf_size = (uint16_t) ((srrctl & IXGBE_SRRCTL_BSIZEPKT_MASK) <<
5152 IXGBE_SRRCTL_BSIZEPKT_SHIFT);
5154 /* It adds dual VLAN length for supporting dual VLAN */
5155 if (frame_size + 2 * RTE_VLAN_HLEN > buf_size)
5156 dev->data->scattered_rx = 1;
5157 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP)
5158 rx_conf->offloads |= RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
5161 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_SCATTER)
5162 dev->data->scattered_rx = 1;
5165 * Device configured with multiple RX queues.
5167 ixgbe_dev_mq_rx_configure(dev);
5170 * Setup the Checksum Register.
5171 * Disable Full-Packet Checksum which is mutually exclusive with RSS.
5172 * Enable IP/L4 checksum computation by hardware if requested to do so.
5174 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
5175 rxcsum |= IXGBE_RXCSUM_PCSD;
5176 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_CHECKSUM)
5177 rxcsum |= IXGBE_RXCSUM_IPPCSE;
5179 rxcsum &= ~IXGBE_RXCSUM_IPPCSE;
5181 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
5183 if (hw->mac.type == ixgbe_mac_82599EB ||
5184 hw->mac.type == ixgbe_mac_X540) {
5185 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
5186 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC)
5187 rdrxctl &= ~IXGBE_RDRXCTL_CRCSTRIP;
5189 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
5190 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
5191 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
5194 rc = ixgbe_set_rsc(dev);
5198 ixgbe_set_rx_function(dev);
5204 * Initializes Transmit Unit.
5207 ixgbe_dev_tx_init(struct rte_eth_dev *dev)
5209 struct ixgbe_hw *hw;
5210 struct ixgbe_tx_queue *txq;
5216 PMD_INIT_FUNC_TRACE();
5217 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5219 /* Enable TX CRC (checksum offload requirement) and hw padding
5222 hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
5223 hlreg0 |= (IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN);
5224 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);
5226 /* Setup the Base and Length of the Tx Descriptor Rings */
5227 for (i = 0; i < dev->data->nb_tx_queues; i++) {
5228 txq = dev->data->tx_queues[i];
5230 bus_addr = txq->tx_ring_phys_addr;
5231 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(txq->reg_idx),
5232 (uint32_t)(bus_addr & 0x00000000ffffffffULL));
5233 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(txq->reg_idx),
5234 (uint32_t)(bus_addr >> 32));
5235 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(txq->reg_idx),
5236 txq->nb_tx_desc * sizeof(union ixgbe_adv_tx_desc));
5237 /* Setup the HW Tx Head and TX Tail descriptor pointers */
5238 IXGBE_WRITE_REG(hw, IXGBE_TDH(txq->reg_idx), 0);
5239 IXGBE_WRITE_REG(hw, IXGBE_TDT(txq->reg_idx), 0);
5242 * Disable Tx Head Writeback RO bit, since this hoses
5243 * bookkeeping if things aren't delivered in order.
5245 switch (hw->mac.type) {
5246 case ixgbe_mac_82598EB:
5247 txctrl = IXGBE_READ_REG(hw,
5248 IXGBE_DCA_TXCTRL(txq->reg_idx));
5249 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
5250 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(txq->reg_idx),
5254 case ixgbe_mac_82599EB:
5255 case ixgbe_mac_X540:
5256 case ixgbe_mac_X550:
5257 case ixgbe_mac_X550EM_x:
5258 case ixgbe_mac_X550EM_a:
5260 txctrl = IXGBE_READ_REG(hw,
5261 IXGBE_DCA_TXCTRL_82599(txq->reg_idx));
5262 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
5263 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(txq->reg_idx),
5269 /* Device configured with multiple TX queues. */
5270 ixgbe_dev_mq_tx_configure(dev);
5274 * Check if requested loopback mode is supported
5277 ixgbe_check_supported_loopback_mode(struct rte_eth_dev *dev)
5279 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5281 if (dev->data->dev_conf.lpbk_mode == IXGBE_LPBK_TX_RX)
5282 if (hw->mac.type == ixgbe_mac_82599EB ||
5283 hw->mac.type == ixgbe_mac_X540 ||
5284 hw->mac.type == ixgbe_mac_X550 ||
5285 hw->mac.type == ixgbe_mac_X550EM_x ||
5286 hw->mac.type == ixgbe_mac_X550EM_a)
5293 * Set up link for 82599 loopback mode Tx->Rx.
5295 static inline void __rte_cold
5296 ixgbe_setup_loopback_link_82599(struct ixgbe_hw *hw)
5298 PMD_INIT_FUNC_TRACE();
5300 if (ixgbe_verify_lesm_fw_enabled_82599(hw)) {
5301 if (hw->mac.ops.acquire_swfw_sync(hw, IXGBE_GSSR_MAC_CSR_SM) !=
5303 PMD_INIT_LOG(ERR, "Could not enable loopback mode");
5312 IXGBE_AUTOC_LMS_10G_LINK_NO_AN | IXGBE_AUTOC_FLU);
5313 ixgbe_reset_pipeline_82599(hw);
5315 hw->mac.ops.release_swfw_sync(hw, IXGBE_GSSR_MAC_CSR_SM);
5321 * Start Transmit and Receive Units.
5324 ixgbe_dev_rxtx_start(struct rte_eth_dev *dev)
5326 struct ixgbe_hw *hw;
5327 struct ixgbe_tx_queue *txq;
5328 struct ixgbe_rx_queue *rxq;
5335 PMD_INIT_FUNC_TRACE();
5336 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5338 for (i = 0; i < dev->data->nb_tx_queues; i++) {
5339 txq = dev->data->tx_queues[i];
5340 /* Setup Transmit Threshold Registers */
5341 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txq->reg_idx));
5342 txdctl |= txq->pthresh & 0x7F;
5343 txdctl |= ((txq->hthresh & 0x7F) << 8);
5344 txdctl |= ((txq->wthresh & 0x7F) << 16);
5345 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txq->reg_idx), txdctl);
5348 if (hw->mac.type != ixgbe_mac_82598EB) {
5349 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
5350 dmatxctl |= IXGBE_DMATXCTL_TE;
5351 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
5354 for (i = 0; i < dev->data->nb_tx_queues; i++) {
5355 txq = dev->data->tx_queues[i];
5356 if (!txq->tx_deferred_start) {
5357 ret = ixgbe_dev_tx_queue_start(dev, i);
5363 for (i = 0; i < dev->data->nb_rx_queues; i++) {
5364 rxq = dev->data->rx_queues[i];
5365 if (!rxq->rx_deferred_start) {
5366 ret = ixgbe_dev_rx_queue_start(dev, i);
5372 /* Enable Receive engine */
5373 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
5374 if (hw->mac.type == ixgbe_mac_82598EB)
5375 rxctrl |= IXGBE_RXCTRL_DMBYPS;
5376 rxctrl |= IXGBE_RXCTRL_RXEN;
5377 hw->mac.ops.enable_rx_dma(hw, rxctrl);
5379 /* If loopback mode is enabled, set up the link accordingly */
5380 if (dev->data->dev_conf.lpbk_mode != 0) {
5381 if (hw->mac.type == ixgbe_mac_82599EB)
5382 ixgbe_setup_loopback_link_82599(hw);
5383 else if (hw->mac.type == ixgbe_mac_X540 ||
5384 hw->mac.type == ixgbe_mac_X550 ||
5385 hw->mac.type == ixgbe_mac_X550EM_x ||
5386 hw->mac.type == ixgbe_mac_X550EM_a)
5387 ixgbe_setup_loopback_link_x540_x550(hw, true);
5390 #ifdef RTE_LIB_SECURITY
5391 if ((dev->data->dev_conf.rxmode.offloads &
5392 RTE_ETH_RX_OFFLOAD_SECURITY) ||
5393 (dev->data->dev_conf.txmode.offloads &
5394 RTE_ETH_TX_OFFLOAD_SECURITY)) {
5395 ret = ixgbe_crypto_enable_ipsec(dev);
5398 "ixgbe_crypto_enable_ipsec fails with %d.",
5409 * Start Receive Units for specified queue.
5412 ixgbe_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
5414 struct ixgbe_hw *hw;
5415 struct ixgbe_rx_queue *rxq;
5419 PMD_INIT_FUNC_TRACE();
5420 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5422 rxq = dev->data->rx_queues[rx_queue_id];
5424 /* Allocate buffers for descriptor rings */
5425 if (ixgbe_alloc_rx_queue_mbufs(rxq) != 0) {
5426 PMD_INIT_LOG(ERR, "Could not alloc mbuf for queue:%d",
5430 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
5431 rxdctl |= IXGBE_RXDCTL_ENABLE;
5432 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxq->reg_idx), rxdctl);
5434 /* Wait until RX Enable ready */
5435 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
5438 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
5439 } while (--poll_ms && !(rxdctl & IXGBE_RXDCTL_ENABLE));
5441 PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", rx_queue_id);
5443 IXGBE_WRITE_REG(hw, IXGBE_RDH(rxq->reg_idx), 0);
5444 IXGBE_WRITE_REG(hw, IXGBE_RDT(rxq->reg_idx), rxq->nb_rx_desc - 1);
5445 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
5451 * Stop Receive Units for specified queue.
5454 ixgbe_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
5456 struct ixgbe_hw *hw;
5457 struct ixgbe_adapter *adapter = dev->data->dev_private;
5458 struct ixgbe_rx_queue *rxq;
5462 PMD_INIT_FUNC_TRACE();
5463 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5465 rxq = dev->data->rx_queues[rx_queue_id];
5467 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
5468 rxdctl &= ~IXGBE_RXDCTL_ENABLE;
5469 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxq->reg_idx), rxdctl);
5471 /* Wait until RX Enable bit clear */
5472 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
5475 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
5476 } while (--poll_ms && (rxdctl & IXGBE_RXDCTL_ENABLE));
5478 PMD_INIT_LOG(ERR, "Could not disable Rx Queue %d", rx_queue_id);
5480 rte_delay_us(RTE_IXGBE_WAIT_100_US);
5482 ixgbe_rx_queue_release_mbufs(rxq);
5483 ixgbe_reset_rx_queue(adapter, rxq);
5484 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
5491 * Start Transmit Units for specified queue.
5494 ixgbe_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
5496 struct ixgbe_hw *hw;
5497 struct ixgbe_tx_queue *txq;
5501 PMD_INIT_FUNC_TRACE();
5502 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5504 txq = dev->data->tx_queues[tx_queue_id];
5505 IXGBE_WRITE_REG(hw, IXGBE_TDH(txq->reg_idx), 0);
5506 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txq->reg_idx));
5507 txdctl |= IXGBE_TXDCTL_ENABLE;
5508 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txq->reg_idx), txdctl);
5510 /* Wait until TX Enable ready */
5511 if (hw->mac.type == ixgbe_mac_82599EB) {
5512 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
5515 txdctl = IXGBE_READ_REG(hw,
5516 IXGBE_TXDCTL(txq->reg_idx));
5517 } while (--poll_ms && !(txdctl & IXGBE_TXDCTL_ENABLE));
5519 PMD_INIT_LOG(ERR, "Could not enable Tx Queue %d",
5523 IXGBE_WRITE_REG(hw, IXGBE_TDT(txq->reg_idx), 0);
5524 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
5530 * Stop Transmit Units for specified queue.
5533 ixgbe_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
5535 struct ixgbe_hw *hw;
5536 struct ixgbe_tx_queue *txq;
5538 uint32_t txtdh, txtdt;
5541 PMD_INIT_FUNC_TRACE();
5542 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5544 txq = dev->data->tx_queues[tx_queue_id];
5546 /* Wait until TX queue is empty */
5547 if (hw->mac.type == ixgbe_mac_82599EB) {
5548 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
5550 rte_delay_us(RTE_IXGBE_WAIT_100_US);
5551 txtdh = IXGBE_READ_REG(hw,
5552 IXGBE_TDH(txq->reg_idx));
5553 txtdt = IXGBE_READ_REG(hw,
5554 IXGBE_TDT(txq->reg_idx));
5555 } while (--poll_ms && (txtdh != txtdt));
5558 "Tx Queue %d is not empty when stopping.",
5562 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txq->reg_idx));
5563 txdctl &= ~IXGBE_TXDCTL_ENABLE;
5564 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txq->reg_idx), txdctl);
5566 /* Wait until TX Enable bit clear */
5567 if (hw->mac.type == ixgbe_mac_82599EB) {
5568 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
5571 txdctl = IXGBE_READ_REG(hw,
5572 IXGBE_TXDCTL(txq->reg_idx));
5573 } while (--poll_ms && (txdctl & IXGBE_TXDCTL_ENABLE));
5575 PMD_INIT_LOG(ERR, "Could not disable Tx Queue %d",
5579 if (txq->ops != NULL) {
5580 txq->ops->release_mbufs(txq);
5581 txq->ops->reset(txq);
5583 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
5589 ixgbe_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
5590 struct rte_eth_rxq_info *qinfo)
5592 struct ixgbe_rx_queue *rxq;
5594 rxq = dev->data->rx_queues[queue_id];
5596 qinfo->mp = rxq->mb_pool;
5597 qinfo->scattered_rx = dev->data->scattered_rx;
5598 qinfo->nb_desc = rxq->nb_rx_desc;
5600 qinfo->conf.rx_free_thresh = rxq->rx_free_thresh;
5601 qinfo->conf.rx_drop_en = rxq->drop_en;
5602 qinfo->conf.rx_deferred_start = rxq->rx_deferred_start;
5603 qinfo->conf.offloads = rxq->offloads;
5607 ixgbe_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
5608 struct rte_eth_txq_info *qinfo)
5610 struct ixgbe_tx_queue *txq;
5612 txq = dev->data->tx_queues[queue_id];
5614 qinfo->nb_desc = txq->nb_tx_desc;
5616 qinfo->conf.tx_thresh.pthresh = txq->pthresh;
5617 qinfo->conf.tx_thresh.hthresh = txq->hthresh;
5618 qinfo->conf.tx_thresh.wthresh = txq->wthresh;
5620 qinfo->conf.tx_free_thresh = txq->tx_free_thresh;
5621 qinfo->conf.tx_rs_thresh = txq->tx_rs_thresh;
5622 qinfo->conf.offloads = txq->offloads;
5623 qinfo->conf.tx_deferred_start = txq->tx_deferred_start;
5627 * [VF] Initializes Receive Unit.
5630 ixgbevf_dev_rx_init(struct rte_eth_dev *dev)
5632 struct ixgbe_hw *hw;
5633 struct ixgbe_rx_queue *rxq;
5634 struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
5635 uint32_t frame_size = dev->data->mtu + IXGBE_ETH_OVERHEAD;
5637 uint32_t srrctl, psrtype = 0;
5642 PMD_INIT_FUNC_TRACE();
5643 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5645 if (rte_is_power_of_2(dev->data->nb_rx_queues) == 0) {
5646 PMD_INIT_LOG(ERR, "The number of Rx queue invalid, "
5647 "it should be power of 2");
5651 if (dev->data->nb_rx_queues > hw->mac.max_rx_queues) {
5652 PMD_INIT_LOG(ERR, "The number of Rx queue invalid, "
5653 "it should be equal to or less than %d",
5654 hw->mac.max_rx_queues);
5659 * When the VF driver issues a IXGBE_VF_RESET request, the PF driver
5660 * disables the VF receipt of packets if the PF MTU is > 1500.
5661 * This is done to deal with 82599 limitations that imposes
5662 * the PF and all VFs to share the same MTU.
5663 * Then, the PF driver enables again the VF receipt of packet when
5664 * the VF driver issues a IXGBE_VF_SET_LPE request.
5665 * In the meantime, the VF device cannot be used, even if the VF driver
5666 * and the Guest VM network stack are ready to accept packets with a
5667 * size up to the PF MTU.
5668 * As a work-around to this PF behaviour, force the call to
5669 * ixgbevf_rlpml_set_vf even if jumbo frames are not used. This way,
5670 * VF packets received can work in all cases.
5672 if (ixgbevf_rlpml_set_vf(hw, frame_size) != 0)
5673 PMD_INIT_LOG(ERR, "Set max packet length to %d failed.",
5677 * Assume no header split and no VLAN strip support
5678 * on any Rx queue first .
5680 rxmode->offloads &= ~RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
5681 /* Setup RX queues */
5682 for (i = 0; i < dev->data->nb_rx_queues; i++) {
5683 rxq = dev->data->rx_queues[i];
5685 /* Allocate buffers for descriptor rings */
5686 ret = ixgbe_alloc_rx_queue_mbufs(rxq);
5690 /* Setup the Base and Length of the Rx Descriptor Rings */
5691 bus_addr = rxq->rx_ring_phys_addr;
5693 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(i),
5694 (uint32_t)(bus_addr & 0x00000000ffffffffULL));
5695 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(i),
5696 (uint32_t)(bus_addr >> 32));
5697 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(i),
5698 rxq->nb_rx_desc * sizeof(union ixgbe_adv_rx_desc));
5699 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(i), 0);
5700 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(i), 0);
5703 /* Configure the SRRCTL register */
5704 srrctl = IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
5706 /* Set if packets are dropped when no descriptors available */
5708 srrctl |= IXGBE_SRRCTL_DROP_EN;
5711 * Configure the RX buffer size in the BSIZEPACKET field of
5712 * the SRRCTL register of the queue.
5713 * The value is in 1 KB resolution. Valid values can be from
5716 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) -
5717 RTE_PKTMBUF_HEADROOM);
5718 srrctl |= ((buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) &
5719 IXGBE_SRRCTL_BSIZEPKT_MASK);
5722 * VF modification to write virtual function SRRCTL register
5724 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(i), srrctl);
5726 buf_size = (uint16_t) ((srrctl & IXGBE_SRRCTL_BSIZEPKT_MASK) <<
5727 IXGBE_SRRCTL_BSIZEPKT_SHIFT);
5729 if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_SCATTER ||
5730 /* It adds dual VLAN length for supporting dual VLAN */
5731 (frame_size + 2 * RTE_VLAN_HLEN) > buf_size) {
5732 if (!dev->data->scattered_rx)
5733 PMD_INIT_LOG(DEBUG, "forcing scatter mode");
5734 dev->data->scattered_rx = 1;
5737 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP)
5738 rxmode->offloads |= RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
5741 /* Set RQPL for VF RSS according to max Rx queue */
5742 psrtype |= (dev->data->nb_rx_queues >> 1) <<
5743 IXGBE_PSRTYPE_RQPL_SHIFT;
5744 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype);
5746 ixgbe_set_rx_function(dev);
5752 * [VF] Initializes Transmit Unit.
5755 ixgbevf_dev_tx_init(struct rte_eth_dev *dev)
5757 struct ixgbe_hw *hw;
5758 struct ixgbe_tx_queue *txq;
5763 PMD_INIT_FUNC_TRACE();
5764 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5766 /* Setup the Base and Length of the Tx Descriptor Rings */
5767 for (i = 0; i < dev->data->nb_tx_queues; i++) {
5768 txq = dev->data->tx_queues[i];
5769 bus_addr = txq->tx_ring_phys_addr;
5770 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(i),
5771 (uint32_t)(bus_addr & 0x00000000ffffffffULL));
5772 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(i),
5773 (uint32_t)(bus_addr >> 32));
5774 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(i),
5775 txq->nb_tx_desc * sizeof(union ixgbe_adv_tx_desc));
5776 /* Setup the HW Tx Head and TX Tail descriptor pointers */
5777 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(i), 0);
5778 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(i), 0);
5781 * Disable Tx Head Writeback RO bit, since this hoses
5782 * bookkeeping if things aren't delivered in order.
5784 txctrl = IXGBE_READ_REG(hw,
5785 IXGBE_VFDCA_TXCTRL(i));
5786 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
5787 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(i),
5793 * [VF] Start Transmit and Receive Units.
5796 ixgbevf_dev_rxtx_start(struct rte_eth_dev *dev)
5798 struct ixgbe_hw *hw;
5799 struct ixgbe_tx_queue *txq;
5800 struct ixgbe_rx_queue *rxq;
5806 PMD_INIT_FUNC_TRACE();
5807 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5809 for (i = 0; i < dev->data->nb_tx_queues; i++) {
5810 txq = dev->data->tx_queues[i];
5811 /* Setup Transmit Threshold Registers */
5812 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i));
5813 txdctl |= txq->pthresh & 0x7F;
5814 txdctl |= ((txq->hthresh & 0x7F) << 8);
5815 txdctl |= ((txq->wthresh & 0x7F) << 16);
5816 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl);
5819 for (i = 0; i < dev->data->nb_tx_queues; i++) {
5821 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i));
5822 txdctl |= IXGBE_TXDCTL_ENABLE;
5823 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl);
5826 /* Wait until TX Enable ready */
5829 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i));
5830 } while (--poll_ms && !(txdctl & IXGBE_TXDCTL_ENABLE));
5832 PMD_INIT_LOG(ERR, "Could not enable Tx Queue %d", i);
5834 for (i = 0; i < dev->data->nb_rx_queues; i++) {
5836 rxq = dev->data->rx_queues[i];
5838 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i));
5839 rxdctl |= IXGBE_RXDCTL_ENABLE;
5840 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), rxdctl);
5842 /* Wait until RX Enable ready */
5846 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i));
5847 } while (--poll_ms && !(rxdctl & IXGBE_RXDCTL_ENABLE));
5849 PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", i);
5851 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(i), rxq->nb_rx_desc - 1);
5857 ixgbe_rss_conf_init(struct ixgbe_rte_flow_rss_conf *out,
5858 const struct rte_flow_action_rss *in)
5860 if (in->key_len > RTE_DIM(out->key) ||
5861 in->queue_num > RTE_DIM(out->queue))
5863 out->conf = (struct rte_flow_action_rss){
5867 .key_len = in->key_len,
5868 .queue_num = in->queue_num,
5869 .key = memcpy(out->key, in->key, in->key_len),
5870 .queue = memcpy(out->queue, in->queue,
5871 sizeof(*in->queue) * in->queue_num),
5877 ixgbe_action_rss_same(const struct rte_flow_action_rss *comp,
5878 const struct rte_flow_action_rss *with)
5880 return (comp->func == with->func &&
5881 comp->level == with->level &&
5882 comp->types == with->types &&
5883 comp->key_len == with->key_len &&
5884 comp->queue_num == with->queue_num &&
5885 !memcmp(comp->key, with->key, with->key_len) &&
5886 !memcmp(comp->queue, with->queue,
5887 sizeof(*with->queue) * with->queue_num));
5891 ixgbe_config_rss_filter(struct rte_eth_dev *dev,
5892 struct ixgbe_rte_flow_rss_conf *conf, bool add)
5894 struct ixgbe_hw *hw;
5898 uint16_t sp_reta_size;
5900 struct rte_eth_rss_conf rss_conf = {
5901 .rss_key = conf->conf.key_len ?
5902 (void *)(uintptr_t)conf->conf.key : NULL,
5903 .rss_key_len = conf->conf.key_len,
5904 .rss_hf = conf->conf.types,
5906 struct ixgbe_filter_info *filter_info =
5907 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
5909 PMD_INIT_FUNC_TRACE();
5910 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5912 sp_reta_size = ixgbe_reta_size_get(hw->mac.type);
5915 if (ixgbe_action_rss_same(&filter_info->rss_info.conf,
5917 ixgbe_rss_disable(dev);
5918 memset(&filter_info->rss_info, 0,
5919 sizeof(struct ixgbe_rte_flow_rss_conf));
5925 if (filter_info->rss_info.conf.queue_num)
5927 /* Fill in redirection table
5928 * The byte-swap is needed because NIC registers are in
5929 * little-endian order.
5932 for (i = 0, j = 0; i < sp_reta_size; i++, j++) {
5933 reta_reg = ixgbe_reta_reg_get(hw->mac.type, i);
5935 if (j == conf->conf.queue_num)
5937 reta = (reta << 8) | conf->conf.queue[j];
5939 IXGBE_WRITE_REG(hw, reta_reg,
5943 /* Configure the RSS key and the RSS protocols used to compute
5944 * the RSS hash of input packets.
5946 if ((rss_conf.rss_hf & IXGBE_RSS_OFFLOAD_ALL) == 0) {
5947 ixgbe_rss_disable(dev);
5950 if (rss_conf.rss_key == NULL)
5951 rss_conf.rss_key = rss_intel_key; /* Default hash key */
5952 ixgbe_hw_rss_hash_set(hw, &rss_conf);
5954 if (ixgbe_rss_conf_init(&filter_info->rss_info, &conf->conf))
5960 /* Stubs needed for linkage when RTE_ARCH_PPC_64 or RTE_ARCH_RISCV is set */
5961 #if defined(RTE_ARCH_PPC_64) || defined(RTE_ARCH_RISCV)
5963 ixgbe_rx_vec_dev_conf_condition_check(struct rte_eth_dev __rte_unused *dev)
5969 ixgbe_recv_pkts_vec(
5970 void __rte_unused *rx_queue,
5971 struct rte_mbuf __rte_unused **rx_pkts,
5972 uint16_t __rte_unused nb_pkts)
5978 ixgbe_recv_scattered_pkts_vec(
5979 void __rte_unused *rx_queue,
5980 struct rte_mbuf __rte_unused **rx_pkts,
5981 uint16_t __rte_unused nb_pkts)
5987 ixgbe_rxq_vec_setup(struct ixgbe_rx_queue __rte_unused *rxq)
5993 ixgbe_xmit_fixed_burst_vec(void __rte_unused *tx_queue,
5994 struct rte_mbuf __rte_unused **tx_pkts,
5995 uint16_t __rte_unused nb_pkts)
6001 ixgbe_txq_vec_setup(struct ixgbe_tx_queue __rte_unused *txq)
6007 ixgbe_rx_queue_release_mbufs_vec(struct ixgbe_rx_queue __rte_unused *rxq)