1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2016 Intel Corporation.
3 * Copyright 2014 6WIND S.A.
17 #include <rte_byteorder.h>
18 #include <rte_common.h>
19 #include <rte_cycles.h>
21 #include <rte_debug.h>
22 #include <rte_interrupts.h>
24 #include <rte_memory.h>
25 #include <rte_memzone.h>
26 #include <rte_launch.h>
28 #include <rte_per_lcore.h>
29 #include <rte_lcore.h>
30 #include <rte_atomic.h>
31 #include <rte_branch_prediction.h>
32 #include <rte_mempool.h>
33 #include <rte_malloc.h>
35 #include <rte_ether.h>
36 #include <ethdev_driver.h>
37 #include <rte_security_driver.h>
38 #include <rte_prefetch.h>
42 #include <rte_string_fns.h>
43 #include <rte_errno.h>
48 #include "ixgbe_logs.h"
49 #include "base/ixgbe_api.h"
50 #include "base/ixgbe_vf.h"
51 #include "ixgbe_ethdev.h"
52 #include "base/ixgbe_dcb.h"
53 #include "base/ixgbe_common.h"
54 #include "ixgbe_rxtx.h"
56 #ifdef RTE_LIBRTE_IEEE1588
57 #define IXGBE_TX_IEEE1588_TMST RTE_MBUF_F_TX_IEEE1588_TMST
59 #define IXGBE_TX_IEEE1588_TMST 0
61 /* Bit Mask to indicate what bits required for building TX context */
62 #define IXGBE_TX_OFFLOAD_MASK (RTE_MBUF_F_TX_OUTER_IPV6 | \
63 RTE_MBUF_F_TX_OUTER_IPV4 | \
64 RTE_MBUF_F_TX_IPV6 | \
65 RTE_MBUF_F_TX_IPV4 | \
66 RTE_MBUF_F_TX_VLAN | \
67 RTE_MBUF_F_TX_IP_CKSUM | \
68 RTE_MBUF_F_TX_L4_MASK | \
69 RTE_MBUF_F_TX_TCP_SEG | \
70 RTE_MBUF_F_TX_MACSEC | \
71 RTE_MBUF_F_TX_OUTER_IP_CKSUM | \
72 RTE_MBUF_F_TX_SEC_OFFLOAD | \
73 IXGBE_TX_IEEE1588_TMST)
75 #define IXGBE_TX_OFFLOAD_NOTSUP_MASK \
76 (RTE_MBUF_F_TX_OFFLOAD_MASK ^ IXGBE_TX_OFFLOAD_MASK)
79 #define RTE_PMD_USE_PREFETCH
82 #ifdef RTE_PMD_USE_PREFETCH
84 * Prefetch a cache line into all cache levels.
86 #define rte_ixgbe_prefetch(p) rte_prefetch0(p)
88 #define rte_ixgbe_prefetch(p) do {} while (0)
91 /*********************************************************************
95 **********************************************************************/
98 * Check for descriptors with their DD bit set and free mbufs.
99 * Return the total number of buffers freed.
101 static __rte_always_inline int
102 ixgbe_tx_free_bufs(struct ixgbe_tx_queue *txq)
104 struct ixgbe_tx_entry *txep;
107 struct rte_mbuf *m, *free[RTE_IXGBE_TX_MAX_FREE_BUF_SZ];
109 /* check DD bit on threshold descriptor */
110 status = txq->tx_ring[txq->tx_next_dd].wb.status;
111 if (!(status & rte_cpu_to_le_32(IXGBE_ADVTXD_STAT_DD)))
115 * first buffer to free from S/W ring is at index
116 * tx_next_dd - (tx_rs_thresh-1)
118 txep = &(txq->sw_ring[txq->tx_next_dd - (txq->tx_rs_thresh - 1)]);
120 for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) {
121 /* free buffers one at a time */
122 m = rte_pktmbuf_prefree_seg(txep->mbuf);
125 if (unlikely(m == NULL))
128 if (nb_free >= RTE_IXGBE_TX_MAX_FREE_BUF_SZ ||
129 (nb_free > 0 && m->pool != free[0]->pool)) {
130 rte_mempool_put_bulk(free[0]->pool,
131 (void **)free, nb_free);
139 rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free);
141 /* buffers were freed, update counters */
142 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh);
143 txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh);
144 if (txq->tx_next_dd >= txq->nb_tx_desc)
145 txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
147 return txq->tx_rs_thresh;
150 /* Populate 4 descriptors with data from 4 mbufs */
152 tx4(volatile union ixgbe_adv_tx_desc *txdp, struct rte_mbuf **pkts)
154 uint64_t buf_dma_addr;
158 for (i = 0; i < 4; ++i, ++txdp, ++pkts) {
159 buf_dma_addr = rte_mbuf_data_iova(*pkts);
160 pkt_len = (*pkts)->data_len;
162 /* write data to descriptor */
163 txdp->read.buffer_addr = rte_cpu_to_le_64(buf_dma_addr);
165 txdp->read.cmd_type_len =
166 rte_cpu_to_le_32((uint32_t)DCMD_DTYP_FLAGS | pkt_len);
168 txdp->read.olinfo_status =
169 rte_cpu_to_le_32(pkt_len << IXGBE_ADVTXD_PAYLEN_SHIFT);
171 rte_prefetch0(&(*pkts)->pool);
175 /* Populate 1 descriptor with data from 1 mbuf */
177 tx1(volatile union ixgbe_adv_tx_desc *txdp, struct rte_mbuf **pkts)
179 uint64_t buf_dma_addr;
182 buf_dma_addr = rte_mbuf_data_iova(*pkts);
183 pkt_len = (*pkts)->data_len;
185 /* write data to descriptor */
186 txdp->read.buffer_addr = rte_cpu_to_le_64(buf_dma_addr);
187 txdp->read.cmd_type_len =
188 rte_cpu_to_le_32((uint32_t)DCMD_DTYP_FLAGS | pkt_len);
189 txdp->read.olinfo_status =
190 rte_cpu_to_le_32(pkt_len << IXGBE_ADVTXD_PAYLEN_SHIFT);
191 rte_prefetch0(&(*pkts)->pool);
195 * Fill H/W descriptor ring with mbuf data.
196 * Copy mbuf pointers to the S/W ring.
199 ixgbe_tx_fill_hw_ring(struct ixgbe_tx_queue *txq, struct rte_mbuf **pkts,
202 volatile union ixgbe_adv_tx_desc *txdp = &(txq->tx_ring[txq->tx_tail]);
203 struct ixgbe_tx_entry *txep = &(txq->sw_ring[txq->tx_tail]);
204 const int N_PER_LOOP = 4;
205 const int N_PER_LOOP_MASK = N_PER_LOOP-1;
206 int mainpart, leftover;
210 * Process most of the packets in chunks of N pkts. Any
211 * leftover packets will get processed one at a time.
213 mainpart = (nb_pkts & ((uint32_t) ~N_PER_LOOP_MASK));
214 leftover = (nb_pkts & ((uint32_t) N_PER_LOOP_MASK));
215 for (i = 0; i < mainpart; i += N_PER_LOOP) {
216 /* Copy N mbuf pointers to the S/W ring */
217 for (j = 0; j < N_PER_LOOP; ++j) {
218 (txep + i + j)->mbuf = *(pkts + i + j);
220 tx4(txdp + i, pkts + i);
223 if (unlikely(leftover > 0)) {
224 for (i = 0; i < leftover; ++i) {
225 (txep + mainpart + i)->mbuf = *(pkts + mainpart + i);
226 tx1(txdp + mainpart + i, pkts + mainpart + i);
231 static inline uint16_t
232 tx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
235 struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue;
236 volatile union ixgbe_adv_tx_desc *tx_r = txq->tx_ring;
240 * Begin scanning the H/W ring for done descriptors when the
241 * number of available descriptors drops below tx_free_thresh. For
242 * each done descriptor, free the associated buffer.
244 if (txq->nb_tx_free < txq->tx_free_thresh)
245 ixgbe_tx_free_bufs(txq);
247 /* Only use descriptors that are available */
248 nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts);
249 if (unlikely(nb_pkts == 0))
252 /* Use exactly nb_pkts descriptors */
253 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts);
256 * At this point, we know there are enough descriptors in the
257 * ring to transmit all the packets. This assumes that each
258 * mbuf contains a single segment, and that no new offloads
259 * are expected, which would require a new context descriptor.
263 * See if we're going to wrap-around. If so, handle the top
264 * of the descriptor ring first, then do the bottom. If not,
265 * the processing looks just like the "bottom" part anyway...
267 if ((txq->tx_tail + nb_pkts) > txq->nb_tx_desc) {
268 n = (uint16_t)(txq->nb_tx_desc - txq->tx_tail);
269 ixgbe_tx_fill_hw_ring(txq, tx_pkts, n);
272 * We know that the last descriptor in the ring will need to
273 * have its RS bit set because tx_rs_thresh has to be
274 * a divisor of the ring size
276 tx_r[txq->tx_next_rs].read.cmd_type_len |=
277 rte_cpu_to_le_32(IXGBE_ADVTXD_DCMD_RS);
278 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
283 /* Fill H/W descriptor ring with mbuf data */
284 ixgbe_tx_fill_hw_ring(txq, tx_pkts + n, (uint16_t)(nb_pkts - n));
285 txq->tx_tail = (uint16_t)(txq->tx_tail + (nb_pkts - n));
288 * Determine if RS bit should be set
289 * This is what we actually want:
290 * if ((txq->tx_tail - 1) >= txq->tx_next_rs)
291 * but instead of subtracting 1 and doing >=, we can just do
292 * greater than without subtracting.
294 if (txq->tx_tail > txq->tx_next_rs) {
295 tx_r[txq->tx_next_rs].read.cmd_type_len |=
296 rte_cpu_to_le_32(IXGBE_ADVTXD_DCMD_RS);
297 txq->tx_next_rs = (uint16_t)(txq->tx_next_rs +
299 if (txq->tx_next_rs >= txq->nb_tx_desc)
300 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
304 * Check for wrap-around. This would only happen if we used
305 * up to the last descriptor in the ring, no more, no less.
307 if (txq->tx_tail >= txq->nb_tx_desc)
310 /* update tail pointer */
312 IXGBE_PCI_REG_WC_WRITE_RELAXED(txq->tdt_reg_addr, txq->tx_tail);
318 ixgbe_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts,
323 /* Try to transmit at least chunks of TX_MAX_BURST pkts */
324 if (likely(nb_pkts <= RTE_PMD_IXGBE_TX_MAX_BURST))
325 return tx_xmit_pkts(tx_queue, tx_pkts, nb_pkts);
327 /* transmit more than the max burst, in chunks of TX_MAX_BURST */
332 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_IXGBE_TX_MAX_BURST);
333 ret = tx_xmit_pkts(tx_queue, &(tx_pkts[nb_tx]), n);
334 nb_tx = (uint16_t)(nb_tx + ret);
335 nb_pkts = (uint16_t)(nb_pkts - ret);
344 ixgbe_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
348 struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue;
353 num = (uint16_t)RTE_MIN(nb_pkts, txq->tx_rs_thresh);
354 ret = ixgbe_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
366 ixgbe_set_xmit_ctx(struct ixgbe_tx_queue *txq,
367 volatile struct ixgbe_adv_tx_context_desc *ctx_txd,
368 uint64_t ol_flags, union ixgbe_tx_offload tx_offload,
369 __rte_unused uint64_t *mdata)
371 uint32_t type_tucmd_mlhl;
372 uint32_t mss_l4len_idx = 0;
374 uint32_t vlan_macip_lens;
375 union ixgbe_tx_offload tx_offload_mask;
376 uint32_t seqnum_seed = 0;
378 ctx_idx = txq->ctx_curr;
379 tx_offload_mask.data[0] = 0;
380 tx_offload_mask.data[1] = 0;
383 /* Specify which HW CTX to upload. */
384 mss_l4len_idx |= (ctx_idx << IXGBE_ADVTXD_IDX_SHIFT);
386 if (ol_flags & RTE_MBUF_F_TX_VLAN)
387 tx_offload_mask.vlan_tci |= ~0;
389 /* check if TCP segmentation required for this packet */
390 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {
391 /* implies IP cksum in IPv4 */
392 if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM)
393 type_tucmd_mlhl = IXGBE_ADVTXD_TUCMD_IPV4 |
394 IXGBE_ADVTXD_TUCMD_L4T_TCP |
395 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
397 type_tucmd_mlhl = IXGBE_ADVTXD_TUCMD_IPV6 |
398 IXGBE_ADVTXD_TUCMD_L4T_TCP |
399 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
401 tx_offload_mask.l2_len |= ~0;
402 tx_offload_mask.l3_len |= ~0;
403 tx_offload_mask.l4_len |= ~0;
404 tx_offload_mask.tso_segsz |= ~0;
405 mss_l4len_idx |= tx_offload.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT;
406 mss_l4len_idx |= tx_offload.l4_len << IXGBE_ADVTXD_L4LEN_SHIFT;
407 } else { /* no TSO, check if hardware checksum is needed */
408 if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM) {
409 type_tucmd_mlhl = IXGBE_ADVTXD_TUCMD_IPV4;
410 tx_offload_mask.l2_len |= ~0;
411 tx_offload_mask.l3_len |= ~0;
414 switch (ol_flags & RTE_MBUF_F_TX_L4_MASK) {
415 case RTE_MBUF_F_TX_UDP_CKSUM:
416 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP |
417 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
418 mss_l4len_idx |= sizeof(struct rte_udp_hdr)
419 << IXGBE_ADVTXD_L4LEN_SHIFT;
420 tx_offload_mask.l2_len |= ~0;
421 tx_offload_mask.l3_len |= ~0;
423 case RTE_MBUF_F_TX_TCP_CKSUM:
424 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP |
425 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
426 mss_l4len_idx |= sizeof(struct rte_tcp_hdr)
427 << IXGBE_ADVTXD_L4LEN_SHIFT;
428 tx_offload_mask.l2_len |= ~0;
429 tx_offload_mask.l3_len |= ~0;
431 case RTE_MBUF_F_TX_SCTP_CKSUM:
432 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP |
433 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
434 mss_l4len_idx |= sizeof(struct rte_sctp_hdr)
435 << IXGBE_ADVTXD_L4LEN_SHIFT;
436 tx_offload_mask.l2_len |= ~0;
437 tx_offload_mask.l3_len |= ~0;
440 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_RSV |
441 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
446 if (ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) {
447 tx_offload_mask.outer_l2_len |= ~0;
448 tx_offload_mask.outer_l3_len |= ~0;
449 tx_offload_mask.l2_len |= ~0;
450 seqnum_seed |= tx_offload.outer_l3_len
451 << IXGBE_ADVTXD_OUTER_IPLEN;
452 seqnum_seed |= tx_offload.l2_len
453 << IXGBE_ADVTXD_TUNNEL_LEN;
455 #ifdef RTE_LIB_SECURITY
456 if (ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD) {
457 union ixgbe_crypto_tx_desc_md *md =
458 (union ixgbe_crypto_tx_desc_md *)mdata;
460 (IXGBE_ADVTXD_IPSEC_SA_INDEX_MASK & md->sa_idx);
461 type_tucmd_mlhl |= md->enc ?
462 (IXGBE_ADVTXD_TUCMD_IPSEC_TYPE_ESP |
463 IXGBE_ADVTXD_TUCMD_IPSEC_ENCRYPT_EN) : 0;
465 (md->pad_len & IXGBE_ADVTXD_IPSEC_ESP_LEN_MASK);
466 tx_offload_mask.sa_idx |= ~0;
467 tx_offload_mask.sec_pad_len |= ~0;
471 txq->ctx_cache[ctx_idx].flags = ol_flags;
472 txq->ctx_cache[ctx_idx].tx_offload.data[0] =
473 tx_offload_mask.data[0] & tx_offload.data[0];
474 txq->ctx_cache[ctx_idx].tx_offload.data[1] =
475 tx_offload_mask.data[1] & tx_offload.data[1];
476 txq->ctx_cache[ctx_idx].tx_offload_mask = tx_offload_mask;
478 ctx_txd->type_tucmd_mlhl = rte_cpu_to_le_32(type_tucmd_mlhl);
479 vlan_macip_lens = tx_offload.l3_len;
480 if (ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM)
481 vlan_macip_lens |= (tx_offload.outer_l2_len <<
482 IXGBE_ADVTXD_MACLEN_SHIFT);
484 vlan_macip_lens |= (tx_offload.l2_len <<
485 IXGBE_ADVTXD_MACLEN_SHIFT);
486 vlan_macip_lens |= ((uint32_t)tx_offload.vlan_tci << IXGBE_ADVTXD_VLAN_SHIFT);
487 ctx_txd->vlan_macip_lens = rte_cpu_to_le_32(vlan_macip_lens);
488 ctx_txd->mss_l4len_idx = rte_cpu_to_le_32(mss_l4len_idx);
489 ctx_txd->seqnum_seed = seqnum_seed;
493 * Check which hardware context can be used. Use the existing match
494 * or create a new context descriptor.
496 static inline uint32_t
497 what_advctx_update(struct ixgbe_tx_queue *txq, uint64_t flags,
498 union ixgbe_tx_offload tx_offload)
500 /* If match with the current used context */
501 if (likely((txq->ctx_cache[txq->ctx_curr].flags == flags) &&
502 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] ==
503 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0]
504 & tx_offload.data[0])) &&
505 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] ==
506 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1]
507 & tx_offload.data[1]))))
508 return txq->ctx_curr;
510 /* What if match with the next context */
512 if (likely((txq->ctx_cache[txq->ctx_curr].flags == flags) &&
513 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] ==
514 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0]
515 & tx_offload.data[0])) &&
516 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] ==
517 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1]
518 & tx_offload.data[1]))))
519 return txq->ctx_curr;
521 /* Mismatch, use the previous context */
522 return IXGBE_CTX_NUM;
525 static inline uint32_t
526 tx_desc_cksum_flags_to_olinfo(uint64_t ol_flags)
530 if ((ol_flags & RTE_MBUF_F_TX_L4_MASK) != RTE_MBUF_F_TX_L4_NO_CKSUM)
531 tmp |= IXGBE_ADVTXD_POPTS_TXSM;
532 if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM)
533 tmp |= IXGBE_ADVTXD_POPTS_IXSM;
534 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG)
535 tmp |= IXGBE_ADVTXD_POPTS_TXSM;
539 static inline uint32_t
540 tx_desc_ol_flags_to_cmdtype(uint64_t ol_flags)
542 uint32_t cmdtype = 0;
544 if (ol_flags & RTE_MBUF_F_TX_VLAN)
545 cmdtype |= IXGBE_ADVTXD_DCMD_VLE;
546 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG)
547 cmdtype |= IXGBE_ADVTXD_DCMD_TSE;
548 if (ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM)
549 cmdtype |= (1 << IXGBE_ADVTXD_OUTERIPCS_SHIFT);
550 if (ol_flags & RTE_MBUF_F_TX_MACSEC)
551 cmdtype |= IXGBE_ADVTXD_MAC_LINKSEC;
555 /* Default RS bit threshold values */
556 #ifndef DEFAULT_TX_RS_THRESH
557 #define DEFAULT_TX_RS_THRESH 32
559 #ifndef DEFAULT_TX_FREE_THRESH
560 #define DEFAULT_TX_FREE_THRESH 32
563 /* Reset transmit descriptors after they have been used */
565 ixgbe_xmit_cleanup(struct ixgbe_tx_queue *txq)
567 struct ixgbe_tx_entry *sw_ring = txq->sw_ring;
568 volatile union ixgbe_adv_tx_desc *txr = txq->tx_ring;
569 uint16_t last_desc_cleaned = txq->last_desc_cleaned;
570 uint16_t nb_tx_desc = txq->nb_tx_desc;
571 uint16_t desc_to_clean_to;
572 uint16_t nb_tx_to_clean;
575 /* Determine the last descriptor needing to be cleaned */
576 desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->tx_rs_thresh);
577 if (desc_to_clean_to >= nb_tx_desc)
578 desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc);
580 /* Check to make sure the last descriptor to clean is done */
581 desc_to_clean_to = sw_ring[desc_to_clean_to].last_id;
582 status = txr[desc_to_clean_to].wb.status;
583 if (!(status & rte_cpu_to_le_32(IXGBE_TXD_STAT_DD))) {
585 "TX descriptor %4u is not done"
586 "(port=%d queue=%d)",
588 txq->port_id, txq->queue_id);
589 /* Failed to clean any descriptors, better luck next time */
593 /* Figure out how many descriptors will be cleaned */
594 if (last_desc_cleaned > desc_to_clean_to)
595 nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) +
598 nb_tx_to_clean = (uint16_t)(desc_to_clean_to -
602 "Cleaning %4u TX descriptors: %4u to %4u "
603 "(port=%d queue=%d)",
604 nb_tx_to_clean, last_desc_cleaned, desc_to_clean_to,
605 txq->port_id, txq->queue_id);
608 * The last descriptor to clean is done, so that means all the
609 * descriptors from the last descriptor that was cleaned
610 * up to the last descriptor with the RS bit set
611 * are done. Only reset the threshold descriptor.
613 txr[desc_to_clean_to].wb.status = 0;
615 /* Update the txq to reflect the last descriptor that was cleaned */
616 txq->last_desc_cleaned = desc_to_clean_to;
617 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + nb_tx_to_clean);
624 ixgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
627 struct ixgbe_tx_queue *txq;
628 struct ixgbe_tx_entry *sw_ring;
629 struct ixgbe_tx_entry *txe, *txn;
630 volatile union ixgbe_adv_tx_desc *txr;
631 volatile union ixgbe_adv_tx_desc *txd, *txp;
632 struct rte_mbuf *tx_pkt;
633 struct rte_mbuf *m_seg;
634 uint64_t buf_dma_addr;
635 uint32_t olinfo_status;
636 uint32_t cmd_type_len;
647 union ixgbe_tx_offload tx_offload;
648 #ifdef RTE_LIB_SECURITY
652 tx_offload.data[0] = 0;
653 tx_offload.data[1] = 0;
655 sw_ring = txq->sw_ring;
657 tx_id = txq->tx_tail;
658 txe = &sw_ring[tx_id];
661 /* Determine if the descriptor ring needs to be cleaned. */
662 if (txq->nb_tx_free < txq->tx_free_thresh)
663 ixgbe_xmit_cleanup(txq);
665 rte_prefetch0(&txe->mbuf->pool);
668 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
671 pkt_len = tx_pkt->pkt_len;
674 * Determine how many (if any) context descriptors
675 * are needed for offload functionality.
677 ol_flags = tx_pkt->ol_flags;
678 #ifdef RTE_LIB_SECURITY
679 use_ipsec = txq->using_ipsec && (ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD);
682 /* If hardware offload required */
683 tx_ol_req = ol_flags & IXGBE_TX_OFFLOAD_MASK;
685 tx_offload.l2_len = tx_pkt->l2_len;
686 tx_offload.l3_len = tx_pkt->l3_len;
687 tx_offload.l4_len = tx_pkt->l4_len;
688 tx_offload.vlan_tci = tx_pkt->vlan_tci;
689 tx_offload.tso_segsz = tx_pkt->tso_segsz;
690 tx_offload.outer_l2_len = tx_pkt->outer_l2_len;
691 tx_offload.outer_l3_len = tx_pkt->outer_l3_len;
692 #ifdef RTE_LIB_SECURITY
694 union ixgbe_crypto_tx_desc_md *ipsec_mdata =
695 (union ixgbe_crypto_tx_desc_md *)
696 rte_security_dynfield(tx_pkt);
697 tx_offload.sa_idx = ipsec_mdata->sa_idx;
698 tx_offload.sec_pad_len = ipsec_mdata->pad_len;
702 /* If new context need be built or reuse the exist ctx. */
703 ctx = what_advctx_update(txq, tx_ol_req,
705 /* Only allocate context descriptor if required*/
706 new_ctx = (ctx == IXGBE_CTX_NUM);
711 * Keep track of how many descriptors are used this loop
712 * This will always be the number of segments + the number of
713 * Context descriptors required to transmit the packet
715 nb_used = (uint16_t)(tx_pkt->nb_segs + new_ctx);
718 nb_used + txq->nb_tx_used >= txq->tx_rs_thresh)
719 /* set RS on the previous packet in the burst */
720 txp->read.cmd_type_len |=
721 rte_cpu_to_le_32(IXGBE_TXD_CMD_RS);
724 * The number of descriptors that must be allocated for a
725 * packet is the number of segments of that packet, plus 1
726 * Context Descriptor for the hardware offload, if any.
727 * Determine the last TX descriptor to allocate in the TX ring
728 * for the packet, starting from the current position (tx_id)
731 tx_last = (uint16_t) (tx_id + nb_used - 1);
734 if (tx_last >= txq->nb_tx_desc)
735 tx_last = (uint16_t) (tx_last - txq->nb_tx_desc);
737 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u pktlen=%u"
738 " tx_first=%u tx_last=%u",
739 (unsigned) txq->port_id,
740 (unsigned) txq->queue_id,
746 * Make sure there are enough TX descriptors available to
747 * transmit the entire packet.
748 * nb_used better be less than or equal to txq->tx_rs_thresh
750 if (nb_used > txq->nb_tx_free) {
752 "Not enough free TX descriptors "
753 "nb_used=%4u nb_free=%4u "
754 "(port=%d queue=%d)",
755 nb_used, txq->nb_tx_free,
756 txq->port_id, txq->queue_id);
758 if (ixgbe_xmit_cleanup(txq) != 0) {
759 /* Could not clean any descriptors */
765 /* nb_used better be <= txq->tx_rs_thresh */
766 if (unlikely(nb_used > txq->tx_rs_thresh)) {
768 "The number of descriptors needed to "
769 "transmit the packet exceeds the "
770 "RS bit threshold. This will impact "
772 "nb_used=%4u nb_free=%4u "
774 "(port=%d queue=%d)",
775 nb_used, txq->nb_tx_free,
777 txq->port_id, txq->queue_id);
779 * Loop here until there are enough TX
780 * descriptors or until the ring cannot be
783 while (nb_used > txq->nb_tx_free) {
784 if (ixgbe_xmit_cleanup(txq) != 0) {
786 * Could not clean any
798 * By now there are enough free TX descriptors to transmit
803 * Set common flags of all TX Data Descriptors.
805 * The following bits must be set in all Data Descriptors:
806 * - IXGBE_ADVTXD_DTYP_DATA
807 * - IXGBE_ADVTXD_DCMD_DEXT
809 * The following bits must be set in the first Data Descriptor
810 * and are ignored in the other ones:
811 * - IXGBE_ADVTXD_DCMD_IFCS
812 * - IXGBE_ADVTXD_MAC_1588
813 * - IXGBE_ADVTXD_DCMD_VLE
815 * The following bits must only be set in the last Data
817 * - IXGBE_TXD_CMD_EOP
819 * The following bits can be set in any Data Descriptor, but
820 * are only set in the last Data Descriptor:
823 cmd_type_len = IXGBE_ADVTXD_DTYP_DATA |
824 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT;
826 #ifdef RTE_LIBRTE_IEEE1588
827 if (ol_flags & RTE_MBUF_F_TX_IEEE1588_TMST)
828 cmd_type_len |= IXGBE_ADVTXD_MAC_1588;
834 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {
835 /* when TSO is on, paylen in descriptor is the
836 * not the packet len but the tcp payload len */
837 pkt_len -= (tx_offload.l2_len +
838 tx_offload.l3_len + tx_offload.l4_len);
842 * Setup the TX Advanced Context Descriptor if required
845 volatile struct ixgbe_adv_tx_context_desc *
848 ctx_txd = (volatile struct
849 ixgbe_adv_tx_context_desc *)
852 txn = &sw_ring[txe->next_id];
853 rte_prefetch0(&txn->mbuf->pool);
855 if (txe->mbuf != NULL) {
856 rte_pktmbuf_free_seg(txe->mbuf);
860 ixgbe_set_xmit_ctx(txq, ctx_txd, tx_ol_req,
862 rte_security_dynfield(tx_pkt));
864 txe->last_id = tx_last;
865 tx_id = txe->next_id;
870 * Setup the TX Advanced Data Descriptor,
871 * This path will go through
872 * whatever new/reuse the context descriptor
874 cmd_type_len |= tx_desc_ol_flags_to_cmdtype(ol_flags);
875 olinfo_status |= tx_desc_cksum_flags_to_olinfo(ol_flags);
876 olinfo_status |= ctx << IXGBE_ADVTXD_IDX_SHIFT;
879 olinfo_status |= (pkt_len << IXGBE_ADVTXD_PAYLEN_SHIFT);
880 #ifdef RTE_LIB_SECURITY
882 olinfo_status |= IXGBE_ADVTXD_POPTS_IPSEC;
888 txn = &sw_ring[txe->next_id];
889 rte_prefetch0(&txn->mbuf->pool);
891 if (txe->mbuf != NULL)
892 rte_pktmbuf_free_seg(txe->mbuf);
896 * Set up Transmit Data Descriptor.
898 slen = m_seg->data_len;
899 buf_dma_addr = rte_mbuf_data_iova(m_seg);
900 txd->read.buffer_addr =
901 rte_cpu_to_le_64(buf_dma_addr);
902 txd->read.cmd_type_len =
903 rte_cpu_to_le_32(cmd_type_len | slen);
904 txd->read.olinfo_status =
905 rte_cpu_to_le_32(olinfo_status);
906 txe->last_id = tx_last;
907 tx_id = txe->next_id;
910 } while (m_seg != NULL);
913 * The last packet data descriptor needs End Of Packet (EOP)
915 cmd_type_len |= IXGBE_TXD_CMD_EOP;
916 txq->nb_tx_used = (uint16_t)(txq->nb_tx_used + nb_used);
917 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_used);
919 /* Set RS bit only on threshold packets' last descriptor */
920 if (txq->nb_tx_used >= txq->tx_rs_thresh) {
922 "Setting RS bit on TXD id="
923 "%4u (port=%d queue=%d)",
924 tx_last, txq->port_id, txq->queue_id);
926 cmd_type_len |= IXGBE_TXD_CMD_RS;
928 /* Update txq RS bit counters */
934 txd->read.cmd_type_len |= rte_cpu_to_le_32(cmd_type_len);
938 /* set RS on last packet in the burst */
940 txp->read.cmd_type_len |= rte_cpu_to_le_32(IXGBE_TXD_CMD_RS);
945 * Set the Transmit Descriptor Tail (TDT)
947 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u",
948 (unsigned) txq->port_id, (unsigned) txq->queue_id,
949 (unsigned) tx_id, (unsigned) nb_tx);
950 IXGBE_PCI_REG_WC_WRITE_RELAXED(txq->tdt_reg_addr, tx_id);
951 txq->tx_tail = tx_id;
956 /*********************************************************************
960 **********************************************************************/
962 ixgbe_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
967 struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue;
969 for (i = 0; i < nb_pkts; i++) {
971 ol_flags = m->ol_flags;
974 * Check if packet meets requirements for number of segments
976 * NOTE: for ixgbe it's always (40 - WTHRESH) for both TSO and
980 if (m->nb_segs > IXGBE_TX_MAX_SEG - txq->wthresh) {
985 if (ol_flags & IXGBE_TX_OFFLOAD_NOTSUP_MASK) {
990 /* check the size of packet */
991 if (m->pkt_len < IXGBE_TX_MIN_PKT_LEN) {
996 #ifdef RTE_ETHDEV_DEBUG_TX
997 ret = rte_validate_tx_offload(m);
1003 ret = rte_net_intel_cksum_prepare(m);
1013 /*********************************************************************
1017 **********************************************************************/
1019 #define IXGBE_PACKET_TYPE_ETHER 0X00
1020 #define IXGBE_PACKET_TYPE_IPV4 0X01
1021 #define IXGBE_PACKET_TYPE_IPV4_TCP 0X11
1022 #define IXGBE_PACKET_TYPE_IPV4_UDP 0X21
1023 #define IXGBE_PACKET_TYPE_IPV4_SCTP 0X41
1024 #define IXGBE_PACKET_TYPE_IPV4_EXT 0X03
1025 #define IXGBE_PACKET_TYPE_IPV4_EXT_TCP 0X13
1026 #define IXGBE_PACKET_TYPE_IPV4_EXT_UDP 0X23
1027 #define IXGBE_PACKET_TYPE_IPV4_EXT_SCTP 0X43
1028 #define IXGBE_PACKET_TYPE_IPV6 0X04
1029 #define IXGBE_PACKET_TYPE_IPV6_TCP 0X14
1030 #define IXGBE_PACKET_TYPE_IPV6_UDP 0X24
1031 #define IXGBE_PACKET_TYPE_IPV6_SCTP 0X44
1032 #define IXGBE_PACKET_TYPE_IPV6_EXT 0X0C
1033 #define IXGBE_PACKET_TYPE_IPV6_EXT_TCP 0X1C
1034 #define IXGBE_PACKET_TYPE_IPV6_EXT_UDP 0X2C
1035 #define IXGBE_PACKET_TYPE_IPV6_EXT_SCTP 0X4C
1036 #define IXGBE_PACKET_TYPE_IPV4_IPV6 0X05
1037 #define IXGBE_PACKET_TYPE_IPV4_IPV6_TCP 0X15
1038 #define IXGBE_PACKET_TYPE_IPV4_IPV6_UDP 0X25
1039 #define IXGBE_PACKET_TYPE_IPV4_IPV6_SCTP 0X45
1040 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6 0X07
1041 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_TCP 0X17
1042 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_UDP 0X27
1043 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_SCTP 0X47
1044 #define IXGBE_PACKET_TYPE_IPV4_IPV6_EXT 0X0D
1045 #define IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_TCP 0X1D
1046 #define IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_UDP 0X2D
1047 #define IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_SCTP 0X4D
1048 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT 0X0F
1049 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_TCP 0X1F
1050 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_UDP 0X2F
1051 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_SCTP 0X4F
1053 #define IXGBE_PACKET_TYPE_NVGRE 0X00
1054 #define IXGBE_PACKET_TYPE_NVGRE_IPV4 0X01
1055 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_TCP 0X11
1056 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_UDP 0X21
1057 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_SCTP 0X41
1058 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT 0X03
1059 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_TCP 0X13
1060 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_UDP 0X23
1061 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_SCTP 0X43
1062 #define IXGBE_PACKET_TYPE_NVGRE_IPV6 0X04
1063 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_TCP 0X14
1064 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_UDP 0X24
1065 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_SCTP 0X44
1066 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT 0X0C
1067 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_TCP 0X1C
1068 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_UDP 0X2C
1069 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_SCTP 0X4C
1070 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6 0X05
1071 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_TCP 0X15
1072 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_UDP 0X25
1073 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT 0X0D
1074 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_TCP 0X1D
1075 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_UDP 0X2D
1077 #define IXGBE_PACKET_TYPE_VXLAN 0X80
1078 #define IXGBE_PACKET_TYPE_VXLAN_IPV4 0X81
1079 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_TCP 0x91
1080 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_UDP 0xA1
1081 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_SCTP 0xC1
1082 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT 0x83
1083 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_TCP 0X93
1084 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_UDP 0XA3
1085 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_SCTP 0XC3
1086 #define IXGBE_PACKET_TYPE_VXLAN_IPV6 0X84
1087 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_TCP 0X94
1088 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_UDP 0XA4
1089 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_SCTP 0XC4
1090 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT 0X8C
1091 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_TCP 0X9C
1092 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_UDP 0XAC
1093 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_SCTP 0XCC
1094 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6 0X85
1095 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_TCP 0X95
1096 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_UDP 0XA5
1097 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT 0X8D
1098 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_TCP 0X9D
1099 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_UDP 0XAD
1102 * Use 2 different table for normal packet and tunnel packet
1103 * to save the space.
1106 ptype_table[IXGBE_PACKET_TYPE_MAX] __rte_cache_aligned = {
1107 [IXGBE_PACKET_TYPE_ETHER] = RTE_PTYPE_L2_ETHER,
1108 [IXGBE_PACKET_TYPE_IPV4] = RTE_PTYPE_L2_ETHER |
1110 [IXGBE_PACKET_TYPE_IPV4_TCP] = RTE_PTYPE_L2_ETHER |
1111 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP,
1112 [IXGBE_PACKET_TYPE_IPV4_UDP] = RTE_PTYPE_L2_ETHER |
1113 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP,
1114 [IXGBE_PACKET_TYPE_IPV4_SCTP] = RTE_PTYPE_L2_ETHER |
1115 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_SCTP,
1116 [IXGBE_PACKET_TYPE_IPV4_EXT] = RTE_PTYPE_L2_ETHER |
1117 RTE_PTYPE_L3_IPV4_EXT,
1118 [IXGBE_PACKET_TYPE_IPV4_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1119 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_TCP,
1120 [IXGBE_PACKET_TYPE_IPV4_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1121 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_UDP,
1122 [IXGBE_PACKET_TYPE_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1123 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_SCTP,
1124 [IXGBE_PACKET_TYPE_IPV6] = RTE_PTYPE_L2_ETHER |
1126 [IXGBE_PACKET_TYPE_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1127 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP,
1128 [IXGBE_PACKET_TYPE_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1129 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP,
1130 [IXGBE_PACKET_TYPE_IPV6_SCTP] = RTE_PTYPE_L2_ETHER |
1131 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_SCTP,
1132 [IXGBE_PACKET_TYPE_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1133 RTE_PTYPE_L3_IPV6_EXT,
1134 [IXGBE_PACKET_TYPE_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1135 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_TCP,
1136 [IXGBE_PACKET_TYPE_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1137 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_UDP,
1138 [IXGBE_PACKET_TYPE_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1139 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_SCTP,
1140 [IXGBE_PACKET_TYPE_IPV4_IPV6] = RTE_PTYPE_L2_ETHER |
1141 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1142 RTE_PTYPE_INNER_L3_IPV6,
1143 [IXGBE_PACKET_TYPE_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1144 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1145 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP,
1146 [IXGBE_PACKET_TYPE_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1147 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1148 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP,
1149 [IXGBE_PACKET_TYPE_IPV4_IPV6_SCTP] = RTE_PTYPE_L2_ETHER |
1150 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1151 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_SCTP,
1152 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6] = RTE_PTYPE_L2_ETHER |
1153 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1154 RTE_PTYPE_INNER_L3_IPV6,
1155 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1156 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1157 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP,
1158 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1159 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1160 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP,
1161 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_SCTP] = RTE_PTYPE_L2_ETHER |
1162 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1163 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_SCTP,
1164 [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1165 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1166 RTE_PTYPE_INNER_L3_IPV6_EXT,
1167 [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1168 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1169 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP,
1170 [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1171 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1172 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP,
1173 [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1174 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1175 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_SCTP,
1176 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1177 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1178 RTE_PTYPE_INNER_L3_IPV6_EXT,
1179 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1180 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1181 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP,
1182 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1183 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1184 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP,
1185 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_SCTP] =
1186 RTE_PTYPE_L2_ETHER |
1187 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1188 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_SCTP,
1192 ptype_table_tn[IXGBE_PACKET_TYPE_TN_MAX] __rte_cache_aligned = {
1193 [IXGBE_PACKET_TYPE_NVGRE] = RTE_PTYPE_L2_ETHER |
1194 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1195 RTE_PTYPE_INNER_L2_ETHER,
1196 [IXGBE_PACKET_TYPE_NVGRE_IPV4] = RTE_PTYPE_L2_ETHER |
1197 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1198 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1199 [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT] = RTE_PTYPE_L2_ETHER |
1200 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1201 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT,
1202 [IXGBE_PACKET_TYPE_NVGRE_IPV6] = RTE_PTYPE_L2_ETHER |
1203 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1204 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6,
1205 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6] = RTE_PTYPE_L2_ETHER |
1206 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1207 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1208 [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1209 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1210 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT,
1211 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1212 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1213 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1214 [IXGBE_PACKET_TYPE_NVGRE_IPV4_TCP] = RTE_PTYPE_L2_ETHER |
1215 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1216 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4 |
1217 RTE_PTYPE_INNER_L4_TCP,
1218 [IXGBE_PACKET_TYPE_NVGRE_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1219 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1220 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6 |
1221 RTE_PTYPE_INNER_L4_TCP,
1222 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1223 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1224 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1225 [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1226 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1227 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT |
1228 RTE_PTYPE_INNER_L4_TCP,
1229 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_TCP] =
1230 RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
1231 RTE_PTYPE_TUNNEL_GRE | RTE_PTYPE_INNER_L2_ETHER |
1232 RTE_PTYPE_INNER_L3_IPV4,
1233 [IXGBE_PACKET_TYPE_NVGRE_IPV4_UDP] = RTE_PTYPE_L2_ETHER |
1234 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1235 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4 |
1236 RTE_PTYPE_INNER_L4_UDP,
1237 [IXGBE_PACKET_TYPE_NVGRE_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1238 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1239 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6 |
1240 RTE_PTYPE_INNER_L4_UDP,
1241 [IXGBE_PACKET_TYPE_NVGRE_IPV6_SCTP] = RTE_PTYPE_L2_ETHER |
1242 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1243 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6 |
1244 RTE_PTYPE_INNER_L4_SCTP,
1245 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1246 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1247 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1248 [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1249 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1250 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT |
1251 RTE_PTYPE_INNER_L4_UDP,
1252 [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1253 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1254 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT |
1255 RTE_PTYPE_INNER_L4_SCTP,
1256 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_UDP] =
1257 RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
1258 RTE_PTYPE_TUNNEL_GRE | RTE_PTYPE_INNER_L2_ETHER |
1259 RTE_PTYPE_INNER_L3_IPV4,
1260 [IXGBE_PACKET_TYPE_NVGRE_IPV4_SCTP] = RTE_PTYPE_L2_ETHER |
1261 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1262 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4 |
1263 RTE_PTYPE_INNER_L4_SCTP,
1264 [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1265 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1266 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT |
1267 RTE_PTYPE_INNER_L4_SCTP,
1268 [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1269 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1270 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT |
1271 RTE_PTYPE_INNER_L4_TCP,
1272 [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1273 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1274 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT |
1275 RTE_PTYPE_INNER_L4_UDP,
1277 [IXGBE_PACKET_TYPE_VXLAN] = RTE_PTYPE_L2_ETHER |
1278 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1279 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER,
1280 [IXGBE_PACKET_TYPE_VXLAN_IPV4] = RTE_PTYPE_L2_ETHER |
1281 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1282 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1283 RTE_PTYPE_INNER_L3_IPV4,
1284 [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT] = RTE_PTYPE_L2_ETHER |
1285 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1286 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1287 RTE_PTYPE_INNER_L3_IPV4_EXT,
1288 [IXGBE_PACKET_TYPE_VXLAN_IPV6] = RTE_PTYPE_L2_ETHER |
1289 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1290 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1291 RTE_PTYPE_INNER_L3_IPV6,
1292 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6] = RTE_PTYPE_L2_ETHER |
1293 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1294 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1295 RTE_PTYPE_INNER_L3_IPV4,
1296 [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1297 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1298 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1299 RTE_PTYPE_INNER_L3_IPV6_EXT,
1300 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1301 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1302 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1303 RTE_PTYPE_INNER_L3_IPV4,
1304 [IXGBE_PACKET_TYPE_VXLAN_IPV4_TCP] = RTE_PTYPE_L2_ETHER |
1305 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1306 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1307 RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L4_TCP,
1308 [IXGBE_PACKET_TYPE_VXLAN_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1309 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1310 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1311 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP,
1312 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1313 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1314 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1315 RTE_PTYPE_INNER_L3_IPV4,
1316 [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1317 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1318 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1319 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP,
1320 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_TCP] =
1321 RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
1322 RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_VXLAN |
1323 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1324 [IXGBE_PACKET_TYPE_VXLAN_IPV4_UDP] = RTE_PTYPE_L2_ETHER |
1325 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1326 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1327 RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L4_UDP,
1328 [IXGBE_PACKET_TYPE_VXLAN_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1329 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1330 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1331 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP,
1332 [IXGBE_PACKET_TYPE_VXLAN_IPV6_SCTP] = RTE_PTYPE_L2_ETHER |
1333 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1334 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1335 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_SCTP,
1336 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1337 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1338 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1339 RTE_PTYPE_INNER_L3_IPV4,
1340 [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1341 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1342 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1343 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP,
1344 [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1345 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1346 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1347 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_SCTP,
1348 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_UDP] =
1349 RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
1350 RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_VXLAN |
1351 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1352 [IXGBE_PACKET_TYPE_VXLAN_IPV4_SCTP] = RTE_PTYPE_L2_ETHER |
1353 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1354 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1355 RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L4_SCTP,
1356 [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1357 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1358 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1359 RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_SCTP,
1360 [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1361 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1362 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1363 RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_TCP,
1364 [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1365 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1366 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1367 RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_UDP,
1371 ixgbe_monitor_callback(const uint64_t value,
1372 const uint64_t arg[RTE_POWER_MONITOR_OPAQUE_SZ] __rte_unused)
1374 const uint64_t m = rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD);
1376 * we expect the DD bit to be set to 1 if this descriptor was already
1379 return (value & m) == m ? -1 : 0;
1383 ixgbe_get_monitor_addr(void *rx_queue, struct rte_power_monitor_cond *pmc)
1385 volatile union ixgbe_adv_rx_desc *rxdp;
1386 struct ixgbe_rx_queue *rxq = rx_queue;
1389 desc = rxq->rx_tail;
1390 rxdp = &rxq->rx_ring[desc];
1391 /* watch for changes in status bit */
1392 pmc->addr = &rxdp->wb.upper.status_error;
1394 /* comparison callback */
1395 pmc->fn = ixgbe_monitor_callback;
1397 /* the registers are 32-bit */
1398 pmc->size = sizeof(uint32_t);
1403 /* @note: fix ixgbe_dev_supported_ptypes_get() if any change here. */
1404 static inline uint32_t
1405 ixgbe_rxd_pkt_info_to_pkt_type(uint32_t pkt_info, uint16_t ptype_mask)
1408 if (unlikely(pkt_info & IXGBE_RXDADV_PKTTYPE_ETQF))
1409 return RTE_PTYPE_UNKNOWN;
1411 pkt_info = (pkt_info >> IXGBE_PACKET_TYPE_SHIFT) & ptype_mask;
1413 /* For tunnel packet */
1414 if (pkt_info & IXGBE_PACKET_TYPE_TUNNEL_BIT) {
1415 /* Remove the tunnel bit to save the space. */
1416 pkt_info &= IXGBE_PACKET_TYPE_MASK_TUNNEL;
1417 return ptype_table_tn[pkt_info];
1421 * For x550, if it's not tunnel,
1422 * tunnel type bit should be set to 0.
1423 * Reuse 82599's mask.
1425 pkt_info &= IXGBE_PACKET_TYPE_MASK_82599;
1427 return ptype_table[pkt_info];
1430 static inline uint64_t
1431 ixgbe_rxd_pkt_info_to_pkt_flags(uint16_t pkt_info)
1433 static uint64_t ip_rss_types_map[16] __rte_cache_aligned = {
1434 0, RTE_MBUF_F_RX_RSS_HASH, RTE_MBUF_F_RX_RSS_HASH, RTE_MBUF_F_RX_RSS_HASH,
1435 0, RTE_MBUF_F_RX_RSS_HASH, 0, RTE_MBUF_F_RX_RSS_HASH,
1436 RTE_MBUF_F_RX_RSS_HASH, 0, 0, 0,
1437 0, 0, 0, RTE_MBUF_F_RX_FDIR,
1439 #ifdef RTE_LIBRTE_IEEE1588
1440 static uint64_t ip_pkt_etqf_map[8] = {
1441 0, 0, 0, RTE_MBUF_F_RX_IEEE1588_PTP,
1445 if (likely(pkt_info & IXGBE_RXDADV_PKTTYPE_ETQF))
1446 return ip_pkt_etqf_map[(pkt_info >> 4) & 0X07] |
1447 ip_rss_types_map[pkt_info & 0XF];
1449 return ip_rss_types_map[pkt_info & 0XF];
1451 return ip_rss_types_map[pkt_info & 0XF];
1455 static inline uint64_t
1456 rx_desc_status_to_pkt_flags(uint32_t rx_status, uint64_t vlan_flags)
1461 * Check if VLAN present only.
1462 * Do not check whether L3/L4 rx checksum done by NIC or not,
1463 * That can be found from rte_eth_rxmode.offloads flag
1465 pkt_flags = (rx_status & IXGBE_RXD_STAT_VP) ? vlan_flags : 0;
1467 #ifdef RTE_LIBRTE_IEEE1588
1468 if (rx_status & IXGBE_RXD_STAT_TMST)
1469 pkt_flags = pkt_flags | RTE_MBUF_F_RX_IEEE1588_TMST;
1474 static inline uint64_t
1475 rx_desc_error_to_pkt_flags(uint32_t rx_status, uint16_t pkt_info,
1476 uint8_t rx_udp_csum_zero_err)
1481 * Bit 31: IPE, IPv4 checksum error
1482 * Bit 30: L4I, L4I integrity error
1484 static uint64_t error_to_pkt_flags_map[4] = {
1485 RTE_MBUF_F_RX_IP_CKSUM_GOOD | RTE_MBUF_F_RX_L4_CKSUM_GOOD,
1486 RTE_MBUF_F_RX_IP_CKSUM_GOOD | RTE_MBUF_F_RX_L4_CKSUM_BAD,
1487 RTE_MBUF_F_RX_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_GOOD,
1488 RTE_MBUF_F_RX_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD
1490 pkt_flags = error_to_pkt_flags_map[(rx_status >>
1491 IXGBE_RXDADV_ERR_CKSUM_BIT) & IXGBE_RXDADV_ERR_CKSUM_MSK];
1493 /* Mask out the bad UDP checksum error if the hardware has UDP zero
1494 * checksum error issue, so that the software application will then
1495 * have to recompute the checksum itself if needed.
1497 if ((rx_status & IXGBE_RXDADV_ERR_TCPE) &&
1498 (pkt_info & IXGBE_RXDADV_PKTTYPE_UDP) &&
1499 rx_udp_csum_zero_err)
1500 pkt_flags &= ~RTE_MBUF_F_RX_L4_CKSUM_BAD;
1502 if ((rx_status & IXGBE_RXD_STAT_OUTERIPCS) &&
1503 (rx_status & IXGBE_RXDADV_ERR_OUTERIPER)) {
1504 pkt_flags |= RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD;
1507 #ifdef RTE_LIB_SECURITY
1508 if (rx_status & IXGBE_RXD_STAT_SECP) {
1509 pkt_flags |= RTE_MBUF_F_RX_SEC_OFFLOAD;
1510 if (rx_status & IXGBE_RXDADV_LNKSEC_ERROR_BAD_SIG)
1511 pkt_flags |= RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED;
1519 * LOOK_AHEAD defines how many desc statuses to check beyond the
1520 * current descriptor.
1521 * It must be a pound define for optimal performance.
1522 * Do not change the value of LOOK_AHEAD, as the ixgbe_rx_scan_hw_ring
1523 * function only works with LOOK_AHEAD=8.
1525 #define LOOK_AHEAD 8
1526 #if (LOOK_AHEAD != 8)
1527 #error "PMD IXGBE: LOOK_AHEAD must be 8\n"
1530 ixgbe_rx_scan_hw_ring(struct ixgbe_rx_queue *rxq)
1532 volatile union ixgbe_adv_rx_desc *rxdp;
1533 struct ixgbe_rx_entry *rxep;
1534 struct rte_mbuf *mb;
1538 uint32_t s[LOOK_AHEAD];
1539 uint32_t pkt_info[LOOK_AHEAD];
1540 int i, j, nb_rx = 0;
1542 uint64_t vlan_flags = rxq->vlan_flags;
1544 /* get references to current descriptor and S/W ring entry */
1545 rxdp = &rxq->rx_ring[rxq->rx_tail];
1546 rxep = &rxq->sw_ring[rxq->rx_tail];
1548 status = rxdp->wb.upper.status_error;
1549 /* check to make sure there is at least 1 packet to receive */
1550 if (!(status & rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD)))
1554 * Scan LOOK_AHEAD descriptors at a time to determine which descriptors
1555 * reference packets that are ready to be received.
1557 for (i = 0; i < RTE_PMD_IXGBE_RX_MAX_BURST;
1558 i += LOOK_AHEAD, rxdp += LOOK_AHEAD, rxep += LOOK_AHEAD) {
1559 /* Read desc statuses backwards to avoid race condition */
1560 for (j = 0; j < LOOK_AHEAD; j++)
1561 s[j] = rte_le_to_cpu_32(rxdp[j].wb.upper.status_error);
1565 /* Compute how many status bits were set */
1566 for (nb_dd = 0; nb_dd < LOOK_AHEAD &&
1567 (s[nb_dd] & IXGBE_RXDADV_STAT_DD); nb_dd++)
1570 for (j = 0; j < nb_dd; j++)
1571 pkt_info[j] = rte_le_to_cpu_32(rxdp[j].wb.lower.
1576 /* Translate descriptor info to mbuf format */
1577 for (j = 0; j < nb_dd; ++j) {
1579 pkt_len = rte_le_to_cpu_16(rxdp[j].wb.upper.length) -
1581 mb->data_len = pkt_len;
1582 mb->pkt_len = pkt_len;
1583 mb->vlan_tci = rte_le_to_cpu_16(rxdp[j].wb.upper.vlan);
1585 /* convert descriptor fields to rte mbuf flags */
1586 pkt_flags = rx_desc_status_to_pkt_flags(s[j],
1588 pkt_flags |= rx_desc_error_to_pkt_flags(s[j],
1589 (uint16_t)pkt_info[j],
1590 rxq->rx_udp_csum_zero_err);
1591 pkt_flags |= ixgbe_rxd_pkt_info_to_pkt_flags
1592 ((uint16_t)pkt_info[j]);
1593 mb->ol_flags = pkt_flags;
1595 ixgbe_rxd_pkt_info_to_pkt_type
1596 (pkt_info[j], rxq->pkt_type_mask);
1598 if (likely(pkt_flags & RTE_MBUF_F_RX_RSS_HASH))
1599 mb->hash.rss = rte_le_to_cpu_32(
1600 rxdp[j].wb.lower.hi_dword.rss);
1601 else if (pkt_flags & RTE_MBUF_F_RX_FDIR) {
1602 mb->hash.fdir.hash = rte_le_to_cpu_16(
1603 rxdp[j].wb.lower.hi_dword.csum_ip.csum) &
1604 IXGBE_ATR_HASH_MASK;
1605 mb->hash.fdir.id = rte_le_to_cpu_16(
1606 rxdp[j].wb.lower.hi_dword.csum_ip.ip_id);
1610 /* Move mbuf pointers from the S/W ring to the stage */
1611 for (j = 0; j < LOOK_AHEAD; ++j) {
1612 rxq->rx_stage[i + j] = rxep[j].mbuf;
1615 /* stop if all requested packets could not be received */
1616 if (nb_dd != LOOK_AHEAD)
1620 /* clear software ring entries so we can cleanup correctly */
1621 for (i = 0; i < nb_rx; ++i) {
1622 rxq->sw_ring[rxq->rx_tail + i].mbuf = NULL;
1630 ixgbe_rx_alloc_bufs(struct ixgbe_rx_queue *rxq, bool reset_mbuf)
1632 volatile union ixgbe_adv_rx_desc *rxdp;
1633 struct ixgbe_rx_entry *rxep;
1634 struct rte_mbuf *mb;
1639 /* allocate buffers in bulk directly into the S/W ring */
1640 alloc_idx = rxq->rx_free_trigger - (rxq->rx_free_thresh - 1);
1641 rxep = &rxq->sw_ring[alloc_idx];
1642 diag = rte_mempool_get_bulk(rxq->mb_pool, (void *)rxep,
1643 rxq->rx_free_thresh);
1644 if (unlikely(diag != 0))
1647 rxdp = &rxq->rx_ring[alloc_idx];
1648 for (i = 0; i < rxq->rx_free_thresh; ++i) {
1649 /* populate the static rte mbuf fields */
1652 mb->port = rxq->port_id;
1655 rte_mbuf_refcnt_set(mb, 1);
1656 mb->data_off = RTE_PKTMBUF_HEADROOM;
1658 /* populate the descriptors */
1659 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mb));
1660 rxdp[i].read.hdr_addr = 0;
1661 rxdp[i].read.pkt_addr = dma_addr;
1664 /* update state of internal queue structure */
1665 rxq->rx_free_trigger = rxq->rx_free_trigger + rxq->rx_free_thresh;
1666 if (rxq->rx_free_trigger >= rxq->nb_rx_desc)
1667 rxq->rx_free_trigger = rxq->rx_free_thresh - 1;
1673 static inline uint16_t
1674 ixgbe_rx_fill_from_stage(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
1677 struct rte_mbuf **stage = &rxq->rx_stage[rxq->rx_next_avail];
1680 /* how many packets are ready to return? */
1681 nb_pkts = (uint16_t)RTE_MIN(nb_pkts, rxq->rx_nb_avail);
1683 /* copy mbuf pointers to the application's packet list */
1684 for (i = 0; i < nb_pkts; ++i)
1685 rx_pkts[i] = stage[i];
1687 /* update internal queue state */
1688 rxq->rx_nb_avail = (uint16_t)(rxq->rx_nb_avail - nb_pkts);
1689 rxq->rx_next_avail = (uint16_t)(rxq->rx_next_avail + nb_pkts);
1694 static inline uint16_t
1695 rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
1698 struct ixgbe_rx_queue *rxq = (struct ixgbe_rx_queue *)rx_queue;
1701 /* Any previously recv'd pkts will be returned from the Rx stage */
1702 if (rxq->rx_nb_avail)
1703 return ixgbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
1705 /* Scan the H/W ring for packets to receive */
1706 nb_rx = (uint16_t)ixgbe_rx_scan_hw_ring(rxq);
1708 /* update internal queue state */
1709 rxq->rx_next_avail = 0;
1710 rxq->rx_nb_avail = nb_rx;
1711 rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_rx);
1713 /* if required, allocate new buffers to replenish descriptors */
1714 if (rxq->rx_tail > rxq->rx_free_trigger) {
1715 uint16_t cur_free_trigger = rxq->rx_free_trigger;
1717 if (ixgbe_rx_alloc_bufs(rxq, true) != 0) {
1720 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u "
1721 "queue_id=%u", (unsigned) rxq->port_id,
1722 (unsigned) rxq->queue_id);
1724 rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed +=
1725 rxq->rx_free_thresh;
1728 * Need to rewind any previous receives if we cannot
1729 * allocate new buffers to replenish the old ones.
1731 rxq->rx_nb_avail = 0;
1732 rxq->rx_tail = (uint16_t)(rxq->rx_tail - nb_rx);
1733 for (i = 0, j = rxq->rx_tail; i < nb_rx; ++i, ++j)
1734 rxq->sw_ring[j].mbuf = rxq->rx_stage[i];
1739 /* update tail pointer */
1741 IXGBE_PCI_REG_WC_WRITE_RELAXED(rxq->rdt_reg_addr,
1745 if (rxq->rx_tail >= rxq->nb_rx_desc)
1748 /* received any packets this loop? */
1749 if (rxq->rx_nb_avail)
1750 return ixgbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
1755 /* split requests into chunks of size RTE_PMD_IXGBE_RX_MAX_BURST */
1757 ixgbe_recv_pkts_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
1762 if (unlikely(nb_pkts == 0))
1765 if (likely(nb_pkts <= RTE_PMD_IXGBE_RX_MAX_BURST))
1766 return rx_recv_pkts(rx_queue, rx_pkts, nb_pkts);
1768 /* request is relatively large, chunk it up */
1773 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_IXGBE_RX_MAX_BURST);
1774 ret = rx_recv_pkts(rx_queue, &rx_pkts[nb_rx], n);
1775 nb_rx = (uint16_t)(nb_rx + ret);
1776 nb_pkts = (uint16_t)(nb_pkts - ret);
1785 ixgbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
1788 struct ixgbe_rx_queue *rxq;
1789 volatile union ixgbe_adv_rx_desc *rx_ring;
1790 volatile union ixgbe_adv_rx_desc *rxdp;
1791 struct ixgbe_rx_entry *sw_ring;
1792 struct ixgbe_rx_entry *rxe;
1793 struct rte_mbuf *rxm;
1794 struct rte_mbuf *nmb;
1795 union ixgbe_adv_rx_desc rxd;
1804 uint64_t vlan_flags;
1809 rx_id = rxq->rx_tail;
1810 rx_ring = rxq->rx_ring;
1811 sw_ring = rxq->sw_ring;
1812 vlan_flags = rxq->vlan_flags;
1813 while (nb_rx < nb_pkts) {
1815 * The order of operations here is important as the DD status
1816 * bit must not be read after any other descriptor fields.
1817 * rx_ring and rxdp are pointing to volatile data so the order
1818 * of accesses cannot be reordered by the compiler. If they were
1819 * not volatile, they could be reordered which could lead to
1820 * using invalid descriptor fields when read from rxd.
1822 rxdp = &rx_ring[rx_id];
1823 staterr = rxdp->wb.upper.status_error;
1824 if (!(staterr & rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD)))
1831 * If the IXGBE_RXDADV_STAT_EOP flag is not set, the RX packet
1832 * is likely to be invalid and to be dropped by the various
1833 * validation checks performed by the network stack.
1835 * Allocate a new mbuf to replenish the RX ring descriptor.
1836 * If the allocation fails:
1837 * - arrange for that RX descriptor to be the first one
1838 * being parsed the next time the receive function is
1839 * invoked [on the same queue].
1841 * - Stop parsing the RX ring and return immediately.
1843 * This policy do not drop the packet received in the RX
1844 * descriptor for which the allocation of a new mbuf failed.
1845 * Thus, it allows that packet to be later retrieved if
1846 * mbuf have been freed in the mean time.
1847 * As a side effect, holding RX descriptors instead of
1848 * systematically giving them back to the NIC may lead to
1849 * RX ring exhaustion situations.
1850 * However, the NIC can gracefully prevent such situations
1851 * to happen by sending specific "back-pressure" flow control
1852 * frames to its peer(s).
1854 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u "
1855 "ext_err_stat=0x%08x pkt_len=%u",
1856 (unsigned) rxq->port_id, (unsigned) rxq->queue_id,
1857 (unsigned) rx_id, (unsigned) staterr,
1858 (unsigned) rte_le_to_cpu_16(rxd.wb.upper.length));
1860 nmb = rte_mbuf_raw_alloc(rxq->mb_pool);
1862 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u "
1863 "queue_id=%u", (unsigned) rxq->port_id,
1864 (unsigned) rxq->queue_id);
1865 rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed++;
1870 rxe = &sw_ring[rx_id];
1872 if (rx_id == rxq->nb_rx_desc)
1875 /* Prefetch next mbuf while processing current one. */
1876 rte_ixgbe_prefetch(sw_ring[rx_id].mbuf);
1879 * When next RX descriptor is on a cache-line boundary,
1880 * prefetch the next 4 RX descriptors and the next 8 pointers
1883 if ((rx_id & 0x3) == 0) {
1884 rte_ixgbe_prefetch(&rx_ring[rx_id]);
1885 rte_ixgbe_prefetch(&sw_ring[rx_id]);
1891 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
1892 rxdp->read.hdr_addr = 0;
1893 rxdp->read.pkt_addr = dma_addr;
1896 * Initialize the returned mbuf.
1897 * 1) setup generic mbuf fields:
1898 * - number of segments,
1901 * - RX port identifier.
1902 * 2) integrate hardware offload data, if any:
1903 * - RSS flag & hash,
1904 * - IP checksum flag,
1905 * - VLAN TCI, if any,
1908 pkt_len = (uint16_t) (rte_le_to_cpu_16(rxd.wb.upper.length) -
1910 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1911 rte_packet_prefetch((char *)rxm->buf_addr + rxm->data_off);
1914 rxm->pkt_len = pkt_len;
1915 rxm->data_len = pkt_len;
1916 rxm->port = rxq->port_id;
1918 pkt_info = rte_le_to_cpu_32(rxd.wb.lower.lo_dword.data);
1919 /* Only valid if RTE_MBUF_F_RX_VLAN set in pkt_flags */
1920 rxm->vlan_tci = rte_le_to_cpu_16(rxd.wb.upper.vlan);
1922 pkt_flags = rx_desc_status_to_pkt_flags(staterr, vlan_flags);
1923 pkt_flags = pkt_flags |
1924 rx_desc_error_to_pkt_flags(staterr, (uint16_t)pkt_info,
1925 rxq->rx_udp_csum_zero_err);
1926 pkt_flags = pkt_flags |
1927 ixgbe_rxd_pkt_info_to_pkt_flags((uint16_t)pkt_info);
1928 rxm->ol_flags = pkt_flags;
1930 ixgbe_rxd_pkt_info_to_pkt_type(pkt_info,
1931 rxq->pkt_type_mask);
1933 if (likely(pkt_flags & RTE_MBUF_F_RX_RSS_HASH))
1934 rxm->hash.rss = rte_le_to_cpu_32(
1935 rxd.wb.lower.hi_dword.rss);
1936 else if (pkt_flags & RTE_MBUF_F_RX_FDIR) {
1937 rxm->hash.fdir.hash = rte_le_to_cpu_16(
1938 rxd.wb.lower.hi_dword.csum_ip.csum) &
1939 IXGBE_ATR_HASH_MASK;
1940 rxm->hash.fdir.id = rte_le_to_cpu_16(
1941 rxd.wb.lower.hi_dword.csum_ip.ip_id);
1944 * Store the mbuf address into the next entry of the array
1945 * of returned packets.
1947 rx_pkts[nb_rx++] = rxm;
1949 rxq->rx_tail = rx_id;
1952 * If the number of free RX descriptors is greater than the RX free
1953 * threshold of the queue, advance the Receive Descriptor Tail (RDT)
1955 * Update the RDT with the value of the last processed RX descriptor
1956 * minus 1, to guarantee that the RDT register is never equal to the
1957 * RDH register, which creates a "full" ring situation from the
1958 * hardware point of view...
1960 nb_hold = (uint16_t) (nb_hold + rxq->nb_rx_hold);
1961 if (nb_hold > rxq->rx_free_thresh) {
1962 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u "
1963 "nb_hold=%u nb_rx=%u",
1964 (unsigned) rxq->port_id, (unsigned) rxq->queue_id,
1965 (unsigned) rx_id, (unsigned) nb_hold,
1967 rx_id = (uint16_t) ((rx_id == 0) ?
1968 (rxq->nb_rx_desc - 1) : (rx_id - 1));
1969 IXGBE_PCI_REG_WC_WRITE(rxq->rdt_reg_addr, rx_id);
1972 rxq->nb_rx_hold = nb_hold;
1977 * Detect an RSC descriptor.
1979 static inline uint32_t
1980 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
1982 return (rte_le_to_cpu_32(rx->wb.lower.lo_dword.data) &
1983 IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
1987 * ixgbe_fill_cluster_head_buf - fill the first mbuf of the returned packet
1989 * Fill the following info in the HEAD buffer of the Rx cluster:
1990 * - RX port identifier
1991 * - hardware offload data, if any:
1993 * - IP checksum flag
1994 * - VLAN TCI, if any
1996 * @head HEAD of the packet cluster
1997 * @desc HW descriptor to get data from
1998 * @rxq Pointer to the Rx queue
2001 ixgbe_fill_cluster_head_buf(
2002 struct rte_mbuf *head,
2003 union ixgbe_adv_rx_desc *desc,
2004 struct ixgbe_rx_queue *rxq,
2010 head->port = rxq->port_id;
2012 /* The vlan_tci field is only valid when RTE_MBUF_F_RX_VLAN is
2013 * set in the pkt_flags field.
2015 head->vlan_tci = rte_le_to_cpu_16(desc->wb.upper.vlan);
2016 pkt_info = rte_le_to_cpu_32(desc->wb.lower.lo_dword.data);
2017 pkt_flags = rx_desc_status_to_pkt_flags(staterr, rxq->vlan_flags);
2018 pkt_flags |= rx_desc_error_to_pkt_flags(staterr, (uint16_t)pkt_info,
2019 rxq->rx_udp_csum_zero_err);
2020 pkt_flags |= ixgbe_rxd_pkt_info_to_pkt_flags((uint16_t)pkt_info);
2021 head->ol_flags = pkt_flags;
2023 ixgbe_rxd_pkt_info_to_pkt_type(pkt_info, rxq->pkt_type_mask);
2025 if (likely(pkt_flags & RTE_MBUF_F_RX_RSS_HASH))
2026 head->hash.rss = rte_le_to_cpu_32(desc->wb.lower.hi_dword.rss);
2027 else if (pkt_flags & RTE_MBUF_F_RX_FDIR) {
2028 head->hash.fdir.hash =
2029 rte_le_to_cpu_16(desc->wb.lower.hi_dword.csum_ip.csum)
2030 & IXGBE_ATR_HASH_MASK;
2031 head->hash.fdir.id =
2032 rte_le_to_cpu_16(desc->wb.lower.hi_dword.csum_ip.ip_id);
2037 * ixgbe_recv_pkts_lro - receive handler for and LRO case.
2039 * @rx_queue Rx queue handle
2040 * @rx_pkts table of received packets
2041 * @nb_pkts size of rx_pkts table
2042 * @bulk_alloc if TRUE bulk allocation is used for a HW ring refilling
2044 * Handles the Rx HW ring completions when RSC feature is configured. Uses an
2045 * additional ring of ixgbe_rsc_entry's that will hold the relevant RSC info.
2047 * We use the same logic as in Linux and in FreeBSD ixgbe drivers:
2048 * 1) When non-EOP RSC completion arrives:
2049 * a) Update the HEAD of the current RSC aggregation cluster with the new
2050 * segment's data length.
2051 * b) Set the "next" pointer of the current segment to point to the segment
2052 * at the NEXTP index.
2053 * c) Pass the HEAD of RSC aggregation cluster on to the next NEXTP entry
2054 * in the sw_rsc_ring.
2055 * 2) When EOP arrives we just update the cluster's total length and offload
2056 * flags and deliver the cluster up to the upper layers. In our case - put it
2057 * in the rx_pkts table.
2059 * Returns the number of received packets/clusters (according to the "bulk
2060 * receive" interface).
2062 static inline uint16_t
2063 ixgbe_recv_pkts_lro(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts,
2066 struct ixgbe_rx_queue *rxq = rx_queue;
2067 volatile union ixgbe_adv_rx_desc *rx_ring = rxq->rx_ring;
2068 struct ixgbe_rx_entry *sw_ring = rxq->sw_ring;
2069 struct ixgbe_scattered_rx_entry *sw_sc_ring = rxq->sw_sc_ring;
2070 uint16_t rx_id = rxq->rx_tail;
2072 uint16_t nb_hold = rxq->nb_rx_hold;
2073 uint16_t prev_id = rxq->rx_tail;
2075 while (nb_rx < nb_pkts) {
2077 struct ixgbe_rx_entry *rxe;
2078 struct ixgbe_scattered_rx_entry *sc_entry;
2079 struct ixgbe_scattered_rx_entry *next_sc_entry = NULL;
2080 struct ixgbe_rx_entry *next_rxe = NULL;
2081 struct rte_mbuf *first_seg;
2082 struct rte_mbuf *rxm;
2083 struct rte_mbuf *nmb = NULL;
2084 union ixgbe_adv_rx_desc rxd;
2087 volatile union ixgbe_adv_rx_desc *rxdp;
2092 * The code in this whole file uses the volatile pointer to
2093 * ensure the read ordering of the status and the rest of the
2094 * descriptor fields (on the compiler level only!!!). This is so
2095 * UGLY - why not to just use the compiler barrier instead? DPDK
2096 * even has the rte_compiler_barrier() for that.
2098 * But most importantly this is just wrong because this doesn't
2099 * ensure memory ordering in a general case at all. For
2100 * instance, DPDK is supposed to work on Power CPUs where
2101 * compiler barrier may just not be enough!
2103 * I tried to write only this function properly to have a
2104 * starting point (as a part of an LRO/RSC series) but the
2105 * compiler cursed at me when I tried to cast away the
2106 * "volatile" from rx_ring (yes, it's volatile too!!!). So, I'm
2107 * keeping it the way it is for now.
2109 * The code in this file is broken in so many other places and
2110 * will just not work on a big endian CPU anyway therefore the
2111 * lines below will have to be revisited together with the rest
2115 * - Get rid of "volatile" and let the compiler do its job.
2116 * - Use the proper memory barrier (rte_rmb()) to ensure the
2117 * memory ordering below.
2119 rxdp = &rx_ring[rx_id];
2120 staterr = rte_le_to_cpu_32(rxdp->wb.upper.status_error);
2122 if (!(staterr & IXGBE_RXDADV_STAT_DD))
2127 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u "
2128 "staterr=0x%x data_len=%u",
2129 rxq->port_id, rxq->queue_id, rx_id, staterr,
2130 rte_le_to_cpu_16(rxd.wb.upper.length));
2133 nmb = rte_mbuf_raw_alloc(rxq->mb_pool);
2135 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed "
2136 "port_id=%u queue_id=%u",
2137 rxq->port_id, rxq->queue_id);
2139 rte_eth_devices[rxq->port_id].data->
2140 rx_mbuf_alloc_failed++;
2143 } else if (nb_hold > rxq->rx_free_thresh) {
2144 uint16_t next_rdt = rxq->rx_free_trigger;
2146 if (!ixgbe_rx_alloc_bufs(rxq, false)) {
2148 IXGBE_PCI_REG_WC_WRITE_RELAXED(
2151 nb_hold -= rxq->rx_free_thresh;
2153 PMD_RX_LOG(DEBUG, "RX bulk alloc failed "
2154 "port_id=%u queue_id=%u",
2155 rxq->port_id, rxq->queue_id);
2157 rte_eth_devices[rxq->port_id].data->
2158 rx_mbuf_alloc_failed++;
2164 rxe = &sw_ring[rx_id];
2165 eop = staterr & IXGBE_RXDADV_STAT_EOP;
2167 next_id = rx_id + 1;
2168 if (next_id == rxq->nb_rx_desc)
2171 /* Prefetch next mbuf while processing current one. */
2172 rte_ixgbe_prefetch(sw_ring[next_id].mbuf);
2175 * When next RX descriptor is on a cache-line boundary,
2176 * prefetch the next 4 RX descriptors and the next 4 pointers
2179 if ((next_id & 0x3) == 0) {
2180 rte_ixgbe_prefetch(&rx_ring[next_id]);
2181 rte_ixgbe_prefetch(&sw_ring[next_id]);
2188 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
2190 * Update RX descriptor with the physical address of the
2191 * new data buffer of the new allocated mbuf.
2195 rxm->data_off = RTE_PKTMBUF_HEADROOM;
2196 rxdp->read.hdr_addr = 0;
2197 rxdp->read.pkt_addr = dma;
2202 * Set data length & data buffer address of mbuf.
2204 data_len = rte_le_to_cpu_16(rxd.wb.upper.length);
2205 rxm->data_len = data_len;
2210 * Get next descriptor index:
2211 * - For RSC it's in the NEXTP field.
2212 * - For a scattered packet - it's just a following
2215 if (ixgbe_rsc_count(&rxd))
2217 (staterr & IXGBE_RXDADV_NEXTP_MASK) >>
2218 IXGBE_RXDADV_NEXTP_SHIFT;
2222 next_sc_entry = &sw_sc_ring[nextp_id];
2223 next_rxe = &sw_ring[nextp_id];
2224 rte_ixgbe_prefetch(next_rxe);
2227 sc_entry = &sw_sc_ring[rx_id];
2228 first_seg = sc_entry->fbuf;
2229 sc_entry->fbuf = NULL;
2232 * If this is the first buffer of the received packet,
2233 * set the pointer to the first mbuf of the packet and
2234 * initialize its context.
2235 * Otherwise, update the total length and the number of segments
2236 * of the current scattered packet, and update the pointer to
2237 * the last mbuf of the current packet.
2239 if (first_seg == NULL) {
2241 first_seg->pkt_len = data_len;
2242 first_seg->nb_segs = 1;
2244 first_seg->pkt_len += data_len;
2245 first_seg->nb_segs++;
2252 * If this is not the last buffer of the received packet, update
2253 * the pointer to the first mbuf at the NEXTP entry in the
2254 * sw_sc_ring and continue to parse the RX ring.
2256 if (!eop && next_rxe) {
2257 rxm->next = next_rxe->mbuf;
2258 next_sc_entry->fbuf = first_seg;
2262 /* Initialize the first mbuf of the returned packet */
2263 ixgbe_fill_cluster_head_buf(first_seg, &rxd, rxq, staterr);
2266 * Deal with the case, when HW CRC srip is disabled.
2267 * That can't happen when LRO is enabled, but still could
2268 * happen for scattered RX mode.
2270 first_seg->pkt_len -= rxq->crc_len;
2271 if (unlikely(rxm->data_len <= rxq->crc_len)) {
2272 struct rte_mbuf *lp;
2274 for (lp = first_seg; lp->next != rxm; lp = lp->next)
2277 first_seg->nb_segs--;
2278 lp->data_len -= rxq->crc_len - rxm->data_len;
2280 rte_pktmbuf_free_seg(rxm);
2282 rxm->data_len -= rxq->crc_len;
2284 /* Prefetch data of first segment, if configured to do so. */
2285 rte_packet_prefetch((char *)first_seg->buf_addr +
2286 first_seg->data_off);
2289 * Store the mbuf address into the next entry of the array
2290 * of returned packets.
2292 rx_pkts[nb_rx++] = first_seg;
2296 * Record index of the next RX descriptor to probe.
2298 rxq->rx_tail = rx_id;
2301 * If the number of free RX descriptors is greater than the RX free
2302 * threshold of the queue, advance the Receive Descriptor Tail (RDT)
2304 * Update the RDT with the value of the last processed RX descriptor
2305 * minus 1, to guarantee that the RDT register is never equal to the
2306 * RDH register, which creates a "full" ring situation from the
2307 * hardware point of view...
2309 if (!bulk_alloc && nb_hold > rxq->rx_free_thresh) {
2310 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u "
2311 "nb_hold=%u nb_rx=%u",
2312 rxq->port_id, rxq->queue_id, rx_id, nb_hold, nb_rx);
2315 IXGBE_PCI_REG_WC_WRITE_RELAXED(rxq->rdt_reg_addr, prev_id);
2319 rxq->nb_rx_hold = nb_hold;
2324 ixgbe_recv_pkts_lro_single_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
2327 return ixgbe_recv_pkts_lro(rx_queue, rx_pkts, nb_pkts, false);
2331 ixgbe_recv_pkts_lro_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
2334 return ixgbe_recv_pkts_lro(rx_queue, rx_pkts, nb_pkts, true);
2337 /*********************************************************************
2339 * Queue management functions
2341 **********************************************************************/
2343 static void __rte_cold
2344 ixgbe_tx_queue_release_mbufs(struct ixgbe_tx_queue *txq)
2348 if (txq->sw_ring != NULL) {
2349 for (i = 0; i < txq->nb_tx_desc; i++) {
2350 if (txq->sw_ring[i].mbuf != NULL) {
2351 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
2352 txq->sw_ring[i].mbuf = NULL;
2359 ixgbe_tx_done_cleanup_full(struct ixgbe_tx_queue *txq, uint32_t free_cnt)
2361 struct ixgbe_tx_entry *swr_ring = txq->sw_ring;
2362 uint16_t i, tx_last, tx_id;
2363 uint16_t nb_tx_free_last;
2364 uint16_t nb_tx_to_clean;
2367 /* Start free mbuf from the next of tx_tail */
2368 tx_last = txq->tx_tail;
2369 tx_id = swr_ring[tx_last].next_id;
2371 if (txq->nb_tx_free == 0 && ixgbe_xmit_cleanup(txq))
2374 nb_tx_to_clean = txq->nb_tx_free;
2375 nb_tx_free_last = txq->nb_tx_free;
2377 free_cnt = txq->nb_tx_desc;
2379 /* Loop through swr_ring to count the amount of
2380 * freeable mubfs and packets.
2382 for (pkt_cnt = 0; pkt_cnt < free_cnt; ) {
2383 for (i = 0; i < nb_tx_to_clean &&
2384 pkt_cnt < free_cnt &&
2385 tx_id != tx_last; i++) {
2386 if (swr_ring[tx_id].mbuf != NULL) {
2387 rte_pktmbuf_free_seg(swr_ring[tx_id].mbuf);
2388 swr_ring[tx_id].mbuf = NULL;
2391 * last segment in the packet,
2392 * increment packet count
2394 pkt_cnt += (swr_ring[tx_id].last_id == tx_id);
2397 tx_id = swr_ring[tx_id].next_id;
2400 if (txq->tx_rs_thresh > txq->nb_tx_desc -
2401 txq->nb_tx_free || tx_id == tx_last)
2404 if (pkt_cnt < free_cnt) {
2405 if (ixgbe_xmit_cleanup(txq))
2408 nb_tx_to_clean = txq->nb_tx_free - nb_tx_free_last;
2409 nb_tx_free_last = txq->nb_tx_free;
2413 return (int)pkt_cnt;
2417 ixgbe_tx_done_cleanup_simple(struct ixgbe_tx_queue *txq,
2422 if (free_cnt == 0 || free_cnt > txq->nb_tx_desc)
2423 free_cnt = txq->nb_tx_desc;
2425 cnt = free_cnt - free_cnt % txq->tx_rs_thresh;
2427 for (i = 0; i < cnt; i += n) {
2428 if (txq->nb_tx_desc - txq->nb_tx_free < txq->tx_rs_thresh)
2431 n = ixgbe_tx_free_bufs(txq);
2441 ixgbe_tx_done_cleanup_vec(struct ixgbe_tx_queue *txq __rte_unused,
2442 uint32_t free_cnt __rte_unused)
2448 ixgbe_dev_tx_done_cleanup(void *tx_queue, uint32_t free_cnt)
2450 struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue;
2451 if (txq->offloads == 0 &&
2452 #ifdef RTE_LIB_SECURITY
2453 !(txq->using_ipsec) &&
2455 txq->tx_rs_thresh >= RTE_PMD_IXGBE_TX_MAX_BURST) {
2456 if (txq->tx_rs_thresh <= RTE_IXGBE_TX_MAX_FREE_BUF_SZ &&
2457 rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128 &&
2458 (rte_eal_process_type() != RTE_PROC_PRIMARY ||
2459 txq->sw_ring_v != NULL)) {
2460 return ixgbe_tx_done_cleanup_vec(txq, free_cnt);
2462 return ixgbe_tx_done_cleanup_simple(txq, free_cnt);
2466 return ixgbe_tx_done_cleanup_full(txq, free_cnt);
2469 static void __rte_cold
2470 ixgbe_tx_free_swring(struct ixgbe_tx_queue *txq)
2473 txq->sw_ring != NULL)
2474 rte_free(txq->sw_ring);
2477 static void __rte_cold
2478 ixgbe_tx_queue_release(struct ixgbe_tx_queue *txq)
2480 if (txq != NULL && txq->ops != NULL) {
2481 txq->ops->release_mbufs(txq);
2482 txq->ops->free_swring(txq);
2483 rte_memzone_free(txq->mz);
2489 ixgbe_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
2491 ixgbe_tx_queue_release(dev->data->tx_queues[qid]);
2494 /* (Re)set dynamic ixgbe_tx_queue fields to defaults */
2495 static void __rte_cold
2496 ixgbe_reset_tx_queue(struct ixgbe_tx_queue *txq)
2498 static const union ixgbe_adv_tx_desc zeroed_desc = {{0}};
2499 struct ixgbe_tx_entry *txe = txq->sw_ring;
2502 /* Zero out HW ring memory */
2503 for (i = 0; i < txq->nb_tx_desc; i++) {
2504 txq->tx_ring[i] = zeroed_desc;
2507 /* Initialize SW ring entries */
2508 prev = (uint16_t) (txq->nb_tx_desc - 1);
2509 for (i = 0; i < txq->nb_tx_desc; i++) {
2510 volatile union ixgbe_adv_tx_desc *txd = &txq->tx_ring[i];
2512 txd->wb.status = rte_cpu_to_le_32(IXGBE_TXD_STAT_DD);
2515 txe[prev].next_id = i;
2519 txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
2520 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
2523 txq->nb_tx_used = 0;
2525 * Always allow 1 descriptor to be un-allocated to avoid
2526 * a H/W race condition
2528 txq->last_desc_cleaned = (uint16_t)(txq->nb_tx_desc - 1);
2529 txq->nb_tx_free = (uint16_t)(txq->nb_tx_desc - 1);
2531 memset((void *)&txq->ctx_cache, 0,
2532 IXGBE_CTX_NUM * sizeof(struct ixgbe_advctx_info));
2535 static const struct ixgbe_txq_ops def_txq_ops = {
2536 .release_mbufs = ixgbe_tx_queue_release_mbufs,
2537 .free_swring = ixgbe_tx_free_swring,
2538 .reset = ixgbe_reset_tx_queue,
2541 /* Takes an ethdev and a queue and sets up the tx function to be used based on
2542 * the queue parameters. Used in tx_queue_setup by primary process and then
2543 * in dev_init by secondary process when attaching to an existing ethdev.
2546 ixgbe_set_tx_function(struct rte_eth_dev *dev, struct ixgbe_tx_queue *txq)
2548 /* Use a simple Tx queue (no offloads, no multi segs) if possible */
2549 if ((txq->offloads == 0) &&
2550 #ifdef RTE_LIB_SECURITY
2551 !(txq->using_ipsec) &&
2553 (txq->tx_rs_thresh >= RTE_PMD_IXGBE_TX_MAX_BURST)) {
2554 PMD_INIT_LOG(DEBUG, "Using simple tx code path");
2555 dev->tx_pkt_prepare = NULL;
2556 if (txq->tx_rs_thresh <= RTE_IXGBE_TX_MAX_FREE_BUF_SZ &&
2557 rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128 &&
2558 (rte_eal_process_type() != RTE_PROC_PRIMARY ||
2559 ixgbe_txq_vec_setup(txq) == 0)) {
2560 PMD_INIT_LOG(DEBUG, "Vector tx enabled.");
2561 dev->tx_pkt_burst = ixgbe_xmit_pkts_vec;
2563 dev->tx_pkt_burst = ixgbe_xmit_pkts_simple;
2565 PMD_INIT_LOG(DEBUG, "Using full-featured tx code path");
2567 " - offloads = 0x%" PRIx64,
2570 " - tx_rs_thresh = %lu " "[RTE_PMD_IXGBE_TX_MAX_BURST=%lu]",
2571 (unsigned long)txq->tx_rs_thresh,
2572 (unsigned long)RTE_PMD_IXGBE_TX_MAX_BURST);
2573 dev->tx_pkt_burst = ixgbe_xmit_pkts;
2574 dev->tx_pkt_prepare = ixgbe_prep_pkts;
2579 ixgbe_get_tx_queue_offloads(struct rte_eth_dev *dev)
2587 ixgbe_get_tx_port_offloads(struct rte_eth_dev *dev)
2589 uint64_t tx_offload_capa;
2590 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2593 RTE_ETH_TX_OFFLOAD_VLAN_INSERT |
2594 RTE_ETH_TX_OFFLOAD_IPV4_CKSUM |
2595 RTE_ETH_TX_OFFLOAD_UDP_CKSUM |
2596 RTE_ETH_TX_OFFLOAD_TCP_CKSUM |
2597 RTE_ETH_TX_OFFLOAD_SCTP_CKSUM |
2598 RTE_ETH_TX_OFFLOAD_TCP_TSO |
2599 RTE_ETH_TX_OFFLOAD_MULTI_SEGS;
2601 if (hw->mac.type == ixgbe_mac_82599EB ||
2602 hw->mac.type == ixgbe_mac_X540)
2603 tx_offload_capa |= RTE_ETH_TX_OFFLOAD_MACSEC_INSERT;
2605 if (hw->mac.type == ixgbe_mac_X550 ||
2606 hw->mac.type == ixgbe_mac_X550EM_x ||
2607 hw->mac.type == ixgbe_mac_X550EM_a)
2608 tx_offload_capa |= RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM;
2610 #ifdef RTE_LIB_SECURITY
2611 if (dev->security_ctx)
2612 tx_offload_capa |= RTE_ETH_TX_OFFLOAD_SECURITY;
2614 return tx_offload_capa;
2618 ixgbe_dev_tx_queue_setup(struct rte_eth_dev *dev,
2621 unsigned int socket_id,
2622 const struct rte_eth_txconf *tx_conf)
2624 const struct rte_memzone *tz;
2625 struct ixgbe_tx_queue *txq;
2626 struct ixgbe_hw *hw;
2627 uint16_t tx_rs_thresh, tx_free_thresh;
2630 PMD_INIT_FUNC_TRACE();
2631 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2633 offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads;
2636 * Validate number of transmit descriptors.
2637 * It must not exceed hardware maximum, and must be multiple
2640 if (nb_desc % IXGBE_TXD_ALIGN != 0 ||
2641 (nb_desc > IXGBE_MAX_RING_DESC) ||
2642 (nb_desc < IXGBE_MIN_RING_DESC)) {
2647 * The following two parameters control the setting of the RS bit on
2648 * transmit descriptors.
2649 * TX descriptors will have their RS bit set after txq->tx_rs_thresh
2650 * descriptors have been used.
2651 * The TX descriptor ring will be cleaned after txq->tx_free_thresh
2652 * descriptors are used or if the number of descriptors required
2653 * to transmit a packet is greater than the number of free TX
2655 * The following constraints must be satisfied:
2656 * tx_rs_thresh must be greater than 0.
2657 * tx_rs_thresh must be less than the size of the ring minus 2.
2658 * tx_rs_thresh must be less than or equal to tx_free_thresh.
2659 * tx_rs_thresh must be a divisor of the ring size.
2660 * tx_free_thresh must be greater than 0.
2661 * tx_free_thresh must be less than the size of the ring minus 3.
2662 * tx_free_thresh + tx_rs_thresh must not exceed nb_desc.
2663 * One descriptor in the TX ring is used as a sentinel to avoid a
2664 * H/W race condition, hence the maximum threshold constraints.
2665 * When set to zero use default values.
2667 tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ?
2668 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH);
2669 /* force tx_rs_thresh to adapt an aggressive tx_free_thresh */
2670 tx_rs_thresh = (DEFAULT_TX_RS_THRESH + tx_free_thresh > nb_desc) ?
2671 nb_desc - tx_free_thresh : DEFAULT_TX_RS_THRESH;
2672 if (tx_conf->tx_rs_thresh > 0)
2673 tx_rs_thresh = tx_conf->tx_rs_thresh;
2674 if (tx_rs_thresh + tx_free_thresh > nb_desc) {
2675 PMD_INIT_LOG(ERR, "tx_rs_thresh + tx_free_thresh must not "
2676 "exceed nb_desc. (tx_rs_thresh=%u "
2677 "tx_free_thresh=%u nb_desc=%u port = %d queue=%d)",
2678 (unsigned int)tx_rs_thresh,
2679 (unsigned int)tx_free_thresh,
2680 (unsigned int)nb_desc,
2681 (int)dev->data->port_id,
2685 if (tx_rs_thresh >= (nb_desc - 2)) {
2686 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than the number "
2687 "of TX descriptors minus 2. (tx_rs_thresh=%u "
2688 "port=%d queue=%d)", (unsigned int)tx_rs_thresh,
2689 (int)dev->data->port_id, (int)queue_idx);
2692 if (tx_rs_thresh > DEFAULT_TX_RS_THRESH) {
2693 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less or equal than %u. "
2694 "(tx_rs_thresh=%u port=%d queue=%d)",
2695 DEFAULT_TX_RS_THRESH, (unsigned int)tx_rs_thresh,
2696 (int)dev->data->port_id, (int)queue_idx);
2699 if (tx_free_thresh >= (nb_desc - 3)) {
2700 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than the "
2701 "tx_free_thresh must be less than the number of "
2702 "TX descriptors minus 3. (tx_free_thresh=%u "
2703 "port=%d queue=%d)",
2704 (unsigned int)tx_free_thresh,
2705 (int)dev->data->port_id, (int)queue_idx);
2708 if (tx_rs_thresh > tx_free_thresh) {
2709 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than or equal to "
2710 "tx_free_thresh. (tx_free_thresh=%u "
2711 "tx_rs_thresh=%u port=%d queue=%d)",
2712 (unsigned int)tx_free_thresh,
2713 (unsigned int)tx_rs_thresh,
2714 (int)dev->data->port_id,
2718 if ((nb_desc % tx_rs_thresh) != 0) {
2719 PMD_INIT_LOG(ERR, "tx_rs_thresh must be a divisor of the "
2720 "number of TX descriptors. (tx_rs_thresh=%u "
2721 "port=%d queue=%d)", (unsigned int)tx_rs_thresh,
2722 (int)dev->data->port_id, (int)queue_idx);
2727 * If rs_bit_thresh is greater than 1, then TX WTHRESH should be
2728 * set to 0. If WTHRESH is greater than zero, the RS bit is ignored
2729 * by the NIC and all descriptors are written back after the NIC
2730 * accumulates WTHRESH descriptors.
2732 if ((tx_rs_thresh > 1) && (tx_conf->tx_thresh.wthresh != 0)) {
2733 PMD_INIT_LOG(ERR, "TX WTHRESH must be set to 0 if "
2734 "tx_rs_thresh is greater than 1. (tx_rs_thresh=%u "
2735 "port=%d queue=%d)", (unsigned int)tx_rs_thresh,
2736 (int)dev->data->port_id, (int)queue_idx);
2740 /* Free memory prior to re-allocation if needed... */
2741 if (dev->data->tx_queues[queue_idx] != NULL) {
2742 ixgbe_tx_queue_release(dev->data->tx_queues[queue_idx]);
2743 dev->data->tx_queues[queue_idx] = NULL;
2746 /* First allocate the tx queue data structure */
2747 txq = rte_zmalloc_socket("ethdev TX queue", sizeof(struct ixgbe_tx_queue),
2748 RTE_CACHE_LINE_SIZE, socket_id);
2753 * Allocate TX ring hardware descriptors. A memzone large enough to
2754 * handle the maximum ring size is allocated in order to allow for
2755 * resizing in later calls to the queue setup function.
2757 tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx,
2758 sizeof(union ixgbe_adv_tx_desc) * IXGBE_MAX_RING_DESC,
2759 IXGBE_ALIGN, socket_id);
2761 ixgbe_tx_queue_release(txq);
2766 txq->nb_tx_desc = nb_desc;
2767 txq->tx_rs_thresh = tx_rs_thresh;
2768 txq->tx_free_thresh = tx_free_thresh;
2769 txq->pthresh = tx_conf->tx_thresh.pthresh;
2770 txq->hthresh = tx_conf->tx_thresh.hthresh;
2771 txq->wthresh = tx_conf->tx_thresh.wthresh;
2772 txq->queue_id = queue_idx;
2773 txq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ?
2774 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx);
2775 txq->port_id = dev->data->port_id;
2776 txq->offloads = offloads;
2777 txq->ops = &def_txq_ops;
2778 txq->tx_deferred_start = tx_conf->tx_deferred_start;
2779 #ifdef RTE_LIB_SECURITY
2780 txq->using_ipsec = !!(dev->data->dev_conf.txmode.offloads &
2781 RTE_ETH_TX_OFFLOAD_SECURITY);
2785 * Modification to set VFTDT for virtual function if vf is detected
2787 if (hw->mac.type == ixgbe_mac_82599_vf ||
2788 hw->mac.type == ixgbe_mac_X540_vf ||
2789 hw->mac.type == ixgbe_mac_X550_vf ||
2790 hw->mac.type == ixgbe_mac_X550EM_x_vf ||
2791 hw->mac.type == ixgbe_mac_X550EM_a_vf)
2792 txq->tdt_reg_addr = IXGBE_PCI_REG_ADDR(hw, IXGBE_VFTDT(queue_idx));
2794 txq->tdt_reg_addr = IXGBE_PCI_REG_ADDR(hw, IXGBE_TDT(txq->reg_idx));
2796 txq->tx_ring_phys_addr = tz->iova;
2797 txq->tx_ring = (union ixgbe_adv_tx_desc *) tz->addr;
2799 /* Allocate software ring */
2800 txq->sw_ring = rte_zmalloc_socket("txq->sw_ring",
2801 sizeof(struct ixgbe_tx_entry) * nb_desc,
2802 RTE_CACHE_LINE_SIZE, socket_id);
2803 if (txq->sw_ring == NULL) {
2804 ixgbe_tx_queue_release(txq);
2807 PMD_INIT_LOG(DEBUG, "sw_ring=%p hw_ring=%p dma_addr=0x%"PRIx64,
2808 txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr);
2810 /* set up vector or scalar TX function as appropriate */
2811 ixgbe_set_tx_function(dev, txq);
2813 txq->ops->reset(txq);
2815 dev->data->tx_queues[queue_idx] = txq;
2822 * ixgbe_free_sc_cluster - free the not-yet-completed scattered cluster
2824 * The "next" pointer of the last segment of (not-yet-completed) RSC clusters
2825 * in the sw_rsc_ring is not set to NULL but rather points to the next
2826 * mbuf of this RSC aggregation (that has not been completed yet and still
2827 * resides on the HW ring). So, instead of calling for rte_pktmbuf_free() we
2828 * will just free first "nb_segs" segments of the cluster explicitly by calling
2829 * an rte_pktmbuf_free_seg().
2831 * @m scattered cluster head
2833 static void __rte_cold
2834 ixgbe_free_sc_cluster(struct rte_mbuf *m)
2836 uint16_t i, nb_segs = m->nb_segs;
2837 struct rte_mbuf *next_seg;
2839 for (i = 0; i < nb_segs; i++) {
2841 rte_pktmbuf_free_seg(m);
2846 static void __rte_cold
2847 ixgbe_rx_queue_release_mbufs(struct ixgbe_rx_queue *rxq)
2851 /* SSE Vector driver has a different way of releasing mbufs. */
2852 if (rxq->rx_using_sse) {
2853 ixgbe_rx_queue_release_mbufs_vec(rxq);
2857 if (rxq->sw_ring != NULL) {
2858 for (i = 0; i < rxq->nb_rx_desc; i++) {
2859 if (rxq->sw_ring[i].mbuf != NULL) {
2860 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
2861 rxq->sw_ring[i].mbuf = NULL;
2864 if (rxq->rx_nb_avail) {
2865 for (i = 0; i < rxq->rx_nb_avail; ++i) {
2866 struct rte_mbuf *mb;
2868 mb = rxq->rx_stage[rxq->rx_next_avail + i];
2869 rte_pktmbuf_free_seg(mb);
2871 rxq->rx_nb_avail = 0;
2875 if (rxq->sw_sc_ring)
2876 for (i = 0; i < rxq->nb_rx_desc; i++)
2877 if (rxq->sw_sc_ring[i].fbuf) {
2878 ixgbe_free_sc_cluster(rxq->sw_sc_ring[i].fbuf);
2879 rxq->sw_sc_ring[i].fbuf = NULL;
2883 static void __rte_cold
2884 ixgbe_rx_queue_release(struct ixgbe_rx_queue *rxq)
2887 ixgbe_rx_queue_release_mbufs(rxq);
2888 rte_free(rxq->sw_ring);
2889 rte_free(rxq->sw_sc_ring);
2890 rte_memzone_free(rxq->mz);
2896 ixgbe_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
2898 ixgbe_rx_queue_release(dev->data->rx_queues[qid]);
2902 * Check if Rx Burst Bulk Alloc function can be used.
2904 * 0: the preconditions are satisfied and the bulk allocation function
2906 * -EINVAL: the preconditions are NOT satisfied and the default Rx burst
2907 * function must be used.
2909 static inline int __rte_cold
2910 check_rx_burst_bulk_alloc_preconditions(struct ixgbe_rx_queue *rxq)
2915 * Make sure the following pre-conditions are satisfied:
2916 * rxq->rx_free_thresh >= RTE_PMD_IXGBE_RX_MAX_BURST
2917 * rxq->rx_free_thresh < rxq->nb_rx_desc
2918 * (rxq->nb_rx_desc % rxq->rx_free_thresh) == 0
2919 * Scattered packets are not supported. This should be checked
2920 * outside of this function.
2922 if (!(rxq->rx_free_thresh >= RTE_PMD_IXGBE_RX_MAX_BURST)) {
2923 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
2924 "rxq->rx_free_thresh=%d, "
2925 "RTE_PMD_IXGBE_RX_MAX_BURST=%d",
2926 rxq->rx_free_thresh, RTE_PMD_IXGBE_RX_MAX_BURST);
2928 } else if (!(rxq->rx_free_thresh < rxq->nb_rx_desc)) {
2929 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
2930 "rxq->rx_free_thresh=%d, "
2931 "rxq->nb_rx_desc=%d",
2932 rxq->rx_free_thresh, rxq->nb_rx_desc);
2934 } else if (!((rxq->nb_rx_desc % rxq->rx_free_thresh) == 0)) {
2935 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
2936 "rxq->nb_rx_desc=%d, "
2937 "rxq->rx_free_thresh=%d",
2938 rxq->nb_rx_desc, rxq->rx_free_thresh);
2945 /* Reset dynamic ixgbe_rx_queue fields back to defaults */
2946 static void __rte_cold
2947 ixgbe_reset_rx_queue(struct ixgbe_adapter *adapter, struct ixgbe_rx_queue *rxq)
2949 static const union ixgbe_adv_rx_desc zeroed_desc = {{0}};
2951 uint16_t len = rxq->nb_rx_desc;
2954 * By default, the Rx queue setup function allocates enough memory for
2955 * IXGBE_MAX_RING_DESC. The Rx Burst bulk allocation function requires
2956 * extra memory at the end of the descriptor ring to be zero'd out.
2958 if (adapter->rx_bulk_alloc_allowed)
2959 /* zero out extra memory */
2960 len += RTE_PMD_IXGBE_RX_MAX_BURST;
2963 * Zero out HW ring memory. Zero out extra memory at the end of
2964 * the H/W ring so look-ahead logic in Rx Burst bulk alloc function
2965 * reads extra memory as zeros.
2967 for (i = 0; i < len; i++) {
2968 rxq->rx_ring[i] = zeroed_desc;
2972 * initialize extra software ring entries. Space for these extra
2973 * entries is always allocated
2975 memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf));
2976 for (i = rxq->nb_rx_desc; i < len; ++i) {
2977 rxq->sw_ring[i].mbuf = &rxq->fake_mbuf;
2980 rxq->rx_nb_avail = 0;
2981 rxq->rx_next_avail = 0;
2982 rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1);
2984 rxq->nb_rx_hold = 0;
2986 if (rxq->pkt_first_seg != NULL)
2987 rte_pktmbuf_free(rxq->pkt_first_seg);
2989 rxq->pkt_first_seg = NULL;
2990 rxq->pkt_last_seg = NULL;
2992 #if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64)
2993 rxq->rxrearm_start = 0;
2994 rxq->rxrearm_nb = 0;
2999 ixgbe_is_vf(struct rte_eth_dev *dev)
3001 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3003 switch (hw->mac.type) {
3004 case ixgbe_mac_82599_vf:
3005 case ixgbe_mac_X540_vf:
3006 case ixgbe_mac_X550_vf:
3007 case ixgbe_mac_X550EM_x_vf:
3008 case ixgbe_mac_X550EM_a_vf:
3016 ixgbe_get_rx_queue_offloads(struct rte_eth_dev *dev)
3018 uint64_t offloads = 0;
3019 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3021 if (hw->mac.type != ixgbe_mac_82598EB)
3022 offloads |= RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
3028 ixgbe_get_rx_port_offloads(struct rte_eth_dev *dev)
3031 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3033 offloads = RTE_ETH_RX_OFFLOAD_IPV4_CKSUM |
3034 RTE_ETH_RX_OFFLOAD_UDP_CKSUM |
3035 RTE_ETH_RX_OFFLOAD_TCP_CKSUM |
3036 RTE_ETH_RX_OFFLOAD_KEEP_CRC |
3037 RTE_ETH_RX_OFFLOAD_VLAN_FILTER |
3038 RTE_ETH_RX_OFFLOAD_SCATTER |
3039 RTE_ETH_RX_OFFLOAD_RSS_HASH;
3041 if (hw->mac.type == ixgbe_mac_82598EB)
3042 offloads |= RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
3044 if (ixgbe_is_vf(dev) == 0)
3045 offloads |= RTE_ETH_RX_OFFLOAD_VLAN_EXTEND;
3048 * RSC is only supported by 82599 and x540 PF devices in a non-SR-IOV
3051 if ((hw->mac.type == ixgbe_mac_82599EB ||
3052 hw->mac.type == ixgbe_mac_X540 ||
3053 hw->mac.type == ixgbe_mac_X550) &&
3054 !RTE_ETH_DEV_SRIOV(dev).active)
3055 offloads |= RTE_ETH_RX_OFFLOAD_TCP_LRO;
3057 if (hw->mac.type == ixgbe_mac_82599EB ||
3058 hw->mac.type == ixgbe_mac_X540)
3059 offloads |= RTE_ETH_RX_OFFLOAD_MACSEC_STRIP;
3061 if (hw->mac.type == ixgbe_mac_X550 ||
3062 hw->mac.type == ixgbe_mac_X550EM_x ||
3063 hw->mac.type == ixgbe_mac_X550EM_a)
3064 offloads |= RTE_ETH_RX_OFFLOAD_OUTER_IPV4_CKSUM;
3066 #ifdef RTE_LIB_SECURITY
3067 if (dev->security_ctx)
3068 offloads |= RTE_ETH_RX_OFFLOAD_SECURITY;
3075 ixgbe_dev_rx_queue_setup(struct rte_eth_dev *dev,
3078 unsigned int socket_id,
3079 const struct rte_eth_rxconf *rx_conf,
3080 struct rte_mempool *mp)
3082 const struct rte_memzone *rz;
3083 struct ixgbe_rx_queue *rxq;
3084 struct ixgbe_hw *hw;
3086 struct ixgbe_adapter *adapter = dev->data->dev_private;
3089 PMD_INIT_FUNC_TRACE();
3090 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3092 offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads;
3095 * Validate number of receive descriptors.
3096 * It must not exceed hardware maximum, and must be multiple
3099 if (nb_desc % IXGBE_RXD_ALIGN != 0 ||
3100 (nb_desc > IXGBE_MAX_RING_DESC) ||
3101 (nb_desc < IXGBE_MIN_RING_DESC)) {
3105 /* Free memory prior to re-allocation if needed... */
3106 if (dev->data->rx_queues[queue_idx] != NULL) {
3107 ixgbe_rx_queue_release(dev->data->rx_queues[queue_idx]);
3108 dev->data->rx_queues[queue_idx] = NULL;
3111 /* First allocate the rx queue data structure */
3112 rxq = rte_zmalloc_socket("ethdev RX queue", sizeof(struct ixgbe_rx_queue),
3113 RTE_CACHE_LINE_SIZE, socket_id);
3117 rxq->nb_rx_desc = nb_desc;
3118 rxq->rx_free_thresh = rx_conf->rx_free_thresh;
3119 rxq->queue_id = queue_idx;
3120 rxq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ?
3121 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx);
3122 rxq->port_id = dev->data->port_id;
3123 if (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC)
3124 rxq->crc_len = RTE_ETHER_CRC_LEN;
3127 rxq->drop_en = rx_conf->rx_drop_en;
3128 rxq->rx_deferred_start = rx_conf->rx_deferred_start;
3129 rxq->offloads = offloads;
3132 * The packet type in RX descriptor is different for different NICs.
3133 * Some bits are used for x550 but reserved for other NICS.
3134 * So set different masks for different NICs.
3136 if (hw->mac.type == ixgbe_mac_X550 ||
3137 hw->mac.type == ixgbe_mac_X550EM_x ||
3138 hw->mac.type == ixgbe_mac_X550EM_a ||
3139 hw->mac.type == ixgbe_mac_X550_vf ||
3140 hw->mac.type == ixgbe_mac_X550EM_x_vf ||
3141 hw->mac.type == ixgbe_mac_X550EM_a_vf)
3142 rxq->pkt_type_mask = IXGBE_PACKET_TYPE_MASK_X550;
3144 rxq->pkt_type_mask = IXGBE_PACKET_TYPE_MASK_82599;
3147 * 82599 errata, UDP frames with a 0 checksum can be marked as checksum
3150 if (hw->mac.type == ixgbe_mac_82599EB)
3151 rxq->rx_udp_csum_zero_err = 1;
3154 * Allocate RX ring hardware descriptors. A memzone large enough to
3155 * handle the maximum ring size is allocated in order to allow for
3156 * resizing in later calls to the queue setup function.
3158 rz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx,
3159 RX_RING_SZ, IXGBE_ALIGN, socket_id);
3161 ixgbe_rx_queue_release(rxq);
3167 * Zero init all the descriptors in the ring.
3169 memset(rz->addr, 0, RX_RING_SZ);
3172 * Modified to setup VFRDT for Virtual Function
3174 if (hw->mac.type == ixgbe_mac_82599_vf ||
3175 hw->mac.type == ixgbe_mac_X540_vf ||
3176 hw->mac.type == ixgbe_mac_X550_vf ||
3177 hw->mac.type == ixgbe_mac_X550EM_x_vf ||
3178 hw->mac.type == ixgbe_mac_X550EM_a_vf) {
3180 IXGBE_PCI_REG_ADDR(hw, IXGBE_VFRDT(queue_idx));
3182 IXGBE_PCI_REG_ADDR(hw, IXGBE_VFRDH(queue_idx));
3185 IXGBE_PCI_REG_ADDR(hw, IXGBE_RDT(rxq->reg_idx));
3187 IXGBE_PCI_REG_ADDR(hw, IXGBE_RDH(rxq->reg_idx));
3190 rxq->rx_ring_phys_addr = rz->iova;
3191 rxq->rx_ring = (union ixgbe_adv_rx_desc *) rz->addr;
3194 * Certain constraints must be met in order to use the bulk buffer
3195 * allocation Rx burst function. If any of Rx queues doesn't meet them
3196 * the feature should be disabled for the whole port.
3198 if (check_rx_burst_bulk_alloc_preconditions(rxq)) {
3199 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Rx Bulk Alloc "
3200 "preconditions - canceling the feature for "
3201 "the whole port[%d]",
3202 rxq->queue_id, rxq->port_id);
3203 adapter->rx_bulk_alloc_allowed = false;
3207 * Allocate software ring. Allow for space at the end of the
3208 * S/W ring to make sure look-ahead logic in bulk alloc Rx burst
3209 * function does not access an invalid memory region.
3212 if (adapter->rx_bulk_alloc_allowed)
3213 len += RTE_PMD_IXGBE_RX_MAX_BURST;
3215 rxq->sw_ring = rte_zmalloc_socket("rxq->sw_ring",
3216 sizeof(struct ixgbe_rx_entry) * len,
3217 RTE_CACHE_LINE_SIZE, socket_id);
3218 if (!rxq->sw_ring) {
3219 ixgbe_rx_queue_release(rxq);
3224 * Always allocate even if it's not going to be needed in order to
3225 * simplify the code.
3227 * This ring is used in LRO and Scattered Rx cases and Scattered Rx may
3228 * be requested in ixgbe_dev_rx_init(), which is called later from
3232 rte_zmalloc_socket("rxq->sw_sc_ring",
3233 sizeof(struct ixgbe_scattered_rx_entry) * len,
3234 RTE_CACHE_LINE_SIZE, socket_id);
3235 if (!rxq->sw_sc_ring) {
3236 ixgbe_rx_queue_release(rxq);
3240 PMD_INIT_LOG(DEBUG, "sw_ring=%p sw_sc_ring=%p hw_ring=%p "
3241 "dma_addr=0x%"PRIx64,
3242 rxq->sw_ring, rxq->sw_sc_ring, rxq->rx_ring,
3243 rxq->rx_ring_phys_addr);
3245 if (!rte_is_power_of_2(nb_desc)) {
3246 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
3247 "preconditions - canceling the feature for "
3248 "the whole port[%d]",
3249 rxq->queue_id, rxq->port_id);
3250 adapter->rx_vec_allowed = false;
3252 ixgbe_rxq_vec_setup(rxq);
3254 dev->data->rx_queues[queue_idx] = rxq;
3256 ixgbe_reset_rx_queue(adapter, rxq);
3262 ixgbe_dev_rx_queue_count(void *rx_queue)
3264 #define IXGBE_RXQ_SCAN_INTERVAL 4
3265 volatile union ixgbe_adv_rx_desc *rxdp;
3266 struct ixgbe_rx_queue *rxq;
3270 rxdp = &(rxq->rx_ring[rxq->rx_tail]);
3272 while ((desc < rxq->nb_rx_desc) &&
3273 (rxdp->wb.upper.status_error &
3274 rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD))) {
3275 desc += IXGBE_RXQ_SCAN_INTERVAL;
3276 rxdp += IXGBE_RXQ_SCAN_INTERVAL;
3277 if (rxq->rx_tail + desc >= rxq->nb_rx_desc)
3278 rxdp = &(rxq->rx_ring[rxq->rx_tail +
3279 desc - rxq->nb_rx_desc]);
3286 ixgbe_dev_rx_descriptor_status(void *rx_queue, uint16_t offset)
3288 struct ixgbe_rx_queue *rxq = rx_queue;
3289 volatile uint32_t *status;
3290 uint32_t nb_hold, desc;
3292 if (unlikely(offset >= rxq->nb_rx_desc))
3295 #if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64)
3296 if (rxq->rx_using_sse)
3297 nb_hold = rxq->rxrearm_nb;
3300 nb_hold = rxq->nb_rx_hold;
3301 if (offset >= rxq->nb_rx_desc - nb_hold)
3302 return RTE_ETH_RX_DESC_UNAVAIL;
3304 desc = rxq->rx_tail + offset;
3305 if (desc >= rxq->nb_rx_desc)
3306 desc -= rxq->nb_rx_desc;
3308 status = &rxq->rx_ring[desc].wb.upper.status_error;
3309 if (*status & rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD))
3310 return RTE_ETH_RX_DESC_DONE;
3312 return RTE_ETH_RX_DESC_AVAIL;
3316 ixgbe_dev_tx_descriptor_status(void *tx_queue, uint16_t offset)
3318 struct ixgbe_tx_queue *txq = tx_queue;
3319 volatile uint32_t *status;
3322 if (unlikely(offset >= txq->nb_tx_desc))
3325 desc = txq->tx_tail + offset;
3326 /* go to next desc that has the RS bit */
3327 desc = ((desc + txq->tx_rs_thresh - 1) / txq->tx_rs_thresh) *
3329 if (desc >= txq->nb_tx_desc) {
3330 desc -= txq->nb_tx_desc;
3331 if (desc >= txq->nb_tx_desc)
3332 desc -= txq->nb_tx_desc;
3335 status = &txq->tx_ring[desc].wb.status;
3336 if (*status & rte_cpu_to_le_32(IXGBE_ADVTXD_STAT_DD))
3337 return RTE_ETH_TX_DESC_DONE;
3339 return RTE_ETH_TX_DESC_FULL;
3343 * Set up link loopback for X540/X550 mode Tx->Rx.
3345 static inline void __rte_cold
3346 ixgbe_setup_loopback_link_x540_x550(struct ixgbe_hw *hw, bool enable)
3349 PMD_INIT_FUNC_TRACE();
3351 u16 autoneg_reg = IXGBE_MII_AUTONEG_REG;
3353 hw->phy.ops.read_reg(hw, IXGBE_MDIO_AUTO_NEG_CONTROL,
3354 IXGBE_MDIO_AUTO_NEG_DEV_TYPE, &autoneg_reg);
3355 macc = IXGBE_READ_REG(hw, IXGBE_MACC);
3358 /* datasheet 15.2.1: disable AUTONEG (PHY Bit 7.0.C) */
3359 autoneg_reg |= IXGBE_MII_AUTONEG_ENABLE;
3360 /* datasheet 15.2.1: MACC.FLU = 1 (force link up) */
3361 macc |= IXGBE_MACC_FLU;
3363 autoneg_reg &= ~IXGBE_MII_AUTONEG_ENABLE;
3364 macc &= ~IXGBE_MACC_FLU;
3367 hw->phy.ops.write_reg(hw, IXGBE_MDIO_AUTO_NEG_CONTROL,
3368 IXGBE_MDIO_AUTO_NEG_DEV_TYPE, autoneg_reg);
3370 IXGBE_WRITE_REG(hw, IXGBE_MACC, macc);
3374 ixgbe_dev_clear_queues(struct rte_eth_dev *dev)
3377 struct ixgbe_adapter *adapter = dev->data->dev_private;
3378 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3380 PMD_INIT_FUNC_TRACE();
3382 for (i = 0; i < dev->data->nb_tx_queues; i++) {
3383 struct ixgbe_tx_queue *txq = dev->data->tx_queues[i];
3386 txq->ops->release_mbufs(txq);
3387 txq->ops->reset(txq);
3391 for (i = 0; i < dev->data->nb_rx_queues; i++) {
3392 struct ixgbe_rx_queue *rxq = dev->data->rx_queues[i];
3395 ixgbe_rx_queue_release_mbufs(rxq);
3396 ixgbe_reset_rx_queue(adapter, rxq);
3399 /* If loopback mode was enabled, reconfigure the link accordingly */
3400 if (dev->data->dev_conf.lpbk_mode != 0) {
3401 if (hw->mac.type == ixgbe_mac_X540 ||
3402 hw->mac.type == ixgbe_mac_X550 ||
3403 hw->mac.type == ixgbe_mac_X550EM_x ||
3404 hw->mac.type == ixgbe_mac_X550EM_a)
3405 ixgbe_setup_loopback_link_x540_x550(hw, false);
3410 ixgbe_dev_free_queues(struct rte_eth_dev *dev)
3414 PMD_INIT_FUNC_TRACE();
3416 for (i = 0; i < dev->data->nb_rx_queues; i++) {
3417 ixgbe_dev_rx_queue_release(dev, i);
3418 dev->data->rx_queues[i] = NULL;
3420 dev->data->nb_rx_queues = 0;
3422 for (i = 0; i < dev->data->nb_tx_queues; i++) {
3423 ixgbe_dev_tx_queue_release(dev, i);
3424 dev->data->tx_queues[i] = NULL;
3426 dev->data->nb_tx_queues = 0;
3429 /*********************************************************************
3431 * Device RX/TX init functions
3433 **********************************************************************/
3436 * Receive Side Scaling (RSS)
3437 * See section 7.1.2.8 in the following document:
3438 * "Intel 82599 10 GbE Controller Datasheet" - Revision 2.1 October 2009
3441 * The source and destination IP addresses of the IP header and the source
3442 * and destination ports of TCP/UDP headers, if any, of received packets are
3443 * hashed against a configurable random key to compute a 32-bit RSS hash result.
3444 * The seven (7) LSBs of the 32-bit hash result are used as an index into a
3445 * 128-entry redirection table (RETA). Each entry of the RETA provides a 3-bit
3446 * RSS output index which is used as the RX queue index where to store the
3448 * The following output is supplied in the RX write-back descriptor:
3449 * - 32-bit result of the Microsoft RSS hash function,
3450 * - 4-bit RSS type field.
3454 * RSS random key supplied in section 7.1.2.8.3 of the Intel 82599 datasheet.
3455 * Used as the default key.
3457 static uint8_t rss_intel_key[40] = {
3458 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
3459 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
3460 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
3461 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
3462 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
3466 ixgbe_rss_disable(struct rte_eth_dev *dev)
3468 struct ixgbe_hw *hw;
3472 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3473 mrqc_reg = ixgbe_mrqc_reg_get(hw->mac.type);
3474 mrqc = IXGBE_READ_REG(hw, mrqc_reg);
3475 mrqc &= ~IXGBE_MRQC_RSSEN;
3476 IXGBE_WRITE_REG(hw, mrqc_reg, mrqc);
3480 ixgbe_hw_rss_hash_set(struct ixgbe_hw *hw, struct rte_eth_rss_conf *rss_conf)
3490 mrqc_reg = ixgbe_mrqc_reg_get(hw->mac.type);
3491 rssrk_reg = ixgbe_rssrk_reg_get(hw->mac.type, 0);
3493 hash_key = rss_conf->rss_key;
3494 if (hash_key != NULL) {
3495 /* Fill in RSS hash key */
3496 for (i = 0; i < 10; i++) {
3497 rss_key = hash_key[(i * 4)];
3498 rss_key |= hash_key[(i * 4) + 1] << 8;
3499 rss_key |= hash_key[(i * 4) + 2] << 16;
3500 rss_key |= hash_key[(i * 4) + 3] << 24;
3501 IXGBE_WRITE_REG_ARRAY(hw, rssrk_reg, i, rss_key);
3505 /* Set configured hashing protocols in MRQC register */
3506 rss_hf = rss_conf->rss_hf;
3507 mrqc = IXGBE_MRQC_RSSEN; /* Enable RSS */
3508 if (rss_hf & RTE_ETH_RSS_IPV4)
3509 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4;
3510 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_TCP)
3511 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP;
3512 if (rss_hf & RTE_ETH_RSS_IPV6)
3513 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6;
3514 if (rss_hf & RTE_ETH_RSS_IPV6_EX)
3515 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX;
3516 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV6_TCP)
3517 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP;
3518 if (rss_hf & RTE_ETH_RSS_IPV6_TCP_EX)
3519 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP;
3520 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_UDP)
3521 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP;
3522 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV6_UDP)
3523 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP;
3524 if (rss_hf & RTE_ETH_RSS_IPV6_UDP_EX)
3525 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
3526 IXGBE_WRITE_REG(hw, mrqc_reg, mrqc);
3530 ixgbe_dev_rss_hash_update(struct rte_eth_dev *dev,
3531 struct rte_eth_rss_conf *rss_conf)
3533 struct ixgbe_hw *hw;
3538 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3540 if (!ixgbe_rss_update_sp(hw->mac.type)) {
3541 PMD_DRV_LOG(ERR, "RSS hash update is not supported on this "
3545 mrqc_reg = ixgbe_mrqc_reg_get(hw->mac.type);
3548 * Excerpt from section 7.1.2.8 Receive-Side Scaling (RSS):
3549 * "RSS enabling cannot be done dynamically while it must be
3550 * preceded by a software reset"
3551 * Before changing anything, first check that the update RSS operation
3552 * does not attempt to disable RSS, if RSS was enabled at
3553 * initialization time, or does not attempt to enable RSS, if RSS was
3554 * disabled at initialization time.
3556 rss_hf = rss_conf->rss_hf & IXGBE_RSS_OFFLOAD_ALL;
3557 mrqc = IXGBE_READ_REG(hw, mrqc_reg);
3558 if (!(mrqc & IXGBE_MRQC_RSSEN)) { /* RSS disabled */
3559 if (rss_hf != 0) /* Enable RSS */
3561 return 0; /* Nothing to do */
3564 if (rss_hf == 0) /* Disable RSS */
3566 ixgbe_hw_rss_hash_set(hw, rss_conf);
3571 ixgbe_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
3572 struct rte_eth_rss_conf *rss_conf)
3574 struct ixgbe_hw *hw;
3583 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3584 mrqc_reg = ixgbe_mrqc_reg_get(hw->mac.type);
3585 rssrk_reg = ixgbe_rssrk_reg_get(hw->mac.type, 0);
3586 hash_key = rss_conf->rss_key;
3587 if (hash_key != NULL) {
3588 /* Return RSS hash key */
3589 for (i = 0; i < 10; i++) {
3590 rss_key = IXGBE_READ_REG_ARRAY(hw, rssrk_reg, i);
3591 hash_key[(i * 4)] = rss_key & 0x000000FF;
3592 hash_key[(i * 4) + 1] = (rss_key >> 8) & 0x000000FF;
3593 hash_key[(i * 4) + 2] = (rss_key >> 16) & 0x000000FF;
3594 hash_key[(i * 4) + 3] = (rss_key >> 24) & 0x000000FF;
3598 /* Get RSS functions configured in MRQC register */
3599 mrqc = IXGBE_READ_REG(hw, mrqc_reg);
3600 if ((mrqc & IXGBE_MRQC_RSSEN) == 0) { /* RSS is disabled */
3601 rss_conf->rss_hf = 0;
3605 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV4)
3606 rss_hf |= RTE_ETH_RSS_IPV4;
3607 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV4_TCP)
3608 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_TCP;
3609 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6)
3610 rss_hf |= RTE_ETH_RSS_IPV6;
3611 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_EX)
3612 rss_hf |= RTE_ETH_RSS_IPV6_EX;
3613 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_TCP)
3614 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_TCP;
3615 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP)
3616 rss_hf |= RTE_ETH_RSS_IPV6_TCP_EX;
3617 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV4_UDP)
3618 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_UDP;
3619 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_UDP)
3620 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_UDP;
3621 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP)
3622 rss_hf |= RTE_ETH_RSS_IPV6_UDP_EX;
3623 rss_conf->rss_hf = rss_hf;
3628 ixgbe_rss_configure(struct rte_eth_dev *dev)
3630 struct rte_eth_rss_conf rss_conf;
3631 struct ixgbe_adapter *adapter;
3632 struct ixgbe_hw *hw;
3636 uint16_t sp_reta_size;
3639 PMD_INIT_FUNC_TRACE();
3640 adapter = dev->data->dev_private;
3641 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3643 sp_reta_size = ixgbe_reta_size_get(hw->mac.type);
3646 * Fill in redirection table
3647 * The byte-swap is needed because NIC registers are in
3648 * little-endian order.
3650 if (adapter->rss_reta_updated == 0) {
3652 for (i = 0, j = 0; i < sp_reta_size; i++, j++) {
3653 reta_reg = ixgbe_reta_reg_get(hw->mac.type, i);
3655 if (j == dev->data->nb_rx_queues)
3657 reta = (reta << 8) | j;
3659 IXGBE_WRITE_REG(hw, reta_reg,
3665 * Configure the RSS key and the RSS protocols used to compute
3666 * the RSS hash of input packets.
3668 rss_conf = dev->data->dev_conf.rx_adv_conf.rss_conf;
3669 if ((rss_conf.rss_hf & IXGBE_RSS_OFFLOAD_ALL) == 0) {
3670 ixgbe_rss_disable(dev);
3673 if (rss_conf.rss_key == NULL)
3674 rss_conf.rss_key = rss_intel_key; /* Default hash key */
3675 ixgbe_hw_rss_hash_set(hw, &rss_conf);
3678 #define NUM_VFTA_REGISTERS 128
3679 #define NIC_RX_BUFFER_SIZE 0x200
3680 #define X550_RX_BUFFER_SIZE 0x180
3683 ixgbe_vmdq_dcb_configure(struct rte_eth_dev *dev)
3685 struct rte_eth_vmdq_dcb_conf *cfg;
3686 struct ixgbe_hw *hw;
3687 enum rte_eth_nb_pools num_pools;
3688 uint32_t mrqc, vt_ctl, queue_mapping, vlanctrl;
3690 uint8_t nb_tcs; /* number of traffic classes */
3693 PMD_INIT_FUNC_TRACE();
3694 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3695 cfg = &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf;
3696 num_pools = cfg->nb_queue_pools;
3697 /* Check we have a valid number of pools */
3698 if (num_pools != RTE_ETH_16_POOLS && num_pools != RTE_ETH_32_POOLS) {
3699 ixgbe_rss_disable(dev);
3702 /* 16 pools -> 8 traffic classes, 32 pools -> 4 traffic classes */
3703 nb_tcs = (uint8_t)(RTE_ETH_VMDQ_DCB_NUM_QUEUES / (int)num_pools);
3707 * split rx buffer up into sections, each for 1 traffic class
3709 switch (hw->mac.type) {
3710 case ixgbe_mac_X550:
3711 case ixgbe_mac_X550EM_x:
3712 case ixgbe_mac_X550EM_a:
3713 pbsize = (uint16_t)(X550_RX_BUFFER_SIZE / nb_tcs);
3716 pbsize = (uint16_t)(NIC_RX_BUFFER_SIZE / nb_tcs);
3719 for (i = 0; i < nb_tcs; i++) {
3720 uint32_t rxpbsize = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(i));
3722 rxpbsize &= (~(0x3FF << IXGBE_RXPBSIZE_SHIFT));
3723 /* clear 10 bits. */
3724 rxpbsize |= (pbsize << IXGBE_RXPBSIZE_SHIFT); /* set value */
3725 IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), rxpbsize);
3727 /* zero alloc all unused TCs */
3728 for (i = nb_tcs; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
3729 uint32_t rxpbsize = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(i));
3731 rxpbsize &= (~(0x3FF << IXGBE_RXPBSIZE_SHIFT));
3732 /* clear 10 bits. */
3733 IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), rxpbsize);
3736 /* MRQC: enable vmdq and dcb */
3737 mrqc = (num_pools == RTE_ETH_16_POOLS) ?
3738 IXGBE_MRQC_VMDQRT8TCEN : IXGBE_MRQC_VMDQRT4TCEN;
3739 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
3741 /* PFVTCTL: turn on virtualisation and set the default pool */
3742 vt_ctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
3743 if (cfg->enable_default_pool) {
3744 vt_ctl |= (cfg->default_pool << IXGBE_VT_CTL_POOL_SHIFT);
3746 vt_ctl |= IXGBE_VT_CTL_DIS_DEFPL;
3749 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vt_ctl);
3751 /* RTRUP2TC: mapping user priorities to traffic classes (TCs) */
3753 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++)
3755 * mapping is done with 3 bits per priority,
3756 * so shift by i*3 each time
3758 queue_mapping |= ((cfg->dcb_tc[i] & 0x07) << (i * 3));
3760 IXGBE_WRITE_REG(hw, IXGBE_RTRUP2TC, queue_mapping);
3762 /* RTRPCS: DCB related */
3763 IXGBE_WRITE_REG(hw, IXGBE_RTRPCS, IXGBE_RMCS_RRM);
3765 /* VLNCTRL: enable vlan filtering and allow all vlan tags through */
3766 vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
3767 vlanctrl |= IXGBE_VLNCTRL_VFE; /* enable vlan filters */
3768 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl);
3770 /* VFTA - enable all vlan filters */
3771 for (i = 0; i < NUM_VFTA_REGISTERS; i++) {
3772 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), 0xFFFFFFFF);
3775 /* VFRE: pool enabling for receive - 16 or 32 */
3776 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0),
3777 num_pools == RTE_ETH_16_POOLS ? 0xFFFF : 0xFFFFFFFF);
3780 * MPSAR - allow pools to read specific mac addresses
3781 * In this case, all pools should be able to read from mac addr 0
3783 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(0), 0xFFFFFFFF);
3784 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(0), 0xFFFFFFFF);
3786 /* PFVLVF, PFVLVFB: set up filters for vlan tags as configured */
3787 for (i = 0; i < cfg->nb_pool_maps; i++) {
3788 /* set vlan id in VF register and set the valid bit */
3789 IXGBE_WRITE_REG(hw, IXGBE_VLVF(i), (IXGBE_VLVF_VIEN |
3790 (cfg->pool_map[i].vlan_id & 0xFFF)));
3792 * Put the allowed pools in VFB reg. As we only have 16 or 32
3793 * pools, we only need to use the first half of the register
3796 IXGBE_WRITE_REG(hw, IXGBE_VLVFB(i*2), cfg->pool_map[i].pools);
3801 * ixgbe_dcb_config_tx_hw_config - Configure general DCB TX parameters
3802 * @dev: pointer to eth_dev structure
3803 * @dcb_config: pointer to ixgbe_dcb_config structure
3806 ixgbe_dcb_tx_hw_config(struct rte_eth_dev *dev,
3807 struct ixgbe_dcb_config *dcb_config)
3810 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3812 PMD_INIT_FUNC_TRACE();
3813 if (hw->mac.type != ixgbe_mac_82598EB) {
3814 /* Disable the Tx desc arbiter so that MTQC can be changed */
3815 reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3816 reg |= IXGBE_RTTDCS_ARBDIS;
3817 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg);
3819 /* Enable DCB for Tx with 8 TCs */
3820 if (dcb_config->num_tcs.pg_tcs == 8) {
3821 reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;
3823 reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
3825 if (dcb_config->vt_mode)
3826 reg |= IXGBE_MTQC_VT_ENA;
3827 IXGBE_WRITE_REG(hw, IXGBE_MTQC, reg);
3829 /* Enable the Tx desc arbiter */
3830 reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3831 reg &= ~IXGBE_RTTDCS_ARBDIS;
3832 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg);
3834 /* Enable Security TX Buffer IFG for DCB */
3835 reg = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG);
3836 reg |= IXGBE_SECTX_DCB;
3837 IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, reg);
3842 * ixgbe_vmdq_dcb_hw_tx_config - Configure general VMDQ+DCB TX parameters
3843 * @dev: pointer to rte_eth_dev structure
3844 * @dcb_config: pointer to ixgbe_dcb_config structure
3847 ixgbe_vmdq_dcb_hw_tx_config(struct rte_eth_dev *dev,
3848 struct ixgbe_dcb_config *dcb_config)
3850 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3851 &dev->data->dev_conf.tx_adv_conf.vmdq_dcb_tx_conf;
3852 struct ixgbe_hw *hw =
3853 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3855 PMD_INIT_FUNC_TRACE();
3856 if (hw->mac.type != ixgbe_mac_82598EB)
3857 /*PF VF Transmit Enable*/
3858 IXGBE_WRITE_REG(hw, IXGBE_VFTE(0),
3859 vmdq_tx_conf->nb_queue_pools == RTE_ETH_16_POOLS ? 0xFFFF : 0xFFFFFFFF);
3861 /*Configure general DCB TX parameters*/
3862 ixgbe_dcb_tx_hw_config(dev, dcb_config);
3866 ixgbe_vmdq_dcb_rx_config(struct rte_eth_dev *dev,
3867 struct ixgbe_dcb_config *dcb_config)
3869 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3870 &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf;
3871 struct ixgbe_dcb_tc_config *tc;
3874 /* convert rte_eth_conf.rx_adv_conf to struct ixgbe_dcb_config */
3875 if (vmdq_rx_conf->nb_queue_pools == RTE_ETH_16_POOLS) {
3876 dcb_config->num_tcs.pg_tcs = RTE_ETH_8_TCS;
3877 dcb_config->num_tcs.pfc_tcs = RTE_ETH_8_TCS;
3879 dcb_config->num_tcs.pg_tcs = RTE_ETH_4_TCS;
3880 dcb_config->num_tcs.pfc_tcs = RTE_ETH_4_TCS;
3883 /* Initialize User Priority to Traffic Class mapping */
3884 for (j = 0; j < IXGBE_DCB_MAX_TRAFFIC_CLASS; j++) {
3885 tc = &dcb_config->tc_config[j];
3886 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0;
3889 /* User Priority to Traffic Class mapping */
3890 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
3891 j = vmdq_rx_conf->dcb_tc[i];
3892 tc = &dcb_config->tc_config[j];
3893 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap |=
3899 ixgbe_dcb_vt_tx_config(struct rte_eth_dev *dev,
3900 struct ixgbe_dcb_config *dcb_config)
3902 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3903 &dev->data->dev_conf.tx_adv_conf.vmdq_dcb_tx_conf;
3904 struct ixgbe_dcb_tc_config *tc;
3907 /* convert rte_eth_conf.rx_adv_conf to struct ixgbe_dcb_config */
3908 if (vmdq_tx_conf->nb_queue_pools == RTE_ETH_16_POOLS) {
3909 dcb_config->num_tcs.pg_tcs = RTE_ETH_8_TCS;
3910 dcb_config->num_tcs.pfc_tcs = RTE_ETH_8_TCS;
3912 dcb_config->num_tcs.pg_tcs = RTE_ETH_4_TCS;
3913 dcb_config->num_tcs.pfc_tcs = RTE_ETH_4_TCS;
3916 /* Initialize User Priority to Traffic Class mapping */
3917 for (j = 0; j < IXGBE_DCB_MAX_TRAFFIC_CLASS; j++) {
3918 tc = &dcb_config->tc_config[j];
3919 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0;
3922 /* User Priority to Traffic Class mapping */
3923 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
3924 j = vmdq_tx_conf->dcb_tc[i];
3925 tc = &dcb_config->tc_config[j];
3926 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap |=
3932 ixgbe_dcb_rx_config(struct rte_eth_dev *dev,
3933 struct ixgbe_dcb_config *dcb_config)
3935 struct rte_eth_dcb_rx_conf *rx_conf =
3936 &dev->data->dev_conf.rx_adv_conf.dcb_rx_conf;
3937 struct ixgbe_dcb_tc_config *tc;
3940 dcb_config->num_tcs.pg_tcs = (uint8_t)rx_conf->nb_tcs;
3941 dcb_config->num_tcs.pfc_tcs = (uint8_t)rx_conf->nb_tcs;
3943 /* Initialize User Priority to Traffic Class mapping */
3944 for (j = 0; j < IXGBE_DCB_MAX_TRAFFIC_CLASS; j++) {
3945 tc = &dcb_config->tc_config[j];
3946 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0;
3949 /* User Priority to Traffic Class mapping */
3950 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
3951 j = rx_conf->dcb_tc[i];
3952 tc = &dcb_config->tc_config[j];
3953 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap |=
3959 ixgbe_dcb_tx_config(struct rte_eth_dev *dev,
3960 struct ixgbe_dcb_config *dcb_config)
3962 struct rte_eth_dcb_tx_conf *tx_conf =
3963 &dev->data->dev_conf.tx_adv_conf.dcb_tx_conf;
3964 struct ixgbe_dcb_tc_config *tc;
3967 dcb_config->num_tcs.pg_tcs = (uint8_t)tx_conf->nb_tcs;
3968 dcb_config->num_tcs.pfc_tcs = (uint8_t)tx_conf->nb_tcs;
3970 /* Initialize User Priority to Traffic Class mapping */
3971 for (j = 0; j < IXGBE_DCB_MAX_TRAFFIC_CLASS; j++) {
3972 tc = &dcb_config->tc_config[j];
3973 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0;
3976 /* User Priority to Traffic Class mapping */
3977 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
3978 j = tx_conf->dcb_tc[i];
3979 tc = &dcb_config->tc_config[j];
3980 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap |=
3986 * ixgbe_dcb_rx_hw_config - Configure general DCB RX HW parameters
3987 * @dev: pointer to eth_dev structure
3988 * @dcb_config: pointer to ixgbe_dcb_config structure
3991 ixgbe_dcb_rx_hw_config(struct rte_eth_dev *dev,
3992 struct ixgbe_dcb_config *dcb_config)
3998 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4000 PMD_INIT_FUNC_TRACE();
4002 * Disable the arbiter before changing parameters
4003 * (always enable recycle mode; WSP)
4005 reg = IXGBE_RTRPCS_RRM | IXGBE_RTRPCS_RAC | IXGBE_RTRPCS_ARBDIS;
4006 IXGBE_WRITE_REG(hw, IXGBE_RTRPCS, reg);
4008 if (hw->mac.type != ixgbe_mac_82598EB) {
4009 reg = IXGBE_READ_REG(hw, IXGBE_MRQC);
4010 if (dcb_config->num_tcs.pg_tcs == 4) {
4011 if (dcb_config->vt_mode)
4012 reg = (reg & ~IXGBE_MRQC_MRQE_MASK) |
4013 IXGBE_MRQC_VMDQRT4TCEN;
4015 /* no matter the mode is DCB or DCB_RSS, just
4016 * set the MRQE to RSSXTCEN. RSS is controlled
4019 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, 0);
4020 reg = (reg & ~IXGBE_MRQC_MRQE_MASK) |
4021 IXGBE_MRQC_RTRSS4TCEN;
4024 if (dcb_config->num_tcs.pg_tcs == 8) {
4025 if (dcb_config->vt_mode)
4026 reg = (reg & ~IXGBE_MRQC_MRQE_MASK) |
4027 IXGBE_MRQC_VMDQRT8TCEN;
4029 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, 0);
4030 reg = (reg & ~IXGBE_MRQC_MRQE_MASK) |
4031 IXGBE_MRQC_RTRSS8TCEN;
4035 IXGBE_WRITE_REG(hw, IXGBE_MRQC, reg);
4037 if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
4038 /* Disable drop for all queues in VMDQ mode*/
4039 for (q = 0; q < IXGBE_MAX_RX_QUEUE_NUM; q++)
4040 IXGBE_WRITE_REG(hw, IXGBE_QDE,
4042 (q << IXGBE_QDE_IDX_SHIFT)));
4044 /* Enable drop for all queues in SRIOV mode */
4045 for (q = 0; q < IXGBE_MAX_RX_QUEUE_NUM; q++)
4046 IXGBE_WRITE_REG(hw, IXGBE_QDE,
4048 (q << IXGBE_QDE_IDX_SHIFT) |
4053 /* VLNCTRL: enable vlan filtering and allow all vlan tags through */
4054 vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4055 vlanctrl |= IXGBE_VLNCTRL_VFE; /* enable vlan filters */
4056 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl);
4058 /* VFTA - enable all vlan filters */
4059 for (i = 0; i < NUM_VFTA_REGISTERS; i++) {
4060 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), 0xFFFFFFFF);
4064 * Configure Rx packet plane (recycle mode; WSP) and
4067 reg = IXGBE_RTRPCS_RRM | IXGBE_RTRPCS_RAC;
4068 IXGBE_WRITE_REG(hw, IXGBE_RTRPCS, reg);
4072 ixgbe_dcb_hw_arbite_rx_config(struct ixgbe_hw *hw, uint16_t *refill,
4073 uint16_t *max, uint8_t *bwg_id, uint8_t *tsa, uint8_t *map)
4075 switch (hw->mac.type) {
4076 case ixgbe_mac_82598EB:
4077 ixgbe_dcb_config_rx_arbiter_82598(hw, refill, max, tsa);
4079 case ixgbe_mac_82599EB:
4080 case ixgbe_mac_X540:
4081 case ixgbe_mac_X550:
4082 case ixgbe_mac_X550EM_x:
4083 case ixgbe_mac_X550EM_a:
4084 ixgbe_dcb_config_rx_arbiter_82599(hw, refill, max, bwg_id,
4093 ixgbe_dcb_hw_arbite_tx_config(struct ixgbe_hw *hw, uint16_t *refill, uint16_t *max,
4094 uint8_t *bwg_id, uint8_t *tsa, uint8_t *map)
4096 switch (hw->mac.type) {
4097 case ixgbe_mac_82598EB:
4098 ixgbe_dcb_config_tx_desc_arbiter_82598(hw, refill, max, bwg_id, tsa);
4099 ixgbe_dcb_config_tx_data_arbiter_82598(hw, refill, max, bwg_id, tsa);
4101 case ixgbe_mac_82599EB:
4102 case ixgbe_mac_X540:
4103 case ixgbe_mac_X550:
4104 case ixgbe_mac_X550EM_x:
4105 case ixgbe_mac_X550EM_a:
4106 ixgbe_dcb_config_tx_desc_arbiter_82599(hw, refill, max, bwg_id, tsa);
4107 ixgbe_dcb_config_tx_data_arbiter_82599(hw, refill, max, bwg_id, tsa, map);
4114 #define DCB_RX_CONFIG 1
4115 #define DCB_TX_CONFIG 1
4116 #define DCB_TX_PB 1024
4118 * ixgbe_dcb_hw_configure - Enable DCB and configure
4119 * general DCB in VT mode and non-VT mode parameters
4120 * @dev: pointer to rte_eth_dev structure
4121 * @dcb_config: pointer to ixgbe_dcb_config structure
4124 ixgbe_dcb_hw_configure(struct rte_eth_dev *dev,
4125 struct ixgbe_dcb_config *dcb_config)
4128 uint8_t i, pfc_en, nb_tcs;
4129 uint16_t pbsize, rx_buffer_size;
4130 uint8_t config_dcb_rx = 0;
4131 uint8_t config_dcb_tx = 0;
4132 uint8_t tsa[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0};
4133 uint8_t bwgid[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0};
4134 uint16_t refill[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0};
4135 uint16_t max[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0};
4136 uint8_t map[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0};
4137 struct ixgbe_dcb_tc_config *tc;
4138 uint32_t max_frame = dev->data->mtu + RTE_ETHER_HDR_LEN +
4140 struct ixgbe_hw *hw =
4141 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4142 struct ixgbe_bw_conf *bw_conf =
4143 IXGBE_DEV_PRIVATE_TO_BW_CONF(dev->data->dev_private);
4145 switch (dev->data->dev_conf.rxmode.mq_mode) {
4146 case RTE_ETH_MQ_RX_VMDQ_DCB:
4147 dcb_config->vt_mode = true;
4148 if (hw->mac.type != ixgbe_mac_82598EB) {
4149 config_dcb_rx = DCB_RX_CONFIG;
4151 *get dcb and VT rx configuration parameters
4154 ixgbe_vmdq_dcb_rx_config(dev, dcb_config);
4155 /*Configure general VMDQ and DCB RX parameters*/
4156 ixgbe_vmdq_dcb_configure(dev);
4159 case RTE_ETH_MQ_RX_DCB:
4160 case RTE_ETH_MQ_RX_DCB_RSS:
4161 dcb_config->vt_mode = false;
4162 config_dcb_rx = DCB_RX_CONFIG;
4163 /* Get dcb TX configuration parameters from rte_eth_conf */
4164 ixgbe_dcb_rx_config(dev, dcb_config);
4165 /*Configure general DCB RX parameters*/
4166 ixgbe_dcb_rx_hw_config(dev, dcb_config);
4169 PMD_INIT_LOG(ERR, "Incorrect DCB RX mode configuration");
4172 switch (dev->data->dev_conf.txmode.mq_mode) {
4173 case RTE_ETH_MQ_TX_VMDQ_DCB:
4174 dcb_config->vt_mode = true;
4175 config_dcb_tx = DCB_TX_CONFIG;
4176 /* get DCB and VT TX configuration parameters
4179 ixgbe_dcb_vt_tx_config(dev, dcb_config);
4180 /*Configure general VMDQ and DCB TX parameters*/
4181 ixgbe_vmdq_dcb_hw_tx_config(dev, dcb_config);
4184 case RTE_ETH_MQ_TX_DCB:
4185 dcb_config->vt_mode = false;
4186 config_dcb_tx = DCB_TX_CONFIG;
4187 /*get DCB TX configuration parameters from rte_eth_conf*/
4188 ixgbe_dcb_tx_config(dev, dcb_config);
4189 /*Configure general DCB TX parameters*/
4190 ixgbe_dcb_tx_hw_config(dev, dcb_config);
4193 PMD_INIT_LOG(ERR, "Incorrect DCB TX mode configuration");
4197 nb_tcs = dcb_config->num_tcs.pfc_tcs;
4199 ixgbe_dcb_unpack_map_cee(dcb_config, IXGBE_DCB_RX_CONFIG, map);
4200 if (nb_tcs == RTE_ETH_4_TCS) {
4201 /* Avoid un-configured priority mapping to TC0 */
4203 uint8_t mask = 0xFF;
4205 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES - 4; i++)
4206 mask = (uint8_t)(mask & (~(1 << map[i])));
4207 for (i = 0; mask && (i < IXGBE_DCB_MAX_TRAFFIC_CLASS); i++) {
4208 if ((mask & 0x1) && j < RTE_ETH_DCB_NUM_USER_PRIORITIES)
4212 /* Re-configure 4 TCs BW */
4213 for (i = 0; i < nb_tcs; i++) {
4214 tc = &dcb_config->tc_config[i];
4215 if (bw_conf->tc_num != nb_tcs)
4216 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent =
4217 (uint8_t)(100 / nb_tcs);
4218 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent =
4219 (uint8_t)(100 / nb_tcs);
4221 for (; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
4222 tc = &dcb_config->tc_config[i];
4223 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent = 0;
4224 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent = 0;
4227 /* Re-configure 8 TCs BW */
4228 for (i = 0; i < nb_tcs; i++) {
4229 tc = &dcb_config->tc_config[i];
4230 if (bw_conf->tc_num != nb_tcs)
4231 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent =
4232 (uint8_t)(100 / nb_tcs + (i & 1));
4233 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent =
4234 (uint8_t)(100 / nb_tcs + (i & 1));
4238 switch (hw->mac.type) {
4239 case ixgbe_mac_X550:
4240 case ixgbe_mac_X550EM_x:
4241 case ixgbe_mac_X550EM_a:
4242 rx_buffer_size = X550_RX_BUFFER_SIZE;
4245 rx_buffer_size = NIC_RX_BUFFER_SIZE;
4249 if (config_dcb_rx) {
4250 /* Set RX buffer size */
4251 pbsize = (uint16_t)(rx_buffer_size / nb_tcs);
4252 uint32_t rxpbsize = pbsize << IXGBE_RXPBSIZE_SHIFT;
4254 for (i = 0; i < nb_tcs; i++) {
4255 IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), rxpbsize);
4257 /* zero alloc all unused TCs */
4258 for (; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++)
4259 IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), 0);
4261 if (config_dcb_tx) {
4262 /* Only support an equally distributed
4263 * Tx packet buffer strategy.
4265 uint32_t txpktsize = IXGBE_TXPBSIZE_MAX / nb_tcs;
4266 uint32_t txpbthresh = (txpktsize / DCB_TX_PB) - IXGBE_TXPKT_SIZE_MAX;
4268 for (i = 0; i < nb_tcs; i++) {
4269 IXGBE_WRITE_REG(hw, IXGBE_TXPBSIZE(i), txpktsize);
4270 IXGBE_WRITE_REG(hw, IXGBE_TXPBTHRESH(i), txpbthresh);
4272 /* Clear unused TCs, if any, to zero buffer size*/
4273 for (; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
4274 IXGBE_WRITE_REG(hw, IXGBE_TXPBSIZE(i), 0);
4275 IXGBE_WRITE_REG(hw, IXGBE_TXPBTHRESH(i), 0);
4279 /*Calculates traffic class credits*/
4280 ixgbe_dcb_calculate_tc_credits_cee(hw, dcb_config, max_frame,
4281 IXGBE_DCB_TX_CONFIG);
4282 ixgbe_dcb_calculate_tc_credits_cee(hw, dcb_config, max_frame,
4283 IXGBE_DCB_RX_CONFIG);
4285 if (config_dcb_rx) {
4286 /* Unpack CEE standard containers */
4287 ixgbe_dcb_unpack_refill_cee(dcb_config, IXGBE_DCB_RX_CONFIG, refill);
4288 ixgbe_dcb_unpack_max_cee(dcb_config, max);
4289 ixgbe_dcb_unpack_bwgid_cee(dcb_config, IXGBE_DCB_RX_CONFIG, bwgid);
4290 ixgbe_dcb_unpack_tsa_cee(dcb_config, IXGBE_DCB_RX_CONFIG, tsa);
4291 /* Configure PG(ETS) RX */
4292 ixgbe_dcb_hw_arbite_rx_config(hw, refill, max, bwgid, tsa, map);
4295 if (config_dcb_tx) {
4296 /* Unpack CEE standard containers */
4297 ixgbe_dcb_unpack_refill_cee(dcb_config, IXGBE_DCB_TX_CONFIG, refill);
4298 ixgbe_dcb_unpack_max_cee(dcb_config, max);
4299 ixgbe_dcb_unpack_bwgid_cee(dcb_config, IXGBE_DCB_TX_CONFIG, bwgid);
4300 ixgbe_dcb_unpack_tsa_cee(dcb_config, IXGBE_DCB_TX_CONFIG, tsa);
4301 /* Configure PG(ETS) TX */
4302 ixgbe_dcb_hw_arbite_tx_config(hw, refill, max, bwgid, tsa, map);
4305 /*Configure queue statistics registers*/
4306 ixgbe_dcb_config_tc_stats_82599(hw, dcb_config);
4308 /* Check if the PFC is supported */
4309 if (dev->data->dev_conf.dcb_capability_en & RTE_ETH_DCB_PFC_SUPPORT) {
4310 pbsize = (uint16_t)(rx_buffer_size / nb_tcs);
4311 for (i = 0; i < nb_tcs; i++) {
4313 * If the TC count is 8,and the default high_water is 48,
4314 * the low_water is 16 as default.
4316 hw->fc.high_water[i] = (pbsize * 3) / 4;
4317 hw->fc.low_water[i] = pbsize / 4;
4318 /* Enable pfc for this TC */
4319 tc = &dcb_config->tc_config[i];
4320 tc->pfc = ixgbe_dcb_pfc_enabled;
4322 ixgbe_dcb_unpack_pfc_cee(dcb_config, map, &pfc_en);
4323 if (dcb_config->num_tcs.pfc_tcs == RTE_ETH_4_TCS)
4325 ret = ixgbe_dcb_config_pfc(hw, pfc_en, map);
4332 * ixgbe_configure_dcb - Configure DCB Hardware
4333 * @dev: pointer to rte_eth_dev
4335 void ixgbe_configure_dcb(struct rte_eth_dev *dev)
4337 struct ixgbe_dcb_config *dcb_cfg =
4338 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private);
4339 struct rte_eth_conf *dev_conf = &(dev->data->dev_conf);
4341 PMD_INIT_FUNC_TRACE();
4343 /* check support mq_mode for DCB */
4344 if (dev_conf->rxmode.mq_mode != RTE_ETH_MQ_RX_VMDQ_DCB &&
4345 dev_conf->rxmode.mq_mode != RTE_ETH_MQ_RX_DCB &&
4346 dev_conf->rxmode.mq_mode != RTE_ETH_MQ_RX_DCB_RSS)
4349 if (dev->data->nb_rx_queues > RTE_ETH_DCB_NUM_QUEUES)
4352 /** Configure DCB hardware **/
4353 ixgbe_dcb_hw_configure(dev, dcb_cfg);
4357 * VMDq only support for 10 GbE NIC.
4360 ixgbe_vmdq_rx_hw_configure(struct rte_eth_dev *dev)
4362 struct rte_eth_vmdq_rx_conf *cfg;
4363 struct ixgbe_hw *hw;
4364 enum rte_eth_nb_pools num_pools;
4365 uint32_t mrqc, vt_ctl, vlanctrl;
4369 PMD_INIT_FUNC_TRACE();
4370 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4371 cfg = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
4372 num_pools = cfg->nb_queue_pools;
4374 ixgbe_rss_disable(dev);
4376 /* MRQC: enable vmdq */
4377 mrqc = IXGBE_MRQC_VMDQEN;
4378 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4380 /* PFVTCTL: turn on virtualisation and set the default pool */
4381 vt_ctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
4382 if (cfg->enable_default_pool)
4383 vt_ctl |= (cfg->default_pool << IXGBE_VT_CTL_POOL_SHIFT);
4385 vt_ctl |= IXGBE_VT_CTL_DIS_DEFPL;
4387 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vt_ctl);
4389 for (i = 0; i < (int)num_pools; i++) {
4390 vmolr = ixgbe_convert_vm_rx_mask_to_val(cfg->rx_mode, vmolr);
4391 IXGBE_WRITE_REG(hw, IXGBE_VMOLR(i), vmolr);
4394 /* VLNCTRL: enable vlan filtering and allow all vlan tags through */
4395 vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4396 vlanctrl |= IXGBE_VLNCTRL_VFE; /* enable vlan filters */
4397 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl);
4399 /* VFTA - enable all vlan filters */
4400 for (i = 0; i < NUM_VFTA_REGISTERS; i++)
4401 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), UINT32_MAX);
4403 /* VFRE: pool enabling for receive - 64 */
4404 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), UINT32_MAX);
4405 if (num_pools == RTE_ETH_64_POOLS)
4406 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), UINT32_MAX);
4409 * MPSAR - allow pools to read specific mac addresses
4410 * In this case, all pools should be able to read from mac addr 0
4412 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(0), UINT32_MAX);
4413 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(0), UINT32_MAX);
4415 /* PFVLVF, PFVLVFB: set up filters for vlan tags as configured */
4416 for (i = 0; i < cfg->nb_pool_maps; i++) {
4417 /* set vlan id in VF register and set the valid bit */
4418 IXGBE_WRITE_REG(hw, IXGBE_VLVF(i), (IXGBE_VLVF_VIEN |
4419 (cfg->pool_map[i].vlan_id & IXGBE_RXD_VLAN_ID_MASK)));
4421 * Put the allowed pools in VFB reg. As we only have 16 or 64
4422 * pools, we only need to use the first half of the register
4425 if (((cfg->pool_map[i].pools >> 32) & UINT32_MAX) == 0)
4426 IXGBE_WRITE_REG(hw, IXGBE_VLVFB(i * 2),
4427 (cfg->pool_map[i].pools & UINT32_MAX));
4429 IXGBE_WRITE_REG(hw, IXGBE_VLVFB((i * 2 + 1)),
4430 ((cfg->pool_map[i].pools >> 32) & UINT32_MAX));
4434 /* PFDMA Tx General Switch Control Enables VMDQ loopback */
4435 if (cfg->enable_loop_back) {
4436 IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN);
4437 for (i = 0; i < RTE_IXGBE_VMTXSW_REGISTER_COUNT; i++)
4438 IXGBE_WRITE_REG(hw, IXGBE_VMTXSW(i), UINT32_MAX);
4441 IXGBE_WRITE_FLUSH(hw);
4445 * ixgbe_dcb_config_tx_hw_config - Configure general VMDq TX parameters
4446 * @hw: pointer to hardware structure
4449 ixgbe_vmdq_tx_hw_configure(struct ixgbe_hw *hw)
4454 PMD_INIT_FUNC_TRACE();
4455 /*PF VF Transmit Enable*/
4456 IXGBE_WRITE_REG(hw, IXGBE_VFTE(0), UINT32_MAX);
4457 IXGBE_WRITE_REG(hw, IXGBE_VFTE(1), UINT32_MAX);
4459 /* Disable the Tx desc arbiter so that MTQC can be changed */
4460 reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
4461 reg |= IXGBE_RTTDCS_ARBDIS;
4462 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg);
4464 reg = IXGBE_MTQC_VT_ENA | IXGBE_MTQC_64VF;
4465 IXGBE_WRITE_REG(hw, IXGBE_MTQC, reg);
4467 /* Disable drop for all queues */
4468 for (q = 0; q < IXGBE_MAX_RX_QUEUE_NUM; q++)
4469 IXGBE_WRITE_REG(hw, IXGBE_QDE,
4470 (IXGBE_QDE_WRITE | (q << IXGBE_QDE_IDX_SHIFT)));
4472 /* Enable the Tx desc arbiter */
4473 reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
4474 reg &= ~IXGBE_RTTDCS_ARBDIS;
4475 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg);
4477 IXGBE_WRITE_FLUSH(hw);
4480 static int __rte_cold
4481 ixgbe_alloc_rx_queue_mbufs(struct ixgbe_rx_queue *rxq)
4483 struct ixgbe_rx_entry *rxe = rxq->sw_ring;
4487 /* Initialize software ring entries */
4488 for (i = 0; i < rxq->nb_rx_desc; i++) {
4489 volatile union ixgbe_adv_rx_desc *rxd;
4490 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mb_pool);
4493 PMD_INIT_LOG(ERR, "RX mbuf alloc failed queue_id=%u",
4494 (unsigned) rxq->queue_id);
4498 mbuf->data_off = RTE_PKTMBUF_HEADROOM;
4499 mbuf->port = rxq->port_id;
4502 rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf));
4503 rxd = &rxq->rx_ring[i];
4504 rxd->read.hdr_addr = 0;
4505 rxd->read.pkt_addr = dma_addr;
4513 ixgbe_config_vf_rss(struct rte_eth_dev *dev)
4515 struct ixgbe_hw *hw;
4518 ixgbe_rss_configure(dev);
4520 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4522 /* MRQC: enable VF RSS */
4523 mrqc = IXGBE_READ_REG(hw, IXGBE_MRQC);
4524 mrqc &= ~IXGBE_MRQC_MRQE_MASK;
4525 switch (RTE_ETH_DEV_SRIOV(dev).active) {
4526 case RTE_ETH_64_POOLS:
4527 mrqc |= IXGBE_MRQC_VMDQRSS64EN;
4530 case RTE_ETH_32_POOLS:
4531 mrqc |= IXGBE_MRQC_VMDQRSS32EN;
4535 PMD_INIT_LOG(ERR, "Invalid pool number in IOV mode with VMDQ RSS");
4539 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4545 ixgbe_config_vf_default(struct rte_eth_dev *dev)
4547 struct ixgbe_hw *hw =
4548 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4550 switch (RTE_ETH_DEV_SRIOV(dev).active) {
4551 case RTE_ETH_64_POOLS:
4552 IXGBE_WRITE_REG(hw, IXGBE_MRQC,
4556 case RTE_ETH_32_POOLS:
4557 IXGBE_WRITE_REG(hw, IXGBE_MRQC,
4558 IXGBE_MRQC_VMDQRT4TCEN);
4561 case RTE_ETH_16_POOLS:
4562 IXGBE_WRITE_REG(hw, IXGBE_MRQC,
4563 IXGBE_MRQC_VMDQRT8TCEN);
4567 "invalid pool number in IOV mode");
4574 ixgbe_dev_mq_rx_configure(struct rte_eth_dev *dev)
4576 struct ixgbe_hw *hw =
4577 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4579 if (hw->mac.type == ixgbe_mac_82598EB)
4582 if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
4584 * SRIOV inactive scheme
4585 * any DCB/RSS w/o VMDq multi-queue setting
4587 switch (dev->data->dev_conf.rxmode.mq_mode) {
4588 case RTE_ETH_MQ_RX_RSS:
4589 case RTE_ETH_MQ_RX_DCB_RSS:
4590 case RTE_ETH_MQ_RX_VMDQ_RSS:
4591 ixgbe_rss_configure(dev);
4594 case RTE_ETH_MQ_RX_VMDQ_DCB:
4595 ixgbe_vmdq_dcb_configure(dev);
4598 case RTE_ETH_MQ_RX_VMDQ_ONLY:
4599 ixgbe_vmdq_rx_hw_configure(dev);
4602 case RTE_ETH_MQ_RX_NONE:
4604 /* if mq_mode is none, disable rss mode.*/
4605 ixgbe_rss_disable(dev);
4609 /* SRIOV active scheme
4610 * Support RSS together with SRIOV.
4612 switch (dev->data->dev_conf.rxmode.mq_mode) {
4613 case RTE_ETH_MQ_RX_RSS:
4614 case RTE_ETH_MQ_RX_VMDQ_RSS:
4615 ixgbe_config_vf_rss(dev);
4617 case RTE_ETH_MQ_RX_VMDQ_DCB:
4618 case RTE_ETH_MQ_RX_DCB:
4619 /* In SRIOV, the configuration is the same as VMDq case */
4620 ixgbe_vmdq_dcb_configure(dev);
4622 /* DCB/RSS together with SRIOV is not supported */
4623 case RTE_ETH_MQ_RX_VMDQ_DCB_RSS:
4624 case RTE_ETH_MQ_RX_DCB_RSS:
4626 "Could not support DCB/RSS with VMDq & SRIOV");
4629 ixgbe_config_vf_default(dev);
4638 ixgbe_dev_mq_tx_configure(struct rte_eth_dev *dev)
4640 struct ixgbe_hw *hw =
4641 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4645 if (hw->mac.type == ixgbe_mac_82598EB)
4648 /* disable arbiter before setting MTQC */
4649 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
4650 rttdcs |= IXGBE_RTTDCS_ARBDIS;
4651 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
4653 if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
4655 * SRIOV inactive scheme
4656 * any DCB w/o VMDq multi-queue setting
4658 if (dev->data->dev_conf.txmode.mq_mode == RTE_ETH_MQ_TX_VMDQ_ONLY)
4659 ixgbe_vmdq_tx_hw_configure(hw);
4661 mtqc = IXGBE_MTQC_64Q_1PB;
4662 IXGBE_WRITE_REG(hw, IXGBE_MTQC, mtqc);
4665 switch (RTE_ETH_DEV_SRIOV(dev).active) {
4668 * SRIOV active scheme
4669 * FIXME if support DCB together with VMDq & SRIOV
4671 case RTE_ETH_64_POOLS:
4672 mtqc = IXGBE_MTQC_VT_ENA | IXGBE_MTQC_64VF;
4674 case RTE_ETH_32_POOLS:
4675 mtqc = IXGBE_MTQC_VT_ENA | IXGBE_MTQC_32VF;
4677 case RTE_ETH_16_POOLS:
4678 mtqc = IXGBE_MTQC_VT_ENA | IXGBE_MTQC_RT_ENA |
4682 mtqc = IXGBE_MTQC_64Q_1PB;
4683 PMD_INIT_LOG(ERR, "invalid pool number in IOV mode");
4685 IXGBE_WRITE_REG(hw, IXGBE_MTQC, mtqc);
4688 /* re-enable arbiter */
4689 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
4690 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
4696 * ixgbe_get_rscctl_maxdesc - Calculate the RSCCTL[n].MAXDESC for PF
4698 * Return the RSCCTL[n].MAXDESC for 82599 and x540 PF devices according to the
4699 * spec rev. 3.0 chapter 8.2.3.8.13.
4701 * @pool Memory pool of the Rx queue
4703 static inline uint32_t
4704 ixgbe_get_rscctl_maxdesc(struct rte_mempool *pool)
4706 struct rte_pktmbuf_pool_private *mp_priv = rte_mempool_get_priv(pool);
4708 /* MAXDESC * SRRCTL.BSIZEPKT must not exceed 64 KB minus one */
4710 RTE_IPV4_MAX_PKT_LEN /
4711 (mp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM);
4714 return IXGBE_RSCCTL_MAXDESC_16;
4715 else if (maxdesc >= 8)
4716 return IXGBE_RSCCTL_MAXDESC_8;
4717 else if (maxdesc >= 4)
4718 return IXGBE_RSCCTL_MAXDESC_4;
4720 return IXGBE_RSCCTL_MAXDESC_1;
4724 * ixgbe_set_ivar - Setup the correct IVAR register for a particular MSIX
4727 * (Taken from FreeBSD tree)
4728 * (yes this is all very magic and confusing :)
4731 * @entry the register array entry
4732 * @vector the MSIX vector for this queue
4736 ixgbe_set_ivar(struct rte_eth_dev *dev, u8 entry, u8 vector, s8 type)
4738 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4741 vector |= IXGBE_IVAR_ALLOC_VAL;
4743 switch (hw->mac.type) {
4745 case ixgbe_mac_82598EB:
4747 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
4749 entry += (type * 64);
4750 index = (entry >> 2) & 0x1F;
4751 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4752 ivar &= ~(0xFF << (8 * (entry & 0x3)));
4753 ivar |= (vector << (8 * (entry & 0x3)));
4754 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
4757 case ixgbe_mac_82599EB:
4758 case ixgbe_mac_X540:
4759 if (type == -1) { /* MISC IVAR */
4760 index = (entry & 1) * 8;
4761 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4762 ivar &= ~(0xFF << index);
4763 ivar |= (vector << index);
4764 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4765 } else { /* RX/TX IVARS */
4766 index = (16 * (entry & 1)) + (8 * type);
4767 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
4768 ivar &= ~(0xFF << index);
4769 ivar |= (vector << index);
4770 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
4781 ixgbe_set_rx_function(struct rte_eth_dev *dev)
4783 uint16_t i, rx_using_sse;
4784 struct ixgbe_adapter *adapter = dev->data->dev_private;
4787 * In order to allow Vector Rx there are a few configuration
4788 * conditions to be met and Rx Bulk Allocation should be allowed.
4790 if (ixgbe_rx_vec_dev_conf_condition_check(dev) ||
4791 !adapter->rx_bulk_alloc_allowed ||
4792 rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128) {
4793 PMD_INIT_LOG(DEBUG, "Port[%d] doesn't meet Vector Rx "
4795 dev->data->port_id);
4797 adapter->rx_vec_allowed = false;
4801 * Initialize the appropriate LRO callback.
4803 * If all queues satisfy the bulk allocation preconditions
4804 * (hw->rx_bulk_alloc_allowed is TRUE) then we may use bulk allocation.
4805 * Otherwise use a single allocation version.
4807 if (dev->data->lro) {
4808 if (adapter->rx_bulk_alloc_allowed) {
4809 PMD_INIT_LOG(DEBUG, "LRO is requested. Using a bulk "
4810 "allocation version");
4811 dev->rx_pkt_burst = ixgbe_recv_pkts_lro_bulk_alloc;
4813 PMD_INIT_LOG(DEBUG, "LRO is requested. Using a single "
4814 "allocation version");
4815 dev->rx_pkt_burst = ixgbe_recv_pkts_lro_single_alloc;
4817 } else if (dev->data->scattered_rx) {
4819 * Set the non-LRO scattered callback: there are Vector and
4820 * single allocation versions.
4822 if (adapter->rx_vec_allowed) {
4823 PMD_INIT_LOG(DEBUG, "Using Vector Scattered Rx "
4824 "callback (port=%d).",
4825 dev->data->port_id);
4827 dev->rx_pkt_burst = ixgbe_recv_scattered_pkts_vec;
4828 } else if (adapter->rx_bulk_alloc_allowed) {
4829 PMD_INIT_LOG(DEBUG, "Using a Scattered with bulk "
4830 "allocation callback (port=%d).",
4831 dev->data->port_id);
4832 dev->rx_pkt_burst = ixgbe_recv_pkts_lro_bulk_alloc;
4834 PMD_INIT_LOG(DEBUG, "Using Regular (non-vector, "
4835 "single allocation) "
4836 "Scattered Rx callback "
4838 dev->data->port_id);
4840 dev->rx_pkt_burst = ixgbe_recv_pkts_lro_single_alloc;
4843 * Below we set "simple" callbacks according to port/queues parameters.
4844 * If parameters allow we are going to choose between the following
4848 * - Single buffer allocation (the simplest one)
4850 } else if (adapter->rx_vec_allowed) {
4851 PMD_INIT_LOG(DEBUG, "Vector rx enabled, please make sure RX "
4852 "burst size no less than %d (port=%d).",
4853 RTE_IXGBE_DESCS_PER_LOOP,
4854 dev->data->port_id);
4856 dev->rx_pkt_burst = ixgbe_recv_pkts_vec;
4857 } else if (adapter->rx_bulk_alloc_allowed) {
4858 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are "
4859 "satisfied. Rx Burst Bulk Alloc function "
4860 "will be used on port=%d.",
4861 dev->data->port_id);
4863 dev->rx_pkt_burst = ixgbe_recv_pkts_bulk_alloc;
4865 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are not "
4866 "satisfied, or Scattered Rx is requested "
4868 dev->data->port_id);
4870 dev->rx_pkt_burst = ixgbe_recv_pkts;
4873 /* Propagate information about RX function choice through all queues. */
4876 (dev->rx_pkt_burst == ixgbe_recv_scattered_pkts_vec ||
4877 dev->rx_pkt_burst == ixgbe_recv_pkts_vec);
4879 for (i = 0; i < dev->data->nb_rx_queues; i++) {
4880 struct ixgbe_rx_queue *rxq = dev->data->rx_queues[i];
4882 rxq->rx_using_sse = rx_using_sse;
4883 #ifdef RTE_LIB_SECURITY
4884 rxq->using_ipsec = !!(dev->data->dev_conf.rxmode.offloads &
4885 RTE_ETH_RX_OFFLOAD_SECURITY);
4891 * ixgbe_set_rsc - configure RSC related port HW registers
4893 * Configures the port's RSC related registers according to the 4.6.7.2 chapter
4894 * of 82599 Spec (x540 configuration is virtually the same).
4898 * Returns 0 in case of success or a non-zero error code
4901 ixgbe_set_rsc(struct rte_eth_dev *dev)
4903 struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode;
4904 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4905 struct rte_eth_dev_info dev_info = { 0 };
4906 bool rsc_capable = false;
4912 dev->dev_ops->dev_infos_get(dev, &dev_info);
4913 if (dev_info.rx_offload_capa & RTE_ETH_RX_OFFLOAD_TCP_LRO)
4916 if (!rsc_capable && (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO)) {
4917 PMD_INIT_LOG(CRIT, "LRO is requested on HW that doesn't "
4922 /* RSC global configuration (chapter 4.6.7.2.1 of 82599 Spec) */
4924 if ((rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) &&
4925 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO)) {
4927 * According to chapter of 4.6.7.2.1 of the Spec Rev.
4928 * 3.0 RSC configuration requires HW CRC stripping being
4929 * enabled. If user requested both HW CRC stripping off
4930 * and RSC on - return an error.
4932 PMD_INIT_LOG(CRIT, "LRO can't be enabled when HW CRC "
4937 /* RFCTL configuration */
4938 rfctl = IXGBE_READ_REG(hw, IXGBE_RFCTL);
4939 if ((rsc_capable) && (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO))
4940 rfctl &= ~IXGBE_RFCTL_RSC_DIS;
4942 rfctl |= IXGBE_RFCTL_RSC_DIS;
4943 /* disable NFS filtering */
4944 rfctl |= IXGBE_RFCTL_NFSW_DIS | IXGBE_RFCTL_NFSR_DIS;
4945 IXGBE_WRITE_REG(hw, IXGBE_RFCTL, rfctl);
4947 /* If LRO hasn't been requested - we are done here. */
4948 if (!(rx_conf->offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO))
4951 /* Set RDRXCTL.RSCACKC bit */
4952 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
4953 rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
4954 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
4956 /* Per-queue RSC configuration (chapter 4.6.7.2.2 of 82599 Spec) */
4957 for (i = 0; i < dev->data->nb_rx_queues; i++) {
4958 struct ixgbe_rx_queue *rxq = dev->data->rx_queues[i];
4960 IXGBE_READ_REG(hw, IXGBE_SRRCTL(rxq->reg_idx));
4962 IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxq->reg_idx));
4964 IXGBE_READ_REG(hw, IXGBE_PSRTYPE(rxq->reg_idx));
4966 IXGBE_READ_REG(hw, IXGBE_EITR(rxq->reg_idx));
4969 * ixgbe PMD doesn't support header-split at the moment.
4971 * Following the 4.6.7.2.1 chapter of the 82599/x540
4972 * Spec if RSC is enabled the SRRCTL[n].BSIZEHEADER
4973 * should be configured even if header split is not
4974 * enabled. We will configure it 128 bytes following the
4975 * recommendation in the spec.
4977 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
4978 srrctl |= (128 << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) &
4979 IXGBE_SRRCTL_BSIZEHDR_MASK;
4982 * TODO: Consider setting the Receive Descriptor Minimum
4983 * Threshold Size for an RSC case. This is not an obviously
4984 * beneficiary option but the one worth considering...
4987 rscctl |= IXGBE_RSCCTL_RSCEN;
4988 rscctl |= ixgbe_get_rscctl_maxdesc(rxq->mb_pool);
4989 psrtype |= IXGBE_PSRTYPE_TCPHDR;
4992 * RSC: Set ITR interval corresponding to 2K ints/s.
4994 * Full-sized RSC aggregations for a 10Gb/s link will
4995 * arrive at about 20K aggregation/s rate.
4997 * 2K inst/s rate will make only 10% of the
4998 * aggregations to be closed due to the interrupt timer
4999 * expiration for a streaming at wire-speed case.
5001 * For a sparse streaming case this setting will yield
5002 * at most 500us latency for a single RSC aggregation.
5004 eitr &= ~IXGBE_EITR_ITR_INT_MASK;
5005 eitr |= IXGBE_EITR_INTERVAL_US(IXGBE_QUEUE_ITR_INTERVAL_DEFAULT);
5006 eitr |= IXGBE_EITR_CNT_WDIS;
5008 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxq->reg_idx), srrctl);
5009 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxq->reg_idx), rscctl);
5010 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(rxq->reg_idx), psrtype);
5011 IXGBE_WRITE_REG(hw, IXGBE_EITR(rxq->reg_idx), eitr);
5014 * RSC requires the mapping of the queue to the
5017 ixgbe_set_ivar(dev, rxq->reg_idx, i, 0);
5022 PMD_INIT_LOG(DEBUG, "enabling LRO mode");
5028 * Initializes Receive Unit.
5031 ixgbe_dev_rx_init(struct rte_eth_dev *dev)
5033 struct ixgbe_hw *hw;
5034 struct ixgbe_rx_queue *rxq;
5045 struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode;
5046 uint32_t frame_size = dev->data->mtu + IXGBE_ETH_OVERHEAD;
5049 PMD_INIT_FUNC_TRACE();
5050 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5053 * Make sure receives are disabled while setting
5054 * up the RX context (registers, descriptor rings, etc.).
5056 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
5057 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl & ~IXGBE_RXCTRL_RXEN);
5059 /* Enable receipt of broadcasted frames */
5060 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
5061 fctrl |= IXGBE_FCTRL_BAM;
5062 fctrl |= IXGBE_FCTRL_DPF;
5063 fctrl |= IXGBE_FCTRL_PMCF;
5064 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
5067 * Configure CRC stripping, if any.
5069 hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
5070 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC)
5071 hlreg0 &= ~IXGBE_HLREG0_RXCRCSTRP;
5073 hlreg0 |= IXGBE_HLREG0_RXCRCSTRP;
5076 * Configure jumbo frame support, if any.
5078 if (dev->data->mtu > RTE_ETHER_MTU) {
5079 hlreg0 |= IXGBE_HLREG0_JUMBOEN;
5080 maxfrs = IXGBE_READ_REG(hw, IXGBE_MAXFRS);
5081 maxfrs &= 0x0000FFFF;
5082 maxfrs |= (frame_size << 16);
5083 IXGBE_WRITE_REG(hw, IXGBE_MAXFRS, maxfrs);
5085 hlreg0 &= ~IXGBE_HLREG0_JUMBOEN;
5088 * If loopback mode is configured, set LPBK bit.
5090 if (dev->data->dev_conf.lpbk_mode != 0) {
5091 rc = ixgbe_check_supported_loopback_mode(dev);
5093 PMD_INIT_LOG(ERR, "Unsupported loopback mode");
5096 hlreg0 |= IXGBE_HLREG0_LPBK;
5098 hlreg0 &= ~IXGBE_HLREG0_LPBK;
5101 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);
5104 * Assume no header split and no VLAN strip support
5105 * on any Rx queue first .
5107 rx_conf->offloads &= ~RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
5108 /* Setup RX queues */
5109 for (i = 0; i < dev->data->nb_rx_queues; i++) {
5110 rxq = dev->data->rx_queues[i];
5113 * Reset crc_len in case it was changed after queue setup by a
5114 * call to configure.
5116 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC)
5117 rxq->crc_len = RTE_ETHER_CRC_LEN;
5121 /* Setup the Base and Length of the Rx Descriptor Rings */
5122 bus_addr = rxq->rx_ring_phys_addr;
5123 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(rxq->reg_idx),
5124 (uint32_t)(bus_addr & 0x00000000ffffffffULL));
5125 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(rxq->reg_idx),
5126 (uint32_t)(bus_addr >> 32));
5127 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(rxq->reg_idx),
5128 rxq->nb_rx_desc * sizeof(union ixgbe_adv_rx_desc));
5129 IXGBE_WRITE_REG(hw, IXGBE_RDH(rxq->reg_idx), 0);
5130 IXGBE_WRITE_REG(hw, IXGBE_RDT(rxq->reg_idx), 0);
5132 /* Configure the SRRCTL register */
5133 srrctl = IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
5135 /* Set if packets are dropped when no descriptors available */
5137 srrctl |= IXGBE_SRRCTL_DROP_EN;
5140 * Configure the RX buffer size in the BSIZEPACKET field of
5141 * the SRRCTL register of the queue.
5142 * The value is in 1 KB resolution. Valid values can be from
5145 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) -
5146 RTE_PKTMBUF_HEADROOM);
5147 srrctl |= ((buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) &
5148 IXGBE_SRRCTL_BSIZEPKT_MASK);
5150 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxq->reg_idx), srrctl);
5152 buf_size = (uint16_t) ((srrctl & IXGBE_SRRCTL_BSIZEPKT_MASK) <<
5153 IXGBE_SRRCTL_BSIZEPKT_SHIFT);
5155 /* It adds dual VLAN length for supporting dual VLAN */
5156 if (frame_size + 2 * RTE_VLAN_HLEN > buf_size)
5157 dev->data->scattered_rx = 1;
5158 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP)
5159 rx_conf->offloads |= RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
5162 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_SCATTER)
5163 dev->data->scattered_rx = 1;
5166 * Device configured with multiple RX queues.
5168 ixgbe_dev_mq_rx_configure(dev);
5171 * Setup the Checksum Register.
5172 * Disable Full-Packet Checksum which is mutually exclusive with RSS.
5173 * Enable IP/L4 checksum computation by hardware if requested to do so.
5175 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
5176 rxcsum |= IXGBE_RXCSUM_PCSD;
5177 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_CHECKSUM)
5178 rxcsum |= IXGBE_RXCSUM_IPPCSE;
5180 rxcsum &= ~IXGBE_RXCSUM_IPPCSE;
5182 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
5184 if (hw->mac.type == ixgbe_mac_82599EB ||
5185 hw->mac.type == ixgbe_mac_X540) {
5186 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
5187 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC)
5188 rdrxctl &= ~IXGBE_RDRXCTL_CRCSTRIP;
5190 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
5191 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
5192 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
5195 rc = ixgbe_set_rsc(dev);
5199 ixgbe_set_rx_function(dev);
5205 * Initializes Transmit Unit.
5208 ixgbe_dev_tx_init(struct rte_eth_dev *dev)
5210 struct ixgbe_hw *hw;
5211 struct ixgbe_tx_queue *txq;
5217 PMD_INIT_FUNC_TRACE();
5218 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5220 /* Enable TX CRC (checksum offload requirement) and hw padding
5223 hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
5224 hlreg0 |= (IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN);
5225 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);
5227 /* Setup the Base and Length of the Tx Descriptor Rings */
5228 for (i = 0; i < dev->data->nb_tx_queues; i++) {
5229 txq = dev->data->tx_queues[i];
5231 bus_addr = txq->tx_ring_phys_addr;
5232 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(txq->reg_idx),
5233 (uint32_t)(bus_addr & 0x00000000ffffffffULL));
5234 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(txq->reg_idx),
5235 (uint32_t)(bus_addr >> 32));
5236 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(txq->reg_idx),
5237 txq->nb_tx_desc * sizeof(union ixgbe_adv_tx_desc));
5238 /* Setup the HW Tx Head and TX Tail descriptor pointers */
5239 IXGBE_WRITE_REG(hw, IXGBE_TDH(txq->reg_idx), 0);
5240 IXGBE_WRITE_REG(hw, IXGBE_TDT(txq->reg_idx), 0);
5243 * Disable Tx Head Writeback RO bit, since this hoses
5244 * bookkeeping if things aren't delivered in order.
5246 switch (hw->mac.type) {
5247 case ixgbe_mac_82598EB:
5248 txctrl = IXGBE_READ_REG(hw,
5249 IXGBE_DCA_TXCTRL(txq->reg_idx));
5250 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
5251 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(txq->reg_idx),
5255 case ixgbe_mac_82599EB:
5256 case ixgbe_mac_X540:
5257 case ixgbe_mac_X550:
5258 case ixgbe_mac_X550EM_x:
5259 case ixgbe_mac_X550EM_a:
5261 txctrl = IXGBE_READ_REG(hw,
5262 IXGBE_DCA_TXCTRL_82599(txq->reg_idx));
5263 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
5264 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(txq->reg_idx),
5270 /* Device configured with multiple TX queues. */
5271 ixgbe_dev_mq_tx_configure(dev);
5275 * Check if requested loopback mode is supported
5278 ixgbe_check_supported_loopback_mode(struct rte_eth_dev *dev)
5280 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5282 if (dev->data->dev_conf.lpbk_mode == IXGBE_LPBK_TX_RX)
5283 if (hw->mac.type == ixgbe_mac_82599EB ||
5284 hw->mac.type == ixgbe_mac_X540 ||
5285 hw->mac.type == ixgbe_mac_X550 ||
5286 hw->mac.type == ixgbe_mac_X550EM_x ||
5287 hw->mac.type == ixgbe_mac_X550EM_a)
5294 * Set up link for 82599 loopback mode Tx->Rx.
5296 static inline void __rte_cold
5297 ixgbe_setup_loopback_link_82599(struct ixgbe_hw *hw)
5299 PMD_INIT_FUNC_TRACE();
5301 if (ixgbe_verify_lesm_fw_enabled_82599(hw)) {
5302 if (hw->mac.ops.acquire_swfw_sync(hw, IXGBE_GSSR_MAC_CSR_SM) !=
5304 PMD_INIT_LOG(ERR, "Could not enable loopback mode");
5313 IXGBE_AUTOC_LMS_10G_LINK_NO_AN | IXGBE_AUTOC_FLU);
5314 ixgbe_reset_pipeline_82599(hw);
5316 hw->mac.ops.release_swfw_sync(hw, IXGBE_GSSR_MAC_CSR_SM);
5322 * Start Transmit and Receive Units.
5325 ixgbe_dev_rxtx_start(struct rte_eth_dev *dev)
5327 struct ixgbe_hw *hw;
5328 struct ixgbe_tx_queue *txq;
5329 struct ixgbe_rx_queue *rxq;
5336 PMD_INIT_FUNC_TRACE();
5337 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5339 for (i = 0; i < dev->data->nb_tx_queues; i++) {
5340 txq = dev->data->tx_queues[i];
5341 /* Setup Transmit Threshold Registers */
5342 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txq->reg_idx));
5343 txdctl |= txq->pthresh & 0x7F;
5344 txdctl |= ((txq->hthresh & 0x7F) << 8);
5345 txdctl |= ((txq->wthresh & 0x7F) << 16);
5346 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txq->reg_idx), txdctl);
5349 if (hw->mac.type != ixgbe_mac_82598EB) {
5350 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
5351 dmatxctl |= IXGBE_DMATXCTL_TE;
5352 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
5355 for (i = 0; i < dev->data->nb_tx_queues; i++) {
5356 txq = dev->data->tx_queues[i];
5357 if (!txq->tx_deferred_start) {
5358 ret = ixgbe_dev_tx_queue_start(dev, i);
5364 for (i = 0; i < dev->data->nb_rx_queues; i++) {
5365 rxq = dev->data->rx_queues[i];
5366 if (!rxq->rx_deferred_start) {
5367 ret = ixgbe_dev_rx_queue_start(dev, i);
5373 /* Enable Receive engine */
5374 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
5375 if (hw->mac.type == ixgbe_mac_82598EB)
5376 rxctrl |= IXGBE_RXCTRL_DMBYPS;
5377 rxctrl |= IXGBE_RXCTRL_RXEN;
5378 hw->mac.ops.enable_rx_dma(hw, rxctrl);
5380 /* If loopback mode is enabled, set up the link accordingly */
5381 if (dev->data->dev_conf.lpbk_mode != 0) {
5382 if (hw->mac.type == ixgbe_mac_82599EB)
5383 ixgbe_setup_loopback_link_82599(hw);
5384 else if (hw->mac.type == ixgbe_mac_X540 ||
5385 hw->mac.type == ixgbe_mac_X550 ||
5386 hw->mac.type == ixgbe_mac_X550EM_x ||
5387 hw->mac.type == ixgbe_mac_X550EM_a)
5388 ixgbe_setup_loopback_link_x540_x550(hw, true);
5391 #ifdef RTE_LIB_SECURITY
5392 if ((dev->data->dev_conf.rxmode.offloads &
5393 RTE_ETH_RX_OFFLOAD_SECURITY) ||
5394 (dev->data->dev_conf.txmode.offloads &
5395 RTE_ETH_TX_OFFLOAD_SECURITY)) {
5396 ret = ixgbe_crypto_enable_ipsec(dev);
5399 "ixgbe_crypto_enable_ipsec fails with %d.",
5410 * Start Receive Units for specified queue.
5413 ixgbe_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
5415 struct ixgbe_hw *hw;
5416 struct ixgbe_rx_queue *rxq;
5420 PMD_INIT_FUNC_TRACE();
5421 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5423 rxq = dev->data->rx_queues[rx_queue_id];
5425 /* Allocate buffers for descriptor rings */
5426 if (ixgbe_alloc_rx_queue_mbufs(rxq) != 0) {
5427 PMD_INIT_LOG(ERR, "Could not alloc mbuf for queue:%d",
5431 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
5432 rxdctl |= IXGBE_RXDCTL_ENABLE;
5433 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxq->reg_idx), rxdctl);
5435 /* Wait until RX Enable ready */
5436 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
5439 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
5440 } while (--poll_ms && !(rxdctl & IXGBE_RXDCTL_ENABLE));
5442 PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", rx_queue_id);
5444 IXGBE_WRITE_REG(hw, IXGBE_RDH(rxq->reg_idx), 0);
5445 IXGBE_WRITE_REG(hw, IXGBE_RDT(rxq->reg_idx), rxq->nb_rx_desc - 1);
5446 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
5452 * Stop Receive Units for specified queue.
5455 ixgbe_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
5457 struct ixgbe_hw *hw;
5458 struct ixgbe_adapter *adapter = dev->data->dev_private;
5459 struct ixgbe_rx_queue *rxq;
5463 PMD_INIT_FUNC_TRACE();
5464 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5466 rxq = dev->data->rx_queues[rx_queue_id];
5468 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
5469 rxdctl &= ~IXGBE_RXDCTL_ENABLE;
5470 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxq->reg_idx), rxdctl);
5472 /* Wait until RX Enable bit clear */
5473 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
5476 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
5477 } while (--poll_ms && (rxdctl & IXGBE_RXDCTL_ENABLE));
5479 PMD_INIT_LOG(ERR, "Could not disable Rx Queue %d", rx_queue_id);
5481 rte_delay_us(RTE_IXGBE_WAIT_100_US);
5483 ixgbe_rx_queue_release_mbufs(rxq);
5484 ixgbe_reset_rx_queue(adapter, rxq);
5485 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
5492 * Start Transmit Units for specified queue.
5495 ixgbe_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
5497 struct ixgbe_hw *hw;
5498 struct ixgbe_tx_queue *txq;
5502 PMD_INIT_FUNC_TRACE();
5503 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5505 txq = dev->data->tx_queues[tx_queue_id];
5506 IXGBE_WRITE_REG(hw, IXGBE_TDH(txq->reg_idx), 0);
5507 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txq->reg_idx));
5508 txdctl |= IXGBE_TXDCTL_ENABLE;
5509 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txq->reg_idx), txdctl);
5511 /* Wait until TX Enable ready */
5512 if (hw->mac.type == ixgbe_mac_82599EB) {
5513 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
5516 txdctl = IXGBE_READ_REG(hw,
5517 IXGBE_TXDCTL(txq->reg_idx));
5518 } while (--poll_ms && !(txdctl & IXGBE_TXDCTL_ENABLE));
5520 PMD_INIT_LOG(ERR, "Could not enable Tx Queue %d",
5524 IXGBE_WRITE_REG(hw, IXGBE_TDT(txq->reg_idx), 0);
5525 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
5531 * Stop Transmit Units for specified queue.
5534 ixgbe_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
5536 struct ixgbe_hw *hw;
5537 struct ixgbe_tx_queue *txq;
5539 uint32_t txtdh, txtdt;
5542 PMD_INIT_FUNC_TRACE();
5543 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5545 txq = dev->data->tx_queues[tx_queue_id];
5547 /* Wait until TX queue is empty */
5548 if (hw->mac.type == ixgbe_mac_82599EB) {
5549 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
5551 rte_delay_us(RTE_IXGBE_WAIT_100_US);
5552 txtdh = IXGBE_READ_REG(hw,
5553 IXGBE_TDH(txq->reg_idx));
5554 txtdt = IXGBE_READ_REG(hw,
5555 IXGBE_TDT(txq->reg_idx));
5556 } while (--poll_ms && (txtdh != txtdt));
5559 "Tx Queue %d is not empty when stopping.",
5563 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txq->reg_idx));
5564 txdctl &= ~IXGBE_TXDCTL_ENABLE;
5565 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txq->reg_idx), txdctl);
5567 /* Wait until TX Enable bit clear */
5568 if (hw->mac.type == ixgbe_mac_82599EB) {
5569 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
5572 txdctl = IXGBE_READ_REG(hw,
5573 IXGBE_TXDCTL(txq->reg_idx));
5574 } while (--poll_ms && (txdctl & IXGBE_TXDCTL_ENABLE));
5576 PMD_INIT_LOG(ERR, "Could not disable Tx Queue %d",
5580 if (txq->ops != NULL) {
5581 txq->ops->release_mbufs(txq);
5582 txq->ops->reset(txq);
5584 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
5590 ixgbe_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
5591 struct rte_eth_rxq_info *qinfo)
5593 struct ixgbe_rx_queue *rxq;
5595 rxq = dev->data->rx_queues[queue_id];
5597 qinfo->mp = rxq->mb_pool;
5598 qinfo->scattered_rx = dev->data->scattered_rx;
5599 qinfo->nb_desc = rxq->nb_rx_desc;
5601 qinfo->conf.rx_free_thresh = rxq->rx_free_thresh;
5602 qinfo->conf.rx_drop_en = rxq->drop_en;
5603 qinfo->conf.rx_deferred_start = rxq->rx_deferred_start;
5604 qinfo->conf.offloads = rxq->offloads;
5608 ixgbe_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
5609 struct rte_eth_txq_info *qinfo)
5611 struct ixgbe_tx_queue *txq;
5613 txq = dev->data->tx_queues[queue_id];
5615 qinfo->nb_desc = txq->nb_tx_desc;
5617 qinfo->conf.tx_thresh.pthresh = txq->pthresh;
5618 qinfo->conf.tx_thresh.hthresh = txq->hthresh;
5619 qinfo->conf.tx_thresh.wthresh = txq->wthresh;
5621 qinfo->conf.tx_free_thresh = txq->tx_free_thresh;
5622 qinfo->conf.tx_rs_thresh = txq->tx_rs_thresh;
5623 qinfo->conf.offloads = txq->offloads;
5624 qinfo->conf.tx_deferred_start = txq->tx_deferred_start;
5628 * [VF] Initializes Receive Unit.
5631 ixgbevf_dev_rx_init(struct rte_eth_dev *dev)
5633 struct ixgbe_hw *hw;
5634 struct ixgbe_rx_queue *rxq;
5635 struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
5636 uint32_t frame_size = dev->data->mtu + IXGBE_ETH_OVERHEAD;
5638 uint32_t srrctl, psrtype = 0;
5643 PMD_INIT_FUNC_TRACE();
5644 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5646 if (rte_is_power_of_2(dev->data->nb_rx_queues) == 0) {
5647 PMD_INIT_LOG(ERR, "The number of Rx queue invalid, "
5648 "it should be power of 2");
5652 if (dev->data->nb_rx_queues > hw->mac.max_rx_queues) {
5653 PMD_INIT_LOG(ERR, "The number of Rx queue invalid, "
5654 "it should be equal to or less than %d",
5655 hw->mac.max_rx_queues);
5660 * When the VF driver issues a IXGBE_VF_RESET request, the PF driver
5661 * disables the VF receipt of packets if the PF MTU is > 1500.
5662 * This is done to deal with 82599 limitations that imposes
5663 * the PF and all VFs to share the same MTU.
5664 * Then, the PF driver enables again the VF receipt of packet when
5665 * the VF driver issues a IXGBE_VF_SET_LPE request.
5666 * In the meantime, the VF device cannot be used, even if the VF driver
5667 * and the Guest VM network stack are ready to accept packets with a
5668 * size up to the PF MTU.
5669 * As a work-around to this PF behaviour, force the call to
5670 * ixgbevf_rlpml_set_vf even if jumbo frames are not used. This way,
5671 * VF packets received can work in all cases.
5673 if (ixgbevf_rlpml_set_vf(hw, frame_size) != 0)
5674 PMD_INIT_LOG(ERR, "Set max packet length to %d failed.",
5678 * Assume no header split and no VLAN strip support
5679 * on any Rx queue first .
5681 rxmode->offloads &= ~RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
5682 /* Setup RX queues */
5683 for (i = 0; i < dev->data->nb_rx_queues; i++) {
5684 rxq = dev->data->rx_queues[i];
5686 /* Allocate buffers for descriptor rings */
5687 ret = ixgbe_alloc_rx_queue_mbufs(rxq);
5691 /* Setup the Base and Length of the Rx Descriptor Rings */
5692 bus_addr = rxq->rx_ring_phys_addr;
5694 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(i),
5695 (uint32_t)(bus_addr & 0x00000000ffffffffULL));
5696 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(i),
5697 (uint32_t)(bus_addr >> 32));
5698 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(i),
5699 rxq->nb_rx_desc * sizeof(union ixgbe_adv_rx_desc));
5700 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(i), 0);
5701 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(i), 0);
5704 /* Configure the SRRCTL register */
5705 srrctl = IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
5707 /* Set if packets are dropped when no descriptors available */
5709 srrctl |= IXGBE_SRRCTL_DROP_EN;
5712 * Configure the RX buffer size in the BSIZEPACKET field of
5713 * the SRRCTL register of the queue.
5714 * The value is in 1 KB resolution. Valid values can be from
5717 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) -
5718 RTE_PKTMBUF_HEADROOM);
5719 srrctl |= ((buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) &
5720 IXGBE_SRRCTL_BSIZEPKT_MASK);
5723 * VF modification to write virtual function SRRCTL register
5725 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(i), srrctl);
5727 buf_size = (uint16_t) ((srrctl & IXGBE_SRRCTL_BSIZEPKT_MASK) <<
5728 IXGBE_SRRCTL_BSIZEPKT_SHIFT);
5730 if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_SCATTER ||
5731 /* It adds dual VLAN length for supporting dual VLAN */
5732 (frame_size + 2 * RTE_VLAN_HLEN) > buf_size) {
5733 if (!dev->data->scattered_rx)
5734 PMD_INIT_LOG(DEBUG, "forcing scatter mode");
5735 dev->data->scattered_rx = 1;
5738 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP)
5739 rxmode->offloads |= RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
5742 /* Set RQPL for VF RSS according to max Rx queue */
5743 psrtype |= (dev->data->nb_rx_queues >> 1) <<
5744 IXGBE_PSRTYPE_RQPL_SHIFT;
5745 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype);
5747 ixgbe_set_rx_function(dev);
5753 * [VF] Initializes Transmit Unit.
5756 ixgbevf_dev_tx_init(struct rte_eth_dev *dev)
5758 struct ixgbe_hw *hw;
5759 struct ixgbe_tx_queue *txq;
5764 PMD_INIT_FUNC_TRACE();
5765 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5767 /* Setup the Base and Length of the Tx Descriptor Rings */
5768 for (i = 0; i < dev->data->nb_tx_queues; i++) {
5769 txq = dev->data->tx_queues[i];
5770 bus_addr = txq->tx_ring_phys_addr;
5771 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(i),
5772 (uint32_t)(bus_addr & 0x00000000ffffffffULL));
5773 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(i),
5774 (uint32_t)(bus_addr >> 32));
5775 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(i),
5776 txq->nb_tx_desc * sizeof(union ixgbe_adv_tx_desc));
5777 /* Setup the HW Tx Head and TX Tail descriptor pointers */
5778 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(i), 0);
5779 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(i), 0);
5782 * Disable Tx Head Writeback RO bit, since this hoses
5783 * bookkeeping if things aren't delivered in order.
5785 txctrl = IXGBE_READ_REG(hw,
5786 IXGBE_VFDCA_TXCTRL(i));
5787 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
5788 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(i),
5794 * [VF] Start Transmit and Receive Units.
5797 ixgbevf_dev_rxtx_start(struct rte_eth_dev *dev)
5799 struct ixgbe_hw *hw;
5800 struct ixgbe_tx_queue *txq;
5801 struct ixgbe_rx_queue *rxq;
5807 PMD_INIT_FUNC_TRACE();
5808 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5810 for (i = 0; i < dev->data->nb_tx_queues; i++) {
5811 txq = dev->data->tx_queues[i];
5812 /* Setup Transmit Threshold Registers */
5813 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i));
5814 txdctl |= txq->pthresh & 0x7F;
5815 txdctl |= ((txq->hthresh & 0x7F) << 8);
5816 txdctl |= ((txq->wthresh & 0x7F) << 16);
5817 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl);
5820 for (i = 0; i < dev->data->nb_tx_queues; i++) {
5822 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i));
5823 txdctl |= IXGBE_TXDCTL_ENABLE;
5824 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl);
5827 /* Wait until TX Enable ready */
5830 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i));
5831 } while (--poll_ms && !(txdctl & IXGBE_TXDCTL_ENABLE));
5833 PMD_INIT_LOG(ERR, "Could not enable Tx Queue %d", i);
5835 for (i = 0; i < dev->data->nb_rx_queues; i++) {
5837 rxq = dev->data->rx_queues[i];
5839 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i));
5840 rxdctl |= IXGBE_RXDCTL_ENABLE;
5841 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), rxdctl);
5843 /* Wait until RX Enable ready */
5847 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i));
5848 } while (--poll_ms && !(rxdctl & IXGBE_RXDCTL_ENABLE));
5850 PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", i);
5852 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(i), rxq->nb_rx_desc - 1);
5858 ixgbe_rss_conf_init(struct ixgbe_rte_flow_rss_conf *out,
5859 const struct rte_flow_action_rss *in)
5861 if (in->key_len > RTE_DIM(out->key) ||
5862 in->queue_num > RTE_DIM(out->queue))
5864 out->conf = (struct rte_flow_action_rss){
5868 .key_len = in->key_len,
5869 .queue_num = in->queue_num,
5870 .key = memcpy(out->key, in->key, in->key_len),
5871 .queue = memcpy(out->queue, in->queue,
5872 sizeof(*in->queue) * in->queue_num),
5878 ixgbe_action_rss_same(const struct rte_flow_action_rss *comp,
5879 const struct rte_flow_action_rss *with)
5881 return (comp->func == with->func &&
5882 comp->level == with->level &&
5883 comp->types == with->types &&
5884 comp->key_len == with->key_len &&
5885 comp->queue_num == with->queue_num &&
5886 !memcmp(comp->key, with->key, with->key_len) &&
5887 !memcmp(comp->queue, with->queue,
5888 sizeof(*with->queue) * with->queue_num));
5892 ixgbe_config_rss_filter(struct rte_eth_dev *dev,
5893 struct ixgbe_rte_flow_rss_conf *conf, bool add)
5895 struct ixgbe_hw *hw;
5899 uint16_t sp_reta_size;
5901 struct rte_eth_rss_conf rss_conf = {
5902 .rss_key = conf->conf.key_len ?
5903 (void *)(uintptr_t)conf->conf.key : NULL,
5904 .rss_key_len = conf->conf.key_len,
5905 .rss_hf = conf->conf.types,
5907 struct ixgbe_filter_info *filter_info =
5908 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
5910 PMD_INIT_FUNC_TRACE();
5911 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5913 sp_reta_size = ixgbe_reta_size_get(hw->mac.type);
5916 if (ixgbe_action_rss_same(&filter_info->rss_info.conf,
5918 ixgbe_rss_disable(dev);
5919 memset(&filter_info->rss_info, 0,
5920 sizeof(struct ixgbe_rte_flow_rss_conf));
5926 if (filter_info->rss_info.conf.queue_num)
5928 /* Fill in redirection table
5929 * The byte-swap is needed because NIC registers are in
5930 * little-endian order.
5933 for (i = 0, j = 0; i < sp_reta_size; i++, j++) {
5934 reta_reg = ixgbe_reta_reg_get(hw->mac.type, i);
5936 if (j == conf->conf.queue_num)
5938 reta = (reta << 8) | conf->conf.queue[j];
5940 IXGBE_WRITE_REG(hw, reta_reg,
5944 /* Configure the RSS key and the RSS protocols used to compute
5945 * the RSS hash of input packets.
5947 if ((rss_conf.rss_hf & IXGBE_RSS_OFFLOAD_ALL) == 0) {
5948 ixgbe_rss_disable(dev);
5951 if (rss_conf.rss_key == NULL)
5952 rss_conf.rss_key = rss_intel_key; /* Default hash key */
5953 ixgbe_hw_rss_hash_set(hw, &rss_conf);
5955 if (ixgbe_rss_conf_init(&filter_info->rss_info, &conf->conf))
5961 /* Stubs needed for linkage when RTE_ARCH_PPC_64 is set */
5962 #if defined(RTE_ARCH_PPC_64)
5964 ixgbe_rx_vec_dev_conf_condition_check(struct rte_eth_dev __rte_unused *dev)
5970 ixgbe_recv_pkts_vec(
5971 void __rte_unused *rx_queue,
5972 struct rte_mbuf __rte_unused **rx_pkts,
5973 uint16_t __rte_unused nb_pkts)
5979 ixgbe_recv_scattered_pkts_vec(
5980 void __rte_unused *rx_queue,
5981 struct rte_mbuf __rte_unused **rx_pkts,
5982 uint16_t __rte_unused nb_pkts)
5988 ixgbe_rxq_vec_setup(struct ixgbe_rx_queue __rte_unused *rxq)
5994 ixgbe_xmit_fixed_burst_vec(void __rte_unused *tx_queue,
5995 struct rte_mbuf __rte_unused **tx_pkts,
5996 uint16_t __rte_unused nb_pkts)
6002 ixgbe_txq_vec_setup(struct ixgbe_tx_queue __rte_unused *txq)
6008 ixgbe_rx_queue_release_mbufs_vec(struct ixgbe_rx_queue __rte_unused *rxq)