1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2016 Intel Corporation.
3 * Copyright 2014 6WIND S.A.
17 #include <rte_byteorder.h>
18 #include <rte_common.h>
19 #include <rte_cycles.h>
21 #include <rte_debug.h>
22 #include <rte_interrupts.h>
24 #include <rte_memory.h>
25 #include <rte_memzone.h>
26 #include <rte_launch.h>
28 #include <rte_per_lcore.h>
29 #include <rte_lcore.h>
30 #include <rte_atomic.h>
31 #include <rte_branch_prediction.h>
32 #include <rte_mempool.h>
33 #include <rte_malloc.h>
35 #include <rte_ether.h>
36 #include <ethdev_driver.h>
37 #include <rte_security_driver.h>
38 #include <rte_prefetch.h>
42 #include <rte_string_fns.h>
43 #include <rte_errno.h>
48 #include "ixgbe_logs.h"
49 #include "base/ixgbe_api.h"
50 #include "base/ixgbe_vf.h"
51 #include "ixgbe_ethdev.h"
52 #include "base/ixgbe_dcb.h"
53 #include "base/ixgbe_common.h"
54 #include "ixgbe_rxtx.h"
56 #ifdef RTE_LIBRTE_IEEE1588
57 #define IXGBE_TX_IEEE1588_TMST PKT_TX_IEEE1588_TMST
59 #define IXGBE_TX_IEEE1588_TMST 0
61 /* Bit Mask to indicate what bits required for building TX context */
62 #define IXGBE_TX_OFFLOAD_MASK ( \
72 PKT_TX_OUTER_IP_CKSUM | \
73 PKT_TX_SEC_OFFLOAD | \
74 IXGBE_TX_IEEE1588_TMST)
76 #define IXGBE_TX_OFFLOAD_NOTSUP_MASK \
77 (PKT_TX_OFFLOAD_MASK ^ IXGBE_TX_OFFLOAD_MASK)
80 #define RTE_PMD_USE_PREFETCH
83 #ifdef RTE_PMD_USE_PREFETCH
85 * Prefetch a cache line into all cache levels.
87 #define rte_ixgbe_prefetch(p) rte_prefetch0(p)
89 #define rte_ixgbe_prefetch(p) do {} while (0)
92 /*********************************************************************
96 **********************************************************************/
99 * Check for descriptors with their DD bit set and free mbufs.
100 * Return the total number of buffers freed.
102 static __rte_always_inline int
103 ixgbe_tx_free_bufs(struct ixgbe_tx_queue *txq)
105 struct ixgbe_tx_entry *txep;
108 struct rte_mbuf *m, *free[RTE_IXGBE_TX_MAX_FREE_BUF_SZ];
110 /* check DD bit on threshold descriptor */
111 status = txq->tx_ring[txq->tx_next_dd].wb.status;
112 if (!(status & rte_cpu_to_le_32(IXGBE_ADVTXD_STAT_DD)))
116 * first buffer to free from S/W ring is at index
117 * tx_next_dd - (tx_rs_thresh-1)
119 txep = &(txq->sw_ring[txq->tx_next_dd - (txq->tx_rs_thresh - 1)]);
121 for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) {
122 /* free buffers one at a time */
123 m = rte_pktmbuf_prefree_seg(txep->mbuf);
126 if (unlikely(m == NULL))
129 if (nb_free >= RTE_IXGBE_TX_MAX_FREE_BUF_SZ ||
130 (nb_free > 0 && m->pool != free[0]->pool)) {
131 rte_mempool_put_bulk(free[0]->pool,
132 (void **)free, nb_free);
140 rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free);
142 /* buffers were freed, update counters */
143 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh);
144 txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh);
145 if (txq->tx_next_dd >= txq->nb_tx_desc)
146 txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
148 return txq->tx_rs_thresh;
151 /* Populate 4 descriptors with data from 4 mbufs */
153 tx4(volatile union ixgbe_adv_tx_desc *txdp, struct rte_mbuf **pkts)
155 uint64_t buf_dma_addr;
159 for (i = 0; i < 4; ++i, ++txdp, ++pkts) {
160 buf_dma_addr = rte_mbuf_data_iova(*pkts);
161 pkt_len = (*pkts)->data_len;
163 /* write data to descriptor */
164 txdp->read.buffer_addr = rte_cpu_to_le_64(buf_dma_addr);
166 txdp->read.cmd_type_len =
167 rte_cpu_to_le_32((uint32_t)DCMD_DTYP_FLAGS | pkt_len);
169 txdp->read.olinfo_status =
170 rte_cpu_to_le_32(pkt_len << IXGBE_ADVTXD_PAYLEN_SHIFT);
172 rte_prefetch0(&(*pkts)->pool);
176 /* Populate 1 descriptor with data from 1 mbuf */
178 tx1(volatile union ixgbe_adv_tx_desc *txdp, struct rte_mbuf **pkts)
180 uint64_t buf_dma_addr;
183 buf_dma_addr = rte_mbuf_data_iova(*pkts);
184 pkt_len = (*pkts)->data_len;
186 /* write data to descriptor */
187 txdp->read.buffer_addr = rte_cpu_to_le_64(buf_dma_addr);
188 txdp->read.cmd_type_len =
189 rte_cpu_to_le_32((uint32_t)DCMD_DTYP_FLAGS | pkt_len);
190 txdp->read.olinfo_status =
191 rte_cpu_to_le_32(pkt_len << IXGBE_ADVTXD_PAYLEN_SHIFT);
192 rte_prefetch0(&(*pkts)->pool);
196 * Fill H/W descriptor ring with mbuf data.
197 * Copy mbuf pointers to the S/W ring.
200 ixgbe_tx_fill_hw_ring(struct ixgbe_tx_queue *txq, struct rte_mbuf **pkts,
203 volatile union ixgbe_adv_tx_desc *txdp = &(txq->tx_ring[txq->tx_tail]);
204 struct ixgbe_tx_entry *txep = &(txq->sw_ring[txq->tx_tail]);
205 const int N_PER_LOOP = 4;
206 const int N_PER_LOOP_MASK = N_PER_LOOP-1;
207 int mainpart, leftover;
211 * Process most of the packets in chunks of N pkts. Any
212 * leftover packets will get processed one at a time.
214 mainpart = (nb_pkts & ((uint32_t) ~N_PER_LOOP_MASK));
215 leftover = (nb_pkts & ((uint32_t) N_PER_LOOP_MASK));
216 for (i = 0; i < mainpart; i += N_PER_LOOP) {
217 /* Copy N mbuf pointers to the S/W ring */
218 for (j = 0; j < N_PER_LOOP; ++j) {
219 (txep + i + j)->mbuf = *(pkts + i + j);
221 tx4(txdp + i, pkts + i);
224 if (unlikely(leftover > 0)) {
225 for (i = 0; i < leftover; ++i) {
226 (txep + mainpart + i)->mbuf = *(pkts + mainpart + i);
227 tx1(txdp + mainpart + i, pkts + mainpart + i);
232 static inline uint16_t
233 tx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
236 struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue;
237 volatile union ixgbe_adv_tx_desc *tx_r = txq->tx_ring;
241 * Begin scanning the H/W ring for done descriptors when the
242 * number of available descriptors drops below tx_free_thresh. For
243 * each done descriptor, free the associated buffer.
245 if (txq->nb_tx_free < txq->tx_free_thresh)
246 ixgbe_tx_free_bufs(txq);
248 /* Only use descriptors that are available */
249 nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts);
250 if (unlikely(nb_pkts == 0))
253 /* Use exactly nb_pkts descriptors */
254 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts);
257 * At this point, we know there are enough descriptors in the
258 * ring to transmit all the packets. This assumes that each
259 * mbuf contains a single segment, and that no new offloads
260 * are expected, which would require a new context descriptor.
264 * See if we're going to wrap-around. If so, handle the top
265 * of the descriptor ring first, then do the bottom. If not,
266 * the processing looks just like the "bottom" part anyway...
268 if ((txq->tx_tail + nb_pkts) > txq->nb_tx_desc) {
269 n = (uint16_t)(txq->nb_tx_desc - txq->tx_tail);
270 ixgbe_tx_fill_hw_ring(txq, tx_pkts, n);
273 * We know that the last descriptor in the ring will need to
274 * have its RS bit set because tx_rs_thresh has to be
275 * a divisor of the ring size
277 tx_r[txq->tx_next_rs].read.cmd_type_len |=
278 rte_cpu_to_le_32(IXGBE_ADVTXD_DCMD_RS);
279 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
284 /* Fill H/W descriptor ring with mbuf data */
285 ixgbe_tx_fill_hw_ring(txq, tx_pkts + n, (uint16_t)(nb_pkts - n));
286 txq->tx_tail = (uint16_t)(txq->tx_tail + (nb_pkts - n));
289 * Determine if RS bit should be set
290 * This is what we actually want:
291 * if ((txq->tx_tail - 1) >= txq->tx_next_rs)
292 * but instead of subtracting 1 and doing >=, we can just do
293 * greater than without subtracting.
295 if (txq->tx_tail > txq->tx_next_rs) {
296 tx_r[txq->tx_next_rs].read.cmd_type_len |=
297 rte_cpu_to_le_32(IXGBE_ADVTXD_DCMD_RS);
298 txq->tx_next_rs = (uint16_t)(txq->tx_next_rs +
300 if (txq->tx_next_rs >= txq->nb_tx_desc)
301 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
305 * Check for wrap-around. This would only happen if we used
306 * up to the last descriptor in the ring, no more, no less.
308 if (txq->tx_tail >= txq->nb_tx_desc)
311 /* update tail pointer */
313 IXGBE_PCI_REG_WC_WRITE_RELAXED(txq->tdt_reg_addr, txq->tx_tail);
319 ixgbe_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts,
324 /* Try to transmit at least chunks of TX_MAX_BURST pkts */
325 if (likely(nb_pkts <= RTE_PMD_IXGBE_TX_MAX_BURST))
326 return tx_xmit_pkts(tx_queue, tx_pkts, nb_pkts);
328 /* transmit more than the max burst, in chunks of TX_MAX_BURST */
333 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_IXGBE_TX_MAX_BURST);
334 ret = tx_xmit_pkts(tx_queue, &(tx_pkts[nb_tx]), n);
335 nb_tx = (uint16_t)(nb_tx + ret);
336 nb_pkts = (uint16_t)(nb_pkts - ret);
345 ixgbe_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
349 struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue;
354 num = (uint16_t)RTE_MIN(nb_pkts, txq->tx_rs_thresh);
355 ret = ixgbe_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
367 ixgbe_set_xmit_ctx(struct ixgbe_tx_queue *txq,
368 volatile struct ixgbe_adv_tx_context_desc *ctx_txd,
369 uint64_t ol_flags, union ixgbe_tx_offload tx_offload,
370 __rte_unused uint64_t *mdata)
372 uint32_t type_tucmd_mlhl;
373 uint32_t mss_l4len_idx = 0;
375 uint32_t vlan_macip_lens;
376 union ixgbe_tx_offload tx_offload_mask;
377 uint32_t seqnum_seed = 0;
379 ctx_idx = txq->ctx_curr;
380 tx_offload_mask.data[0] = 0;
381 tx_offload_mask.data[1] = 0;
384 /* Specify which HW CTX to upload. */
385 mss_l4len_idx |= (ctx_idx << IXGBE_ADVTXD_IDX_SHIFT);
387 if (ol_flags & PKT_TX_VLAN_PKT) {
388 tx_offload_mask.vlan_tci |= ~0;
391 /* check if TCP segmentation required for this packet */
392 if (ol_flags & PKT_TX_TCP_SEG) {
393 /* implies IP cksum in IPv4 */
394 if (ol_flags & PKT_TX_IP_CKSUM)
395 type_tucmd_mlhl = IXGBE_ADVTXD_TUCMD_IPV4 |
396 IXGBE_ADVTXD_TUCMD_L4T_TCP |
397 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
399 type_tucmd_mlhl = IXGBE_ADVTXD_TUCMD_IPV6 |
400 IXGBE_ADVTXD_TUCMD_L4T_TCP |
401 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
403 tx_offload_mask.l2_len |= ~0;
404 tx_offload_mask.l3_len |= ~0;
405 tx_offload_mask.l4_len |= ~0;
406 tx_offload_mask.tso_segsz |= ~0;
407 mss_l4len_idx |= tx_offload.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT;
408 mss_l4len_idx |= tx_offload.l4_len << IXGBE_ADVTXD_L4LEN_SHIFT;
409 } else { /* no TSO, check if hardware checksum is needed */
410 if (ol_flags & PKT_TX_IP_CKSUM) {
411 type_tucmd_mlhl = IXGBE_ADVTXD_TUCMD_IPV4;
412 tx_offload_mask.l2_len |= ~0;
413 tx_offload_mask.l3_len |= ~0;
416 switch (ol_flags & PKT_TX_L4_MASK) {
417 case PKT_TX_UDP_CKSUM:
418 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP |
419 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
420 mss_l4len_idx |= sizeof(struct rte_udp_hdr)
421 << IXGBE_ADVTXD_L4LEN_SHIFT;
422 tx_offload_mask.l2_len |= ~0;
423 tx_offload_mask.l3_len |= ~0;
425 case PKT_TX_TCP_CKSUM:
426 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP |
427 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
428 mss_l4len_idx |= sizeof(struct rte_tcp_hdr)
429 << IXGBE_ADVTXD_L4LEN_SHIFT;
430 tx_offload_mask.l2_len |= ~0;
431 tx_offload_mask.l3_len |= ~0;
433 case PKT_TX_SCTP_CKSUM:
434 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP |
435 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
436 mss_l4len_idx |= sizeof(struct rte_sctp_hdr)
437 << IXGBE_ADVTXD_L4LEN_SHIFT;
438 tx_offload_mask.l2_len |= ~0;
439 tx_offload_mask.l3_len |= ~0;
442 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_RSV |
443 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
448 if (ol_flags & PKT_TX_OUTER_IP_CKSUM) {
449 tx_offload_mask.outer_l2_len |= ~0;
450 tx_offload_mask.outer_l3_len |= ~0;
451 tx_offload_mask.l2_len |= ~0;
452 seqnum_seed |= tx_offload.outer_l3_len
453 << IXGBE_ADVTXD_OUTER_IPLEN;
454 seqnum_seed |= tx_offload.l2_len
455 << IXGBE_ADVTXD_TUNNEL_LEN;
457 #ifdef RTE_LIB_SECURITY
458 if (ol_flags & PKT_TX_SEC_OFFLOAD) {
459 union ixgbe_crypto_tx_desc_md *md =
460 (union ixgbe_crypto_tx_desc_md *)mdata;
462 (IXGBE_ADVTXD_IPSEC_SA_INDEX_MASK & md->sa_idx);
463 type_tucmd_mlhl |= md->enc ?
464 (IXGBE_ADVTXD_TUCMD_IPSEC_TYPE_ESP |
465 IXGBE_ADVTXD_TUCMD_IPSEC_ENCRYPT_EN) : 0;
467 (md->pad_len & IXGBE_ADVTXD_IPSEC_ESP_LEN_MASK);
468 tx_offload_mask.sa_idx |= ~0;
469 tx_offload_mask.sec_pad_len |= ~0;
473 txq->ctx_cache[ctx_idx].flags = ol_flags;
474 txq->ctx_cache[ctx_idx].tx_offload.data[0] =
475 tx_offload_mask.data[0] & tx_offload.data[0];
476 txq->ctx_cache[ctx_idx].tx_offload.data[1] =
477 tx_offload_mask.data[1] & tx_offload.data[1];
478 txq->ctx_cache[ctx_idx].tx_offload_mask = tx_offload_mask;
480 ctx_txd->type_tucmd_mlhl = rte_cpu_to_le_32(type_tucmd_mlhl);
481 vlan_macip_lens = tx_offload.l3_len;
482 if (ol_flags & PKT_TX_OUTER_IP_CKSUM)
483 vlan_macip_lens |= (tx_offload.outer_l2_len <<
484 IXGBE_ADVTXD_MACLEN_SHIFT);
486 vlan_macip_lens |= (tx_offload.l2_len <<
487 IXGBE_ADVTXD_MACLEN_SHIFT);
488 vlan_macip_lens |= ((uint32_t)tx_offload.vlan_tci << IXGBE_ADVTXD_VLAN_SHIFT);
489 ctx_txd->vlan_macip_lens = rte_cpu_to_le_32(vlan_macip_lens);
490 ctx_txd->mss_l4len_idx = rte_cpu_to_le_32(mss_l4len_idx);
491 ctx_txd->seqnum_seed = seqnum_seed;
495 * Check which hardware context can be used. Use the existing match
496 * or create a new context descriptor.
498 static inline uint32_t
499 what_advctx_update(struct ixgbe_tx_queue *txq, uint64_t flags,
500 union ixgbe_tx_offload tx_offload)
502 /* If match with the current used context */
503 if (likely((txq->ctx_cache[txq->ctx_curr].flags == flags) &&
504 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] ==
505 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0]
506 & tx_offload.data[0])) &&
507 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] ==
508 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1]
509 & tx_offload.data[1]))))
510 return txq->ctx_curr;
512 /* What if match with the next context */
514 if (likely((txq->ctx_cache[txq->ctx_curr].flags == flags) &&
515 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] ==
516 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0]
517 & tx_offload.data[0])) &&
518 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] ==
519 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1]
520 & tx_offload.data[1]))))
521 return txq->ctx_curr;
523 /* Mismatch, use the previous context */
524 return IXGBE_CTX_NUM;
527 static inline uint32_t
528 tx_desc_cksum_flags_to_olinfo(uint64_t ol_flags)
532 if ((ol_flags & PKT_TX_L4_MASK) != PKT_TX_L4_NO_CKSUM)
533 tmp |= IXGBE_ADVTXD_POPTS_TXSM;
534 if (ol_flags & PKT_TX_IP_CKSUM)
535 tmp |= IXGBE_ADVTXD_POPTS_IXSM;
536 if (ol_flags & PKT_TX_TCP_SEG)
537 tmp |= IXGBE_ADVTXD_POPTS_TXSM;
541 static inline uint32_t
542 tx_desc_ol_flags_to_cmdtype(uint64_t ol_flags)
544 uint32_t cmdtype = 0;
546 if (ol_flags & PKT_TX_VLAN_PKT)
547 cmdtype |= IXGBE_ADVTXD_DCMD_VLE;
548 if (ol_flags & PKT_TX_TCP_SEG)
549 cmdtype |= IXGBE_ADVTXD_DCMD_TSE;
550 if (ol_flags & PKT_TX_OUTER_IP_CKSUM)
551 cmdtype |= (1 << IXGBE_ADVTXD_OUTERIPCS_SHIFT);
552 if (ol_flags & PKT_TX_MACSEC)
553 cmdtype |= IXGBE_ADVTXD_MAC_LINKSEC;
557 /* Default RS bit threshold values */
558 #ifndef DEFAULT_TX_RS_THRESH
559 #define DEFAULT_TX_RS_THRESH 32
561 #ifndef DEFAULT_TX_FREE_THRESH
562 #define DEFAULT_TX_FREE_THRESH 32
565 /* Reset transmit descriptors after they have been used */
567 ixgbe_xmit_cleanup(struct ixgbe_tx_queue *txq)
569 struct ixgbe_tx_entry *sw_ring = txq->sw_ring;
570 volatile union ixgbe_adv_tx_desc *txr = txq->tx_ring;
571 uint16_t last_desc_cleaned = txq->last_desc_cleaned;
572 uint16_t nb_tx_desc = txq->nb_tx_desc;
573 uint16_t desc_to_clean_to;
574 uint16_t nb_tx_to_clean;
577 /* Determine the last descriptor needing to be cleaned */
578 desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->tx_rs_thresh);
579 if (desc_to_clean_to >= nb_tx_desc)
580 desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc);
582 /* Check to make sure the last descriptor to clean is done */
583 desc_to_clean_to = sw_ring[desc_to_clean_to].last_id;
584 status = txr[desc_to_clean_to].wb.status;
585 if (!(status & rte_cpu_to_le_32(IXGBE_TXD_STAT_DD))) {
587 "TX descriptor %4u is not done"
588 "(port=%d queue=%d)",
590 txq->port_id, txq->queue_id);
591 /* Failed to clean any descriptors, better luck next time */
595 /* Figure out how many descriptors will be cleaned */
596 if (last_desc_cleaned > desc_to_clean_to)
597 nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) +
600 nb_tx_to_clean = (uint16_t)(desc_to_clean_to -
604 "Cleaning %4u TX descriptors: %4u to %4u "
605 "(port=%d queue=%d)",
606 nb_tx_to_clean, last_desc_cleaned, desc_to_clean_to,
607 txq->port_id, txq->queue_id);
610 * The last descriptor to clean is done, so that means all the
611 * descriptors from the last descriptor that was cleaned
612 * up to the last descriptor with the RS bit set
613 * are done. Only reset the threshold descriptor.
615 txr[desc_to_clean_to].wb.status = 0;
617 /* Update the txq to reflect the last descriptor that was cleaned */
618 txq->last_desc_cleaned = desc_to_clean_to;
619 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + nb_tx_to_clean);
626 ixgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
629 struct ixgbe_tx_queue *txq;
630 struct ixgbe_tx_entry *sw_ring;
631 struct ixgbe_tx_entry *txe, *txn;
632 volatile union ixgbe_adv_tx_desc *txr;
633 volatile union ixgbe_adv_tx_desc *txd, *txp;
634 struct rte_mbuf *tx_pkt;
635 struct rte_mbuf *m_seg;
636 uint64_t buf_dma_addr;
637 uint32_t olinfo_status;
638 uint32_t cmd_type_len;
649 union ixgbe_tx_offload tx_offload;
650 #ifdef RTE_LIB_SECURITY
654 tx_offload.data[0] = 0;
655 tx_offload.data[1] = 0;
657 sw_ring = txq->sw_ring;
659 tx_id = txq->tx_tail;
660 txe = &sw_ring[tx_id];
663 /* Determine if the descriptor ring needs to be cleaned. */
664 if (txq->nb_tx_free < txq->tx_free_thresh)
665 ixgbe_xmit_cleanup(txq);
667 rte_prefetch0(&txe->mbuf->pool);
670 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
673 pkt_len = tx_pkt->pkt_len;
676 * Determine how many (if any) context descriptors
677 * are needed for offload functionality.
679 ol_flags = tx_pkt->ol_flags;
680 #ifdef RTE_LIB_SECURITY
681 use_ipsec = txq->using_ipsec && (ol_flags & PKT_TX_SEC_OFFLOAD);
684 /* If hardware offload required */
685 tx_ol_req = ol_flags & IXGBE_TX_OFFLOAD_MASK;
687 tx_offload.l2_len = tx_pkt->l2_len;
688 tx_offload.l3_len = tx_pkt->l3_len;
689 tx_offload.l4_len = tx_pkt->l4_len;
690 tx_offload.vlan_tci = tx_pkt->vlan_tci;
691 tx_offload.tso_segsz = tx_pkt->tso_segsz;
692 tx_offload.outer_l2_len = tx_pkt->outer_l2_len;
693 tx_offload.outer_l3_len = tx_pkt->outer_l3_len;
694 #ifdef RTE_LIB_SECURITY
696 union ixgbe_crypto_tx_desc_md *ipsec_mdata =
697 (union ixgbe_crypto_tx_desc_md *)
698 rte_security_dynfield(tx_pkt);
699 tx_offload.sa_idx = ipsec_mdata->sa_idx;
700 tx_offload.sec_pad_len = ipsec_mdata->pad_len;
704 /* If new context need be built or reuse the exist ctx. */
705 ctx = what_advctx_update(txq, tx_ol_req,
707 /* Only allocate context descriptor if required*/
708 new_ctx = (ctx == IXGBE_CTX_NUM);
713 * Keep track of how many descriptors are used this loop
714 * This will always be the number of segments + the number of
715 * Context descriptors required to transmit the packet
717 nb_used = (uint16_t)(tx_pkt->nb_segs + new_ctx);
720 nb_used + txq->nb_tx_used >= txq->tx_rs_thresh)
721 /* set RS on the previous packet in the burst */
722 txp->read.cmd_type_len |=
723 rte_cpu_to_le_32(IXGBE_TXD_CMD_RS);
726 * The number of descriptors that must be allocated for a
727 * packet is the number of segments of that packet, plus 1
728 * Context Descriptor for the hardware offload, if any.
729 * Determine the last TX descriptor to allocate in the TX ring
730 * for the packet, starting from the current position (tx_id)
733 tx_last = (uint16_t) (tx_id + nb_used - 1);
736 if (tx_last >= txq->nb_tx_desc)
737 tx_last = (uint16_t) (tx_last - txq->nb_tx_desc);
739 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u pktlen=%u"
740 " tx_first=%u tx_last=%u",
741 (unsigned) txq->port_id,
742 (unsigned) txq->queue_id,
748 * Make sure there are enough TX descriptors available to
749 * transmit the entire packet.
750 * nb_used better be less than or equal to txq->tx_rs_thresh
752 if (nb_used > txq->nb_tx_free) {
754 "Not enough free TX descriptors "
755 "nb_used=%4u nb_free=%4u "
756 "(port=%d queue=%d)",
757 nb_used, txq->nb_tx_free,
758 txq->port_id, txq->queue_id);
760 if (ixgbe_xmit_cleanup(txq) != 0) {
761 /* Could not clean any descriptors */
767 /* nb_used better be <= txq->tx_rs_thresh */
768 if (unlikely(nb_used > txq->tx_rs_thresh)) {
770 "The number of descriptors needed to "
771 "transmit the packet exceeds the "
772 "RS bit threshold. This will impact "
774 "nb_used=%4u nb_free=%4u "
776 "(port=%d queue=%d)",
777 nb_used, txq->nb_tx_free,
779 txq->port_id, txq->queue_id);
781 * Loop here until there are enough TX
782 * descriptors or until the ring cannot be
785 while (nb_used > txq->nb_tx_free) {
786 if (ixgbe_xmit_cleanup(txq) != 0) {
788 * Could not clean any
800 * By now there are enough free TX descriptors to transmit
805 * Set common flags of all TX Data Descriptors.
807 * The following bits must be set in all Data Descriptors:
808 * - IXGBE_ADVTXD_DTYP_DATA
809 * - IXGBE_ADVTXD_DCMD_DEXT
811 * The following bits must be set in the first Data Descriptor
812 * and are ignored in the other ones:
813 * - IXGBE_ADVTXD_DCMD_IFCS
814 * - IXGBE_ADVTXD_MAC_1588
815 * - IXGBE_ADVTXD_DCMD_VLE
817 * The following bits must only be set in the last Data
819 * - IXGBE_TXD_CMD_EOP
821 * The following bits can be set in any Data Descriptor, but
822 * are only set in the last Data Descriptor:
825 cmd_type_len = IXGBE_ADVTXD_DTYP_DATA |
826 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT;
828 #ifdef RTE_LIBRTE_IEEE1588
829 if (ol_flags & PKT_TX_IEEE1588_TMST)
830 cmd_type_len |= IXGBE_ADVTXD_MAC_1588;
836 if (ol_flags & PKT_TX_TCP_SEG) {
837 /* when TSO is on, paylen in descriptor is the
838 * not the packet len but the tcp payload len */
839 pkt_len -= (tx_offload.l2_len +
840 tx_offload.l3_len + tx_offload.l4_len);
844 * Setup the TX Advanced Context Descriptor if required
847 volatile struct ixgbe_adv_tx_context_desc *
850 ctx_txd = (volatile struct
851 ixgbe_adv_tx_context_desc *)
854 txn = &sw_ring[txe->next_id];
855 rte_prefetch0(&txn->mbuf->pool);
857 if (txe->mbuf != NULL) {
858 rte_pktmbuf_free_seg(txe->mbuf);
862 ixgbe_set_xmit_ctx(txq, ctx_txd, tx_ol_req,
864 rte_security_dynfield(tx_pkt));
866 txe->last_id = tx_last;
867 tx_id = txe->next_id;
872 * Setup the TX Advanced Data Descriptor,
873 * This path will go through
874 * whatever new/reuse the context descriptor
876 cmd_type_len |= tx_desc_ol_flags_to_cmdtype(ol_flags);
877 olinfo_status |= tx_desc_cksum_flags_to_olinfo(ol_flags);
878 olinfo_status |= ctx << IXGBE_ADVTXD_IDX_SHIFT;
881 olinfo_status |= (pkt_len << IXGBE_ADVTXD_PAYLEN_SHIFT);
882 #ifdef RTE_LIB_SECURITY
884 olinfo_status |= IXGBE_ADVTXD_POPTS_IPSEC;
890 txn = &sw_ring[txe->next_id];
891 rte_prefetch0(&txn->mbuf->pool);
893 if (txe->mbuf != NULL)
894 rte_pktmbuf_free_seg(txe->mbuf);
898 * Set up Transmit Data Descriptor.
900 slen = m_seg->data_len;
901 buf_dma_addr = rte_mbuf_data_iova(m_seg);
902 txd->read.buffer_addr =
903 rte_cpu_to_le_64(buf_dma_addr);
904 txd->read.cmd_type_len =
905 rte_cpu_to_le_32(cmd_type_len | slen);
906 txd->read.olinfo_status =
907 rte_cpu_to_le_32(olinfo_status);
908 txe->last_id = tx_last;
909 tx_id = txe->next_id;
912 } while (m_seg != NULL);
915 * The last packet data descriptor needs End Of Packet (EOP)
917 cmd_type_len |= IXGBE_TXD_CMD_EOP;
918 txq->nb_tx_used = (uint16_t)(txq->nb_tx_used + nb_used);
919 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_used);
921 /* Set RS bit only on threshold packets' last descriptor */
922 if (txq->nb_tx_used >= txq->tx_rs_thresh) {
924 "Setting RS bit on TXD id="
925 "%4u (port=%d queue=%d)",
926 tx_last, txq->port_id, txq->queue_id);
928 cmd_type_len |= IXGBE_TXD_CMD_RS;
930 /* Update txq RS bit counters */
936 txd->read.cmd_type_len |= rte_cpu_to_le_32(cmd_type_len);
940 /* set RS on last packet in the burst */
942 txp->read.cmd_type_len |= rte_cpu_to_le_32(IXGBE_TXD_CMD_RS);
947 * Set the Transmit Descriptor Tail (TDT)
949 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u",
950 (unsigned) txq->port_id, (unsigned) txq->queue_id,
951 (unsigned) tx_id, (unsigned) nb_tx);
952 IXGBE_PCI_REG_WC_WRITE_RELAXED(txq->tdt_reg_addr, tx_id);
953 txq->tx_tail = tx_id;
958 /*********************************************************************
962 **********************************************************************/
964 ixgbe_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
969 struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue;
971 for (i = 0; i < nb_pkts; i++) {
973 ol_flags = m->ol_flags;
976 * Check if packet meets requirements for number of segments
978 * NOTE: for ixgbe it's always (40 - WTHRESH) for both TSO and
982 if (m->nb_segs > IXGBE_TX_MAX_SEG - txq->wthresh) {
987 if (ol_flags & IXGBE_TX_OFFLOAD_NOTSUP_MASK) {
992 /* check the size of packet */
993 if (m->pkt_len < IXGBE_TX_MIN_PKT_LEN) {
998 #ifdef RTE_ETHDEV_DEBUG_TX
999 ret = rte_validate_tx_offload(m);
1005 ret = rte_net_intel_cksum_prepare(m);
1015 /*********************************************************************
1019 **********************************************************************/
1021 #define IXGBE_PACKET_TYPE_ETHER 0X00
1022 #define IXGBE_PACKET_TYPE_IPV4 0X01
1023 #define IXGBE_PACKET_TYPE_IPV4_TCP 0X11
1024 #define IXGBE_PACKET_TYPE_IPV4_UDP 0X21
1025 #define IXGBE_PACKET_TYPE_IPV4_SCTP 0X41
1026 #define IXGBE_PACKET_TYPE_IPV4_EXT 0X03
1027 #define IXGBE_PACKET_TYPE_IPV4_EXT_TCP 0X13
1028 #define IXGBE_PACKET_TYPE_IPV4_EXT_UDP 0X23
1029 #define IXGBE_PACKET_TYPE_IPV4_EXT_SCTP 0X43
1030 #define IXGBE_PACKET_TYPE_IPV6 0X04
1031 #define IXGBE_PACKET_TYPE_IPV6_TCP 0X14
1032 #define IXGBE_PACKET_TYPE_IPV6_UDP 0X24
1033 #define IXGBE_PACKET_TYPE_IPV6_SCTP 0X44
1034 #define IXGBE_PACKET_TYPE_IPV6_EXT 0X0C
1035 #define IXGBE_PACKET_TYPE_IPV6_EXT_TCP 0X1C
1036 #define IXGBE_PACKET_TYPE_IPV6_EXT_UDP 0X2C
1037 #define IXGBE_PACKET_TYPE_IPV6_EXT_SCTP 0X4C
1038 #define IXGBE_PACKET_TYPE_IPV4_IPV6 0X05
1039 #define IXGBE_PACKET_TYPE_IPV4_IPV6_TCP 0X15
1040 #define IXGBE_PACKET_TYPE_IPV4_IPV6_UDP 0X25
1041 #define IXGBE_PACKET_TYPE_IPV4_IPV6_SCTP 0X45
1042 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6 0X07
1043 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_TCP 0X17
1044 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_UDP 0X27
1045 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_SCTP 0X47
1046 #define IXGBE_PACKET_TYPE_IPV4_IPV6_EXT 0X0D
1047 #define IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_TCP 0X1D
1048 #define IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_UDP 0X2D
1049 #define IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_SCTP 0X4D
1050 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT 0X0F
1051 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_TCP 0X1F
1052 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_UDP 0X2F
1053 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_SCTP 0X4F
1055 #define IXGBE_PACKET_TYPE_NVGRE 0X00
1056 #define IXGBE_PACKET_TYPE_NVGRE_IPV4 0X01
1057 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_TCP 0X11
1058 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_UDP 0X21
1059 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_SCTP 0X41
1060 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT 0X03
1061 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_TCP 0X13
1062 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_UDP 0X23
1063 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_SCTP 0X43
1064 #define IXGBE_PACKET_TYPE_NVGRE_IPV6 0X04
1065 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_TCP 0X14
1066 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_UDP 0X24
1067 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_SCTP 0X44
1068 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT 0X0C
1069 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_TCP 0X1C
1070 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_UDP 0X2C
1071 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_SCTP 0X4C
1072 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6 0X05
1073 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_TCP 0X15
1074 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_UDP 0X25
1075 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT 0X0D
1076 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_TCP 0X1D
1077 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_UDP 0X2D
1079 #define IXGBE_PACKET_TYPE_VXLAN 0X80
1080 #define IXGBE_PACKET_TYPE_VXLAN_IPV4 0X81
1081 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_TCP 0x91
1082 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_UDP 0xA1
1083 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_SCTP 0xC1
1084 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT 0x83
1085 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_TCP 0X93
1086 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_UDP 0XA3
1087 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_SCTP 0XC3
1088 #define IXGBE_PACKET_TYPE_VXLAN_IPV6 0X84
1089 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_TCP 0X94
1090 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_UDP 0XA4
1091 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_SCTP 0XC4
1092 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT 0X8C
1093 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_TCP 0X9C
1094 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_UDP 0XAC
1095 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_SCTP 0XCC
1096 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6 0X85
1097 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_TCP 0X95
1098 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_UDP 0XA5
1099 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT 0X8D
1100 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_TCP 0X9D
1101 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_UDP 0XAD
1104 * Use 2 different table for normal packet and tunnel packet
1105 * to save the space.
1108 ptype_table[IXGBE_PACKET_TYPE_MAX] __rte_cache_aligned = {
1109 [IXGBE_PACKET_TYPE_ETHER] = RTE_PTYPE_L2_ETHER,
1110 [IXGBE_PACKET_TYPE_IPV4] = RTE_PTYPE_L2_ETHER |
1112 [IXGBE_PACKET_TYPE_IPV4_TCP] = RTE_PTYPE_L2_ETHER |
1113 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP,
1114 [IXGBE_PACKET_TYPE_IPV4_UDP] = RTE_PTYPE_L2_ETHER |
1115 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP,
1116 [IXGBE_PACKET_TYPE_IPV4_SCTP] = RTE_PTYPE_L2_ETHER |
1117 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_SCTP,
1118 [IXGBE_PACKET_TYPE_IPV4_EXT] = RTE_PTYPE_L2_ETHER |
1119 RTE_PTYPE_L3_IPV4_EXT,
1120 [IXGBE_PACKET_TYPE_IPV4_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1121 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_TCP,
1122 [IXGBE_PACKET_TYPE_IPV4_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1123 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_UDP,
1124 [IXGBE_PACKET_TYPE_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1125 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_SCTP,
1126 [IXGBE_PACKET_TYPE_IPV6] = RTE_PTYPE_L2_ETHER |
1128 [IXGBE_PACKET_TYPE_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1129 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP,
1130 [IXGBE_PACKET_TYPE_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1131 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP,
1132 [IXGBE_PACKET_TYPE_IPV6_SCTP] = RTE_PTYPE_L2_ETHER |
1133 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_SCTP,
1134 [IXGBE_PACKET_TYPE_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1135 RTE_PTYPE_L3_IPV6_EXT,
1136 [IXGBE_PACKET_TYPE_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1137 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_TCP,
1138 [IXGBE_PACKET_TYPE_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1139 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_UDP,
1140 [IXGBE_PACKET_TYPE_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1141 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_SCTP,
1142 [IXGBE_PACKET_TYPE_IPV4_IPV6] = RTE_PTYPE_L2_ETHER |
1143 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1144 RTE_PTYPE_INNER_L3_IPV6,
1145 [IXGBE_PACKET_TYPE_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1146 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1147 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP,
1148 [IXGBE_PACKET_TYPE_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1149 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1150 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP,
1151 [IXGBE_PACKET_TYPE_IPV4_IPV6_SCTP] = RTE_PTYPE_L2_ETHER |
1152 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1153 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_SCTP,
1154 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6] = RTE_PTYPE_L2_ETHER |
1155 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1156 RTE_PTYPE_INNER_L3_IPV6,
1157 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1158 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1159 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP,
1160 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1161 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1162 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP,
1163 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_SCTP] = RTE_PTYPE_L2_ETHER |
1164 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1165 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_SCTP,
1166 [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1167 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1168 RTE_PTYPE_INNER_L3_IPV6_EXT,
1169 [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1170 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1171 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP,
1172 [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1173 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1174 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP,
1175 [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1176 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1177 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_SCTP,
1178 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1179 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1180 RTE_PTYPE_INNER_L3_IPV6_EXT,
1181 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1182 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1183 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP,
1184 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1185 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1186 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP,
1187 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_SCTP] =
1188 RTE_PTYPE_L2_ETHER |
1189 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1190 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_SCTP,
1194 ptype_table_tn[IXGBE_PACKET_TYPE_TN_MAX] __rte_cache_aligned = {
1195 [IXGBE_PACKET_TYPE_NVGRE] = RTE_PTYPE_L2_ETHER |
1196 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1197 RTE_PTYPE_INNER_L2_ETHER,
1198 [IXGBE_PACKET_TYPE_NVGRE_IPV4] = RTE_PTYPE_L2_ETHER |
1199 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1200 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1201 [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT] = RTE_PTYPE_L2_ETHER |
1202 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1203 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT,
1204 [IXGBE_PACKET_TYPE_NVGRE_IPV6] = RTE_PTYPE_L2_ETHER |
1205 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1206 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6,
1207 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6] = RTE_PTYPE_L2_ETHER |
1208 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1209 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1210 [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1211 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1212 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT,
1213 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1214 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1215 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1216 [IXGBE_PACKET_TYPE_NVGRE_IPV4_TCP] = RTE_PTYPE_L2_ETHER |
1217 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1218 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4 |
1219 RTE_PTYPE_INNER_L4_TCP,
1220 [IXGBE_PACKET_TYPE_NVGRE_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1221 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1222 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6 |
1223 RTE_PTYPE_INNER_L4_TCP,
1224 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1225 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1226 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1227 [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1228 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1229 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT |
1230 RTE_PTYPE_INNER_L4_TCP,
1231 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_TCP] =
1232 RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
1233 RTE_PTYPE_TUNNEL_GRE | RTE_PTYPE_INNER_L2_ETHER |
1234 RTE_PTYPE_INNER_L3_IPV4,
1235 [IXGBE_PACKET_TYPE_NVGRE_IPV4_UDP] = RTE_PTYPE_L2_ETHER |
1236 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1237 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4 |
1238 RTE_PTYPE_INNER_L4_UDP,
1239 [IXGBE_PACKET_TYPE_NVGRE_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1240 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1241 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6 |
1242 RTE_PTYPE_INNER_L4_UDP,
1243 [IXGBE_PACKET_TYPE_NVGRE_IPV6_SCTP] = RTE_PTYPE_L2_ETHER |
1244 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1245 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6 |
1246 RTE_PTYPE_INNER_L4_SCTP,
1247 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1248 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1249 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1250 [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1251 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1252 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT |
1253 RTE_PTYPE_INNER_L4_UDP,
1254 [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1255 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1256 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT |
1257 RTE_PTYPE_INNER_L4_SCTP,
1258 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_UDP] =
1259 RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
1260 RTE_PTYPE_TUNNEL_GRE | RTE_PTYPE_INNER_L2_ETHER |
1261 RTE_PTYPE_INNER_L3_IPV4,
1262 [IXGBE_PACKET_TYPE_NVGRE_IPV4_SCTP] = RTE_PTYPE_L2_ETHER |
1263 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1264 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4 |
1265 RTE_PTYPE_INNER_L4_SCTP,
1266 [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1267 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1268 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT |
1269 RTE_PTYPE_INNER_L4_SCTP,
1270 [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1271 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1272 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT |
1273 RTE_PTYPE_INNER_L4_TCP,
1274 [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1275 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1276 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT |
1277 RTE_PTYPE_INNER_L4_UDP,
1279 [IXGBE_PACKET_TYPE_VXLAN] = RTE_PTYPE_L2_ETHER |
1280 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1281 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER,
1282 [IXGBE_PACKET_TYPE_VXLAN_IPV4] = RTE_PTYPE_L2_ETHER |
1283 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1284 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1285 RTE_PTYPE_INNER_L3_IPV4,
1286 [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT] = RTE_PTYPE_L2_ETHER |
1287 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1288 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1289 RTE_PTYPE_INNER_L3_IPV4_EXT,
1290 [IXGBE_PACKET_TYPE_VXLAN_IPV6] = RTE_PTYPE_L2_ETHER |
1291 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1292 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1293 RTE_PTYPE_INNER_L3_IPV6,
1294 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6] = RTE_PTYPE_L2_ETHER |
1295 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1296 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1297 RTE_PTYPE_INNER_L3_IPV4,
1298 [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1299 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1300 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1301 RTE_PTYPE_INNER_L3_IPV6_EXT,
1302 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1303 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1304 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1305 RTE_PTYPE_INNER_L3_IPV4,
1306 [IXGBE_PACKET_TYPE_VXLAN_IPV4_TCP] = RTE_PTYPE_L2_ETHER |
1307 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1308 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1309 RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L4_TCP,
1310 [IXGBE_PACKET_TYPE_VXLAN_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1311 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1312 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1313 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP,
1314 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1315 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1316 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1317 RTE_PTYPE_INNER_L3_IPV4,
1318 [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1319 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1320 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1321 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP,
1322 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_TCP] =
1323 RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
1324 RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_VXLAN |
1325 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1326 [IXGBE_PACKET_TYPE_VXLAN_IPV4_UDP] = RTE_PTYPE_L2_ETHER |
1327 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1328 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1329 RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L4_UDP,
1330 [IXGBE_PACKET_TYPE_VXLAN_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1331 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1332 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1333 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP,
1334 [IXGBE_PACKET_TYPE_VXLAN_IPV6_SCTP] = RTE_PTYPE_L2_ETHER |
1335 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1336 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1337 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_SCTP,
1338 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1339 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1340 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1341 RTE_PTYPE_INNER_L3_IPV4,
1342 [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1343 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1344 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1345 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP,
1346 [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1347 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1348 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1349 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_SCTP,
1350 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_UDP] =
1351 RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
1352 RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_VXLAN |
1353 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1354 [IXGBE_PACKET_TYPE_VXLAN_IPV4_SCTP] = RTE_PTYPE_L2_ETHER |
1355 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1356 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1357 RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L4_SCTP,
1358 [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1359 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1360 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1361 RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_SCTP,
1362 [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1363 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1364 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1365 RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_TCP,
1366 [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1367 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1368 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1369 RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_UDP,
1373 ixgbe_get_monitor_addr(void *rx_queue, struct rte_power_monitor_cond *pmc)
1375 volatile union ixgbe_adv_rx_desc *rxdp;
1376 struct ixgbe_rx_queue *rxq = rx_queue;
1379 desc = rxq->rx_tail;
1380 rxdp = &rxq->rx_ring[desc];
1381 /* watch for changes in status bit */
1382 pmc->addr = &rxdp->wb.upper.status_error;
1385 * we expect the DD bit to be set to 1 if this descriptor was already
1388 pmc->val = rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD);
1389 pmc->mask = rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD);
1391 /* the registers are 32-bit */
1392 pmc->size = sizeof(uint32_t);
1397 /* @note: fix ixgbe_dev_supported_ptypes_get() if any change here. */
1398 static inline uint32_t
1399 ixgbe_rxd_pkt_info_to_pkt_type(uint32_t pkt_info, uint16_t ptype_mask)
1402 if (unlikely(pkt_info & IXGBE_RXDADV_PKTTYPE_ETQF))
1403 return RTE_PTYPE_UNKNOWN;
1405 pkt_info = (pkt_info >> IXGBE_PACKET_TYPE_SHIFT) & ptype_mask;
1407 /* For tunnel packet */
1408 if (pkt_info & IXGBE_PACKET_TYPE_TUNNEL_BIT) {
1409 /* Remove the tunnel bit to save the space. */
1410 pkt_info &= IXGBE_PACKET_TYPE_MASK_TUNNEL;
1411 return ptype_table_tn[pkt_info];
1415 * For x550, if it's not tunnel,
1416 * tunnel type bit should be set to 0.
1417 * Reuse 82599's mask.
1419 pkt_info &= IXGBE_PACKET_TYPE_MASK_82599;
1421 return ptype_table[pkt_info];
1424 static inline uint64_t
1425 ixgbe_rxd_pkt_info_to_pkt_flags(uint16_t pkt_info)
1427 static uint64_t ip_rss_types_map[16] __rte_cache_aligned = {
1428 0, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH,
1429 0, PKT_RX_RSS_HASH, 0, PKT_RX_RSS_HASH,
1430 PKT_RX_RSS_HASH, 0, 0, 0,
1431 0, 0, 0, PKT_RX_FDIR,
1433 #ifdef RTE_LIBRTE_IEEE1588
1434 static uint64_t ip_pkt_etqf_map[8] = {
1435 0, 0, 0, PKT_RX_IEEE1588_PTP,
1439 if (likely(pkt_info & IXGBE_RXDADV_PKTTYPE_ETQF))
1440 return ip_pkt_etqf_map[(pkt_info >> 4) & 0X07] |
1441 ip_rss_types_map[pkt_info & 0XF];
1443 return ip_rss_types_map[pkt_info & 0XF];
1445 return ip_rss_types_map[pkt_info & 0XF];
1449 static inline uint64_t
1450 rx_desc_status_to_pkt_flags(uint32_t rx_status, uint64_t vlan_flags)
1455 * Check if VLAN present only.
1456 * Do not check whether L3/L4 rx checksum done by NIC or not,
1457 * That can be found from rte_eth_rxmode.offloads flag
1459 pkt_flags = (rx_status & IXGBE_RXD_STAT_VP) ? vlan_flags : 0;
1461 #ifdef RTE_LIBRTE_IEEE1588
1462 if (rx_status & IXGBE_RXD_STAT_TMST)
1463 pkt_flags = pkt_flags | PKT_RX_IEEE1588_TMST;
1468 static inline uint64_t
1469 rx_desc_error_to_pkt_flags(uint32_t rx_status, uint16_t pkt_info,
1470 uint8_t rx_udp_csum_zero_err)
1475 * Bit 31: IPE, IPv4 checksum error
1476 * Bit 30: L4I, L4I integrity error
1478 static uint64_t error_to_pkt_flags_map[4] = {
1479 PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD,
1480 PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD,
1481 PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_GOOD,
1482 PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD
1484 pkt_flags = error_to_pkt_flags_map[(rx_status >>
1485 IXGBE_RXDADV_ERR_CKSUM_BIT) & IXGBE_RXDADV_ERR_CKSUM_MSK];
1487 /* Mask out the bad UDP checksum error if the hardware has UDP zero
1488 * checksum error issue, so that the software application will then
1489 * have to recompute the checksum itself if needed.
1491 if ((rx_status & IXGBE_RXDADV_ERR_TCPE) &&
1492 (pkt_info & IXGBE_RXDADV_PKTTYPE_UDP) &&
1493 rx_udp_csum_zero_err)
1494 pkt_flags &= ~PKT_RX_L4_CKSUM_BAD;
1496 if ((rx_status & IXGBE_RXD_STAT_OUTERIPCS) &&
1497 (rx_status & IXGBE_RXDADV_ERR_OUTERIPER)) {
1498 pkt_flags |= PKT_RX_OUTER_IP_CKSUM_BAD;
1501 #ifdef RTE_LIB_SECURITY
1502 if (rx_status & IXGBE_RXD_STAT_SECP) {
1503 pkt_flags |= PKT_RX_SEC_OFFLOAD;
1504 if (rx_status & IXGBE_RXDADV_LNKSEC_ERROR_BAD_SIG)
1505 pkt_flags |= PKT_RX_SEC_OFFLOAD_FAILED;
1513 * LOOK_AHEAD defines how many desc statuses to check beyond the
1514 * current descriptor.
1515 * It must be a pound define for optimal performance.
1516 * Do not change the value of LOOK_AHEAD, as the ixgbe_rx_scan_hw_ring
1517 * function only works with LOOK_AHEAD=8.
1519 #define LOOK_AHEAD 8
1520 #if (LOOK_AHEAD != 8)
1521 #error "PMD IXGBE: LOOK_AHEAD must be 8\n"
1524 ixgbe_rx_scan_hw_ring(struct ixgbe_rx_queue *rxq)
1526 volatile union ixgbe_adv_rx_desc *rxdp;
1527 struct ixgbe_rx_entry *rxep;
1528 struct rte_mbuf *mb;
1532 uint32_t s[LOOK_AHEAD];
1533 uint32_t pkt_info[LOOK_AHEAD];
1534 int i, j, nb_rx = 0;
1536 uint64_t vlan_flags = rxq->vlan_flags;
1538 /* get references to current descriptor and S/W ring entry */
1539 rxdp = &rxq->rx_ring[rxq->rx_tail];
1540 rxep = &rxq->sw_ring[rxq->rx_tail];
1542 status = rxdp->wb.upper.status_error;
1543 /* check to make sure there is at least 1 packet to receive */
1544 if (!(status & rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD)))
1548 * Scan LOOK_AHEAD descriptors at a time to determine which descriptors
1549 * reference packets that are ready to be received.
1551 for (i = 0; i < RTE_PMD_IXGBE_RX_MAX_BURST;
1552 i += LOOK_AHEAD, rxdp += LOOK_AHEAD, rxep += LOOK_AHEAD) {
1553 /* Read desc statuses backwards to avoid race condition */
1554 for (j = 0; j < LOOK_AHEAD; j++)
1555 s[j] = rte_le_to_cpu_32(rxdp[j].wb.upper.status_error);
1559 /* Compute how many status bits were set */
1560 for (nb_dd = 0; nb_dd < LOOK_AHEAD &&
1561 (s[nb_dd] & IXGBE_RXDADV_STAT_DD); nb_dd++)
1564 for (j = 0; j < nb_dd; j++)
1565 pkt_info[j] = rte_le_to_cpu_32(rxdp[j].wb.lower.
1570 /* Translate descriptor info to mbuf format */
1571 for (j = 0; j < nb_dd; ++j) {
1573 pkt_len = rte_le_to_cpu_16(rxdp[j].wb.upper.length) -
1575 mb->data_len = pkt_len;
1576 mb->pkt_len = pkt_len;
1577 mb->vlan_tci = rte_le_to_cpu_16(rxdp[j].wb.upper.vlan);
1579 /* convert descriptor fields to rte mbuf flags */
1580 pkt_flags = rx_desc_status_to_pkt_flags(s[j],
1582 pkt_flags |= rx_desc_error_to_pkt_flags(s[j],
1583 (uint16_t)pkt_info[j],
1584 rxq->rx_udp_csum_zero_err);
1585 pkt_flags |= ixgbe_rxd_pkt_info_to_pkt_flags
1586 ((uint16_t)pkt_info[j]);
1587 mb->ol_flags = pkt_flags;
1589 ixgbe_rxd_pkt_info_to_pkt_type
1590 (pkt_info[j], rxq->pkt_type_mask);
1592 if (likely(pkt_flags & PKT_RX_RSS_HASH))
1593 mb->hash.rss = rte_le_to_cpu_32(
1594 rxdp[j].wb.lower.hi_dword.rss);
1595 else if (pkt_flags & PKT_RX_FDIR) {
1596 mb->hash.fdir.hash = rte_le_to_cpu_16(
1597 rxdp[j].wb.lower.hi_dword.csum_ip.csum) &
1598 IXGBE_ATR_HASH_MASK;
1599 mb->hash.fdir.id = rte_le_to_cpu_16(
1600 rxdp[j].wb.lower.hi_dword.csum_ip.ip_id);
1604 /* Move mbuf pointers from the S/W ring to the stage */
1605 for (j = 0; j < LOOK_AHEAD; ++j) {
1606 rxq->rx_stage[i + j] = rxep[j].mbuf;
1609 /* stop if all requested packets could not be received */
1610 if (nb_dd != LOOK_AHEAD)
1614 /* clear software ring entries so we can cleanup correctly */
1615 for (i = 0; i < nb_rx; ++i) {
1616 rxq->sw_ring[rxq->rx_tail + i].mbuf = NULL;
1624 ixgbe_rx_alloc_bufs(struct ixgbe_rx_queue *rxq, bool reset_mbuf)
1626 volatile union ixgbe_adv_rx_desc *rxdp;
1627 struct ixgbe_rx_entry *rxep;
1628 struct rte_mbuf *mb;
1633 /* allocate buffers in bulk directly into the S/W ring */
1634 alloc_idx = rxq->rx_free_trigger - (rxq->rx_free_thresh - 1);
1635 rxep = &rxq->sw_ring[alloc_idx];
1636 diag = rte_mempool_get_bulk(rxq->mb_pool, (void *)rxep,
1637 rxq->rx_free_thresh);
1638 if (unlikely(diag != 0))
1641 rxdp = &rxq->rx_ring[alloc_idx];
1642 for (i = 0; i < rxq->rx_free_thresh; ++i) {
1643 /* populate the static rte mbuf fields */
1646 mb->port = rxq->port_id;
1649 rte_mbuf_refcnt_set(mb, 1);
1650 mb->data_off = RTE_PKTMBUF_HEADROOM;
1652 /* populate the descriptors */
1653 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mb));
1654 rxdp[i].read.hdr_addr = 0;
1655 rxdp[i].read.pkt_addr = dma_addr;
1658 /* update state of internal queue structure */
1659 rxq->rx_free_trigger = rxq->rx_free_trigger + rxq->rx_free_thresh;
1660 if (rxq->rx_free_trigger >= rxq->nb_rx_desc)
1661 rxq->rx_free_trigger = rxq->rx_free_thresh - 1;
1667 static inline uint16_t
1668 ixgbe_rx_fill_from_stage(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
1671 struct rte_mbuf **stage = &rxq->rx_stage[rxq->rx_next_avail];
1674 /* how many packets are ready to return? */
1675 nb_pkts = (uint16_t)RTE_MIN(nb_pkts, rxq->rx_nb_avail);
1677 /* copy mbuf pointers to the application's packet list */
1678 for (i = 0; i < nb_pkts; ++i)
1679 rx_pkts[i] = stage[i];
1681 /* update internal queue state */
1682 rxq->rx_nb_avail = (uint16_t)(rxq->rx_nb_avail - nb_pkts);
1683 rxq->rx_next_avail = (uint16_t)(rxq->rx_next_avail + nb_pkts);
1688 static inline uint16_t
1689 rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
1692 struct ixgbe_rx_queue *rxq = (struct ixgbe_rx_queue *)rx_queue;
1695 /* Any previously recv'd pkts will be returned from the Rx stage */
1696 if (rxq->rx_nb_avail)
1697 return ixgbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
1699 /* Scan the H/W ring for packets to receive */
1700 nb_rx = (uint16_t)ixgbe_rx_scan_hw_ring(rxq);
1702 /* update internal queue state */
1703 rxq->rx_next_avail = 0;
1704 rxq->rx_nb_avail = nb_rx;
1705 rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_rx);
1707 /* if required, allocate new buffers to replenish descriptors */
1708 if (rxq->rx_tail > rxq->rx_free_trigger) {
1709 uint16_t cur_free_trigger = rxq->rx_free_trigger;
1711 if (ixgbe_rx_alloc_bufs(rxq, true) != 0) {
1714 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u "
1715 "queue_id=%u", (unsigned) rxq->port_id,
1716 (unsigned) rxq->queue_id);
1718 rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed +=
1719 rxq->rx_free_thresh;
1722 * Need to rewind any previous receives if we cannot
1723 * allocate new buffers to replenish the old ones.
1725 rxq->rx_nb_avail = 0;
1726 rxq->rx_tail = (uint16_t)(rxq->rx_tail - nb_rx);
1727 for (i = 0, j = rxq->rx_tail; i < nb_rx; ++i, ++j)
1728 rxq->sw_ring[j].mbuf = rxq->rx_stage[i];
1733 /* update tail pointer */
1735 IXGBE_PCI_REG_WC_WRITE_RELAXED(rxq->rdt_reg_addr,
1739 if (rxq->rx_tail >= rxq->nb_rx_desc)
1742 /* received any packets this loop? */
1743 if (rxq->rx_nb_avail)
1744 return ixgbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
1749 /* split requests into chunks of size RTE_PMD_IXGBE_RX_MAX_BURST */
1751 ixgbe_recv_pkts_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
1756 if (unlikely(nb_pkts == 0))
1759 if (likely(nb_pkts <= RTE_PMD_IXGBE_RX_MAX_BURST))
1760 return rx_recv_pkts(rx_queue, rx_pkts, nb_pkts);
1762 /* request is relatively large, chunk it up */
1767 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_IXGBE_RX_MAX_BURST);
1768 ret = rx_recv_pkts(rx_queue, &rx_pkts[nb_rx], n);
1769 nb_rx = (uint16_t)(nb_rx + ret);
1770 nb_pkts = (uint16_t)(nb_pkts - ret);
1779 ixgbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
1782 struct ixgbe_rx_queue *rxq;
1783 volatile union ixgbe_adv_rx_desc *rx_ring;
1784 volatile union ixgbe_adv_rx_desc *rxdp;
1785 struct ixgbe_rx_entry *sw_ring;
1786 struct ixgbe_rx_entry *rxe;
1787 struct rte_mbuf *rxm;
1788 struct rte_mbuf *nmb;
1789 union ixgbe_adv_rx_desc rxd;
1798 uint64_t vlan_flags;
1803 rx_id = rxq->rx_tail;
1804 rx_ring = rxq->rx_ring;
1805 sw_ring = rxq->sw_ring;
1806 vlan_flags = rxq->vlan_flags;
1807 while (nb_rx < nb_pkts) {
1809 * The order of operations here is important as the DD status
1810 * bit must not be read after any other descriptor fields.
1811 * rx_ring and rxdp are pointing to volatile data so the order
1812 * of accesses cannot be reordered by the compiler. If they were
1813 * not volatile, they could be reordered which could lead to
1814 * using invalid descriptor fields when read from rxd.
1816 rxdp = &rx_ring[rx_id];
1817 staterr = rxdp->wb.upper.status_error;
1818 if (!(staterr & rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD)))
1825 * If the IXGBE_RXDADV_STAT_EOP flag is not set, the RX packet
1826 * is likely to be invalid and to be dropped by the various
1827 * validation checks performed by the network stack.
1829 * Allocate a new mbuf to replenish the RX ring descriptor.
1830 * If the allocation fails:
1831 * - arrange for that RX descriptor to be the first one
1832 * being parsed the next time the receive function is
1833 * invoked [on the same queue].
1835 * - Stop parsing the RX ring and return immediately.
1837 * This policy do not drop the packet received in the RX
1838 * descriptor for which the allocation of a new mbuf failed.
1839 * Thus, it allows that packet to be later retrieved if
1840 * mbuf have been freed in the mean time.
1841 * As a side effect, holding RX descriptors instead of
1842 * systematically giving them back to the NIC may lead to
1843 * RX ring exhaustion situations.
1844 * However, the NIC can gracefully prevent such situations
1845 * to happen by sending specific "back-pressure" flow control
1846 * frames to its peer(s).
1848 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u "
1849 "ext_err_stat=0x%08x pkt_len=%u",
1850 (unsigned) rxq->port_id, (unsigned) rxq->queue_id,
1851 (unsigned) rx_id, (unsigned) staterr,
1852 (unsigned) rte_le_to_cpu_16(rxd.wb.upper.length));
1854 nmb = rte_mbuf_raw_alloc(rxq->mb_pool);
1856 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u "
1857 "queue_id=%u", (unsigned) rxq->port_id,
1858 (unsigned) rxq->queue_id);
1859 rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed++;
1864 rxe = &sw_ring[rx_id];
1866 if (rx_id == rxq->nb_rx_desc)
1869 /* Prefetch next mbuf while processing current one. */
1870 rte_ixgbe_prefetch(sw_ring[rx_id].mbuf);
1873 * When next RX descriptor is on a cache-line boundary,
1874 * prefetch the next 4 RX descriptors and the next 8 pointers
1877 if ((rx_id & 0x3) == 0) {
1878 rte_ixgbe_prefetch(&rx_ring[rx_id]);
1879 rte_ixgbe_prefetch(&sw_ring[rx_id]);
1885 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
1886 rxdp->read.hdr_addr = 0;
1887 rxdp->read.pkt_addr = dma_addr;
1890 * Initialize the returned mbuf.
1891 * 1) setup generic mbuf fields:
1892 * - number of segments,
1895 * - RX port identifier.
1896 * 2) integrate hardware offload data, if any:
1897 * - RSS flag & hash,
1898 * - IP checksum flag,
1899 * - VLAN TCI, if any,
1902 pkt_len = (uint16_t) (rte_le_to_cpu_16(rxd.wb.upper.length) -
1904 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1905 rte_packet_prefetch((char *)rxm->buf_addr + rxm->data_off);
1908 rxm->pkt_len = pkt_len;
1909 rxm->data_len = pkt_len;
1910 rxm->port = rxq->port_id;
1912 pkt_info = rte_le_to_cpu_32(rxd.wb.lower.lo_dword.data);
1913 /* Only valid if PKT_RX_VLAN set in pkt_flags */
1914 rxm->vlan_tci = rte_le_to_cpu_16(rxd.wb.upper.vlan);
1916 pkt_flags = rx_desc_status_to_pkt_flags(staterr, vlan_flags);
1917 pkt_flags = pkt_flags |
1918 rx_desc_error_to_pkt_flags(staterr, (uint16_t)pkt_info,
1919 rxq->rx_udp_csum_zero_err);
1920 pkt_flags = pkt_flags |
1921 ixgbe_rxd_pkt_info_to_pkt_flags((uint16_t)pkt_info);
1922 rxm->ol_flags = pkt_flags;
1924 ixgbe_rxd_pkt_info_to_pkt_type(pkt_info,
1925 rxq->pkt_type_mask);
1927 if (likely(pkt_flags & PKT_RX_RSS_HASH))
1928 rxm->hash.rss = rte_le_to_cpu_32(
1929 rxd.wb.lower.hi_dword.rss);
1930 else if (pkt_flags & PKT_RX_FDIR) {
1931 rxm->hash.fdir.hash = rte_le_to_cpu_16(
1932 rxd.wb.lower.hi_dword.csum_ip.csum) &
1933 IXGBE_ATR_HASH_MASK;
1934 rxm->hash.fdir.id = rte_le_to_cpu_16(
1935 rxd.wb.lower.hi_dword.csum_ip.ip_id);
1938 * Store the mbuf address into the next entry of the array
1939 * of returned packets.
1941 rx_pkts[nb_rx++] = rxm;
1943 rxq->rx_tail = rx_id;
1946 * If the number of free RX descriptors is greater than the RX free
1947 * threshold of the queue, advance the Receive Descriptor Tail (RDT)
1949 * Update the RDT with the value of the last processed RX descriptor
1950 * minus 1, to guarantee that the RDT register is never equal to the
1951 * RDH register, which creates a "full" ring situtation from the
1952 * hardware point of view...
1954 nb_hold = (uint16_t) (nb_hold + rxq->nb_rx_hold);
1955 if (nb_hold > rxq->rx_free_thresh) {
1956 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u "
1957 "nb_hold=%u nb_rx=%u",
1958 (unsigned) rxq->port_id, (unsigned) rxq->queue_id,
1959 (unsigned) rx_id, (unsigned) nb_hold,
1961 rx_id = (uint16_t) ((rx_id == 0) ?
1962 (rxq->nb_rx_desc - 1) : (rx_id - 1));
1963 IXGBE_PCI_REG_WC_WRITE(rxq->rdt_reg_addr, rx_id);
1966 rxq->nb_rx_hold = nb_hold;
1971 * Detect an RSC descriptor.
1973 static inline uint32_t
1974 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
1976 return (rte_le_to_cpu_32(rx->wb.lower.lo_dword.data) &
1977 IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
1981 * ixgbe_fill_cluster_head_buf - fill the first mbuf of the returned packet
1983 * Fill the following info in the HEAD buffer of the Rx cluster:
1984 * - RX port identifier
1985 * - hardware offload data, if any:
1987 * - IP checksum flag
1988 * - VLAN TCI, if any
1990 * @head HEAD of the packet cluster
1991 * @desc HW descriptor to get data from
1992 * @rxq Pointer to the Rx queue
1995 ixgbe_fill_cluster_head_buf(
1996 struct rte_mbuf *head,
1997 union ixgbe_adv_rx_desc *desc,
1998 struct ixgbe_rx_queue *rxq,
2004 head->port = rxq->port_id;
2006 /* The vlan_tci field is only valid when PKT_RX_VLAN is
2007 * set in the pkt_flags field.
2009 head->vlan_tci = rte_le_to_cpu_16(desc->wb.upper.vlan);
2010 pkt_info = rte_le_to_cpu_32(desc->wb.lower.lo_dword.data);
2011 pkt_flags = rx_desc_status_to_pkt_flags(staterr, rxq->vlan_flags);
2012 pkt_flags |= rx_desc_error_to_pkt_flags(staterr, (uint16_t)pkt_info,
2013 rxq->rx_udp_csum_zero_err);
2014 pkt_flags |= ixgbe_rxd_pkt_info_to_pkt_flags((uint16_t)pkt_info);
2015 head->ol_flags = pkt_flags;
2017 ixgbe_rxd_pkt_info_to_pkt_type(pkt_info, rxq->pkt_type_mask);
2019 if (likely(pkt_flags & PKT_RX_RSS_HASH))
2020 head->hash.rss = rte_le_to_cpu_32(desc->wb.lower.hi_dword.rss);
2021 else if (pkt_flags & PKT_RX_FDIR) {
2022 head->hash.fdir.hash =
2023 rte_le_to_cpu_16(desc->wb.lower.hi_dword.csum_ip.csum)
2024 & IXGBE_ATR_HASH_MASK;
2025 head->hash.fdir.id =
2026 rte_le_to_cpu_16(desc->wb.lower.hi_dword.csum_ip.ip_id);
2031 * ixgbe_recv_pkts_lro - receive handler for and LRO case.
2033 * @rx_queue Rx queue handle
2034 * @rx_pkts table of received packets
2035 * @nb_pkts size of rx_pkts table
2036 * @bulk_alloc if TRUE bulk allocation is used for a HW ring refilling
2038 * Handles the Rx HW ring completions when RSC feature is configured. Uses an
2039 * additional ring of ixgbe_rsc_entry's that will hold the relevant RSC info.
2041 * We use the same logic as in Linux and in FreeBSD ixgbe drivers:
2042 * 1) When non-EOP RSC completion arrives:
2043 * a) Update the HEAD of the current RSC aggregation cluster with the new
2044 * segment's data length.
2045 * b) Set the "next" pointer of the current segment to point to the segment
2046 * at the NEXTP index.
2047 * c) Pass the HEAD of RSC aggregation cluster on to the next NEXTP entry
2048 * in the sw_rsc_ring.
2049 * 2) When EOP arrives we just update the cluster's total length and offload
2050 * flags and deliver the cluster up to the upper layers. In our case - put it
2051 * in the rx_pkts table.
2053 * Returns the number of received packets/clusters (according to the "bulk
2054 * receive" interface).
2056 static inline uint16_t
2057 ixgbe_recv_pkts_lro(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts,
2060 struct ixgbe_rx_queue *rxq = rx_queue;
2061 volatile union ixgbe_adv_rx_desc *rx_ring = rxq->rx_ring;
2062 struct ixgbe_rx_entry *sw_ring = rxq->sw_ring;
2063 struct ixgbe_scattered_rx_entry *sw_sc_ring = rxq->sw_sc_ring;
2064 uint16_t rx_id = rxq->rx_tail;
2066 uint16_t nb_hold = rxq->nb_rx_hold;
2067 uint16_t prev_id = rxq->rx_tail;
2069 while (nb_rx < nb_pkts) {
2071 struct ixgbe_rx_entry *rxe;
2072 struct ixgbe_scattered_rx_entry *sc_entry;
2073 struct ixgbe_scattered_rx_entry *next_sc_entry = NULL;
2074 struct ixgbe_rx_entry *next_rxe = NULL;
2075 struct rte_mbuf *first_seg;
2076 struct rte_mbuf *rxm;
2077 struct rte_mbuf *nmb = NULL;
2078 union ixgbe_adv_rx_desc rxd;
2081 volatile union ixgbe_adv_rx_desc *rxdp;
2086 * The code in this whole file uses the volatile pointer to
2087 * ensure the read ordering of the status and the rest of the
2088 * descriptor fields (on the compiler level only!!!). This is so
2089 * UGLY - why not to just use the compiler barrier instead? DPDK
2090 * even has the rte_compiler_barrier() for that.
2092 * But most importantly this is just wrong because this doesn't
2093 * ensure memory ordering in a general case at all. For
2094 * instance, DPDK is supposed to work on Power CPUs where
2095 * compiler barrier may just not be enough!
2097 * I tried to write only this function properly to have a
2098 * starting point (as a part of an LRO/RSC series) but the
2099 * compiler cursed at me when I tried to cast away the
2100 * "volatile" from rx_ring (yes, it's volatile too!!!). So, I'm
2101 * keeping it the way it is for now.
2103 * The code in this file is broken in so many other places and
2104 * will just not work on a big endian CPU anyway therefore the
2105 * lines below will have to be revisited together with the rest
2109 * - Get rid of "volatile" and let the compiler do its job.
2110 * - Use the proper memory barrier (rte_rmb()) to ensure the
2111 * memory ordering below.
2113 rxdp = &rx_ring[rx_id];
2114 staterr = rte_le_to_cpu_32(rxdp->wb.upper.status_error);
2116 if (!(staterr & IXGBE_RXDADV_STAT_DD))
2121 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u "
2122 "staterr=0x%x data_len=%u",
2123 rxq->port_id, rxq->queue_id, rx_id, staterr,
2124 rte_le_to_cpu_16(rxd.wb.upper.length));
2127 nmb = rte_mbuf_raw_alloc(rxq->mb_pool);
2129 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed "
2130 "port_id=%u queue_id=%u",
2131 rxq->port_id, rxq->queue_id);
2133 rte_eth_devices[rxq->port_id].data->
2134 rx_mbuf_alloc_failed++;
2137 } else if (nb_hold > rxq->rx_free_thresh) {
2138 uint16_t next_rdt = rxq->rx_free_trigger;
2140 if (!ixgbe_rx_alloc_bufs(rxq, false)) {
2142 IXGBE_PCI_REG_WC_WRITE_RELAXED(
2145 nb_hold -= rxq->rx_free_thresh;
2147 PMD_RX_LOG(DEBUG, "RX bulk alloc failed "
2148 "port_id=%u queue_id=%u",
2149 rxq->port_id, rxq->queue_id);
2151 rte_eth_devices[rxq->port_id].data->
2152 rx_mbuf_alloc_failed++;
2158 rxe = &sw_ring[rx_id];
2159 eop = staterr & IXGBE_RXDADV_STAT_EOP;
2161 next_id = rx_id + 1;
2162 if (next_id == rxq->nb_rx_desc)
2165 /* Prefetch next mbuf while processing current one. */
2166 rte_ixgbe_prefetch(sw_ring[next_id].mbuf);
2169 * When next RX descriptor is on a cache-line boundary,
2170 * prefetch the next 4 RX descriptors and the next 4 pointers
2173 if ((next_id & 0x3) == 0) {
2174 rte_ixgbe_prefetch(&rx_ring[next_id]);
2175 rte_ixgbe_prefetch(&sw_ring[next_id]);
2182 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
2184 * Update RX descriptor with the physical address of the
2185 * new data buffer of the new allocated mbuf.
2189 rxm->data_off = RTE_PKTMBUF_HEADROOM;
2190 rxdp->read.hdr_addr = 0;
2191 rxdp->read.pkt_addr = dma;
2196 * Set data length & data buffer address of mbuf.
2198 data_len = rte_le_to_cpu_16(rxd.wb.upper.length);
2199 rxm->data_len = data_len;
2204 * Get next descriptor index:
2205 * - For RSC it's in the NEXTP field.
2206 * - For a scattered packet - it's just a following
2209 if (ixgbe_rsc_count(&rxd))
2211 (staterr & IXGBE_RXDADV_NEXTP_MASK) >>
2212 IXGBE_RXDADV_NEXTP_SHIFT;
2216 next_sc_entry = &sw_sc_ring[nextp_id];
2217 next_rxe = &sw_ring[nextp_id];
2218 rte_ixgbe_prefetch(next_rxe);
2221 sc_entry = &sw_sc_ring[rx_id];
2222 first_seg = sc_entry->fbuf;
2223 sc_entry->fbuf = NULL;
2226 * If this is the first buffer of the received packet,
2227 * set the pointer to the first mbuf of the packet and
2228 * initialize its context.
2229 * Otherwise, update the total length and the number of segments
2230 * of the current scattered packet, and update the pointer to
2231 * the last mbuf of the current packet.
2233 if (first_seg == NULL) {
2235 first_seg->pkt_len = data_len;
2236 first_seg->nb_segs = 1;
2238 first_seg->pkt_len += data_len;
2239 first_seg->nb_segs++;
2246 * If this is not the last buffer of the received packet, update
2247 * the pointer to the first mbuf at the NEXTP entry in the
2248 * sw_sc_ring and continue to parse the RX ring.
2250 if (!eop && next_rxe) {
2251 rxm->next = next_rxe->mbuf;
2252 next_sc_entry->fbuf = first_seg;
2256 /* Initialize the first mbuf of the returned packet */
2257 ixgbe_fill_cluster_head_buf(first_seg, &rxd, rxq, staterr);
2260 * Deal with the case, when HW CRC srip is disabled.
2261 * That can't happen when LRO is enabled, but still could
2262 * happen for scattered RX mode.
2264 first_seg->pkt_len -= rxq->crc_len;
2265 if (unlikely(rxm->data_len <= rxq->crc_len)) {
2266 struct rte_mbuf *lp;
2268 for (lp = first_seg; lp->next != rxm; lp = lp->next)
2271 first_seg->nb_segs--;
2272 lp->data_len -= rxq->crc_len - rxm->data_len;
2274 rte_pktmbuf_free_seg(rxm);
2276 rxm->data_len -= rxq->crc_len;
2278 /* Prefetch data of first segment, if configured to do so. */
2279 rte_packet_prefetch((char *)first_seg->buf_addr +
2280 first_seg->data_off);
2283 * Store the mbuf address into the next entry of the array
2284 * of returned packets.
2286 rx_pkts[nb_rx++] = first_seg;
2290 * Record index of the next RX descriptor to probe.
2292 rxq->rx_tail = rx_id;
2295 * If the number of free RX descriptors is greater than the RX free
2296 * threshold of the queue, advance the Receive Descriptor Tail (RDT)
2298 * Update the RDT with the value of the last processed RX descriptor
2299 * minus 1, to guarantee that the RDT register is never equal to the
2300 * RDH register, which creates a "full" ring situtation from the
2301 * hardware point of view...
2303 if (!bulk_alloc && nb_hold > rxq->rx_free_thresh) {
2304 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u "
2305 "nb_hold=%u nb_rx=%u",
2306 rxq->port_id, rxq->queue_id, rx_id, nb_hold, nb_rx);
2309 IXGBE_PCI_REG_WC_WRITE_RELAXED(rxq->rdt_reg_addr, prev_id);
2313 rxq->nb_rx_hold = nb_hold;
2318 ixgbe_recv_pkts_lro_single_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
2321 return ixgbe_recv_pkts_lro(rx_queue, rx_pkts, nb_pkts, false);
2325 ixgbe_recv_pkts_lro_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
2328 return ixgbe_recv_pkts_lro(rx_queue, rx_pkts, nb_pkts, true);
2331 /*********************************************************************
2333 * Queue management functions
2335 **********************************************************************/
2337 static void __rte_cold
2338 ixgbe_tx_queue_release_mbufs(struct ixgbe_tx_queue *txq)
2342 if (txq->sw_ring != NULL) {
2343 for (i = 0; i < txq->nb_tx_desc; i++) {
2344 if (txq->sw_ring[i].mbuf != NULL) {
2345 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
2346 txq->sw_ring[i].mbuf = NULL;
2353 ixgbe_tx_done_cleanup_full(struct ixgbe_tx_queue *txq, uint32_t free_cnt)
2355 struct ixgbe_tx_entry *swr_ring = txq->sw_ring;
2356 uint16_t i, tx_last, tx_id;
2357 uint16_t nb_tx_free_last;
2358 uint16_t nb_tx_to_clean;
2361 /* Start free mbuf from the next of tx_tail */
2362 tx_last = txq->tx_tail;
2363 tx_id = swr_ring[tx_last].next_id;
2365 if (txq->nb_tx_free == 0 && ixgbe_xmit_cleanup(txq))
2368 nb_tx_to_clean = txq->nb_tx_free;
2369 nb_tx_free_last = txq->nb_tx_free;
2371 free_cnt = txq->nb_tx_desc;
2373 /* Loop through swr_ring to count the amount of
2374 * freeable mubfs and packets.
2376 for (pkt_cnt = 0; pkt_cnt < free_cnt; ) {
2377 for (i = 0; i < nb_tx_to_clean &&
2378 pkt_cnt < free_cnt &&
2379 tx_id != tx_last; i++) {
2380 if (swr_ring[tx_id].mbuf != NULL) {
2381 rte_pktmbuf_free_seg(swr_ring[tx_id].mbuf);
2382 swr_ring[tx_id].mbuf = NULL;
2385 * last segment in the packet,
2386 * increment packet count
2388 pkt_cnt += (swr_ring[tx_id].last_id == tx_id);
2391 tx_id = swr_ring[tx_id].next_id;
2394 if (txq->tx_rs_thresh > txq->nb_tx_desc -
2395 txq->nb_tx_free || tx_id == tx_last)
2398 if (pkt_cnt < free_cnt) {
2399 if (ixgbe_xmit_cleanup(txq))
2402 nb_tx_to_clean = txq->nb_tx_free - nb_tx_free_last;
2403 nb_tx_free_last = txq->nb_tx_free;
2407 return (int)pkt_cnt;
2411 ixgbe_tx_done_cleanup_simple(struct ixgbe_tx_queue *txq,
2416 if (free_cnt == 0 || free_cnt > txq->nb_tx_desc)
2417 free_cnt = txq->nb_tx_desc;
2419 cnt = free_cnt - free_cnt % txq->tx_rs_thresh;
2421 for (i = 0; i < cnt; i += n) {
2422 if (txq->nb_tx_desc - txq->nb_tx_free < txq->tx_rs_thresh)
2425 n = ixgbe_tx_free_bufs(txq);
2435 ixgbe_tx_done_cleanup_vec(struct ixgbe_tx_queue *txq __rte_unused,
2436 uint32_t free_cnt __rte_unused)
2442 ixgbe_dev_tx_done_cleanup(void *tx_queue, uint32_t free_cnt)
2444 struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue;
2445 if (txq->offloads == 0 &&
2446 #ifdef RTE_LIB_SECURITY
2447 !(txq->using_ipsec) &&
2449 txq->tx_rs_thresh >= RTE_PMD_IXGBE_TX_MAX_BURST) {
2450 if (txq->tx_rs_thresh <= RTE_IXGBE_TX_MAX_FREE_BUF_SZ &&
2451 rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128 &&
2452 (rte_eal_process_type() != RTE_PROC_PRIMARY ||
2453 txq->sw_ring_v != NULL)) {
2454 return ixgbe_tx_done_cleanup_vec(txq, free_cnt);
2456 return ixgbe_tx_done_cleanup_simple(txq, free_cnt);
2460 return ixgbe_tx_done_cleanup_full(txq, free_cnt);
2463 static void __rte_cold
2464 ixgbe_tx_free_swring(struct ixgbe_tx_queue *txq)
2467 txq->sw_ring != NULL)
2468 rte_free(txq->sw_ring);
2471 static void __rte_cold
2472 ixgbe_tx_queue_release(struct ixgbe_tx_queue *txq)
2474 if (txq != NULL && txq->ops != NULL) {
2475 txq->ops->release_mbufs(txq);
2476 txq->ops->free_swring(txq);
2482 ixgbe_dev_tx_queue_release(void *txq)
2484 ixgbe_tx_queue_release(txq);
2487 /* (Re)set dynamic ixgbe_tx_queue fields to defaults */
2488 static void __rte_cold
2489 ixgbe_reset_tx_queue(struct ixgbe_tx_queue *txq)
2491 static const union ixgbe_adv_tx_desc zeroed_desc = {{0}};
2492 struct ixgbe_tx_entry *txe = txq->sw_ring;
2495 /* Zero out HW ring memory */
2496 for (i = 0; i < txq->nb_tx_desc; i++) {
2497 txq->tx_ring[i] = zeroed_desc;
2500 /* Initialize SW ring entries */
2501 prev = (uint16_t) (txq->nb_tx_desc - 1);
2502 for (i = 0; i < txq->nb_tx_desc; i++) {
2503 volatile union ixgbe_adv_tx_desc *txd = &txq->tx_ring[i];
2505 txd->wb.status = rte_cpu_to_le_32(IXGBE_TXD_STAT_DD);
2508 txe[prev].next_id = i;
2512 txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
2513 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
2516 txq->nb_tx_used = 0;
2518 * Always allow 1 descriptor to be un-allocated to avoid
2519 * a H/W race condition
2521 txq->last_desc_cleaned = (uint16_t)(txq->nb_tx_desc - 1);
2522 txq->nb_tx_free = (uint16_t)(txq->nb_tx_desc - 1);
2524 memset((void *)&txq->ctx_cache, 0,
2525 IXGBE_CTX_NUM * sizeof(struct ixgbe_advctx_info));
2528 static const struct ixgbe_txq_ops def_txq_ops = {
2529 .release_mbufs = ixgbe_tx_queue_release_mbufs,
2530 .free_swring = ixgbe_tx_free_swring,
2531 .reset = ixgbe_reset_tx_queue,
2534 /* Takes an ethdev and a queue and sets up the tx function to be used based on
2535 * the queue parameters. Used in tx_queue_setup by primary process and then
2536 * in dev_init by secondary process when attaching to an existing ethdev.
2539 ixgbe_set_tx_function(struct rte_eth_dev *dev, struct ixgbe_tx_queue *txq)
2541 /* Use a simple Tx queue (no offloads, no multi segs) if possible */
2542 if ((txq->offloads == 0) &&
2543 #ifdef RTE_LIB_SECURITY
2544 !(txq->using_ipsec) &&
2546 (txq->tx_rs_thresh >= RTE_PMD_IXGBE_TX_MAX_BURST)) {
2547 PMD_INIT_LOG(DEBUG, "Using simple tx code path");
2548 dev->tx_pkt_prepare = NULL;
2549 if (txq->tx_rs_thresh <= RTE_IXGBE_TX_MAX_FREE_BUF_SZ &&
2550 rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128 &&
2551 (rte_eal_process_type() != RTE_PROC_PRIMARY ||
2552 ixgbe_txq_vec_setup(txq) == 0)) {
2553 PMD_INIT_LOG(DEBUG, "Vector tx enabled.");
2554 dev->tx_pkt_burst = ixgbe_xmit_pkts_vec;
2556 dev->tx_pkt_burst = ixgbe_xmit_pkts_simple;
2558 PMD_INIT_LOG(DEBUG, "Using full-featured tx code path");
2560 " - offloads = 0x%" PRIx64,
2563 " - tx_rs_thresh = %lu " "[RTE_PMD_IXGBE_TX_MAX_BURST=%lu]",
2564 (unsigned long)txq->tx_rs_thresh,
2565 (unsigned long)RTE_PMD_IXGBE_TX_MAX_BURST);
2566 dev->tx_pkt_burst = ixgbe_xmit_pkts;
2567 dev->tx_pkt_prepare = ixgbe_prep_pkts;
2572 ixgbe_get_tx_queue_offloads(struct rte_eth_dev *dev)
2580 ixgbe_get_tx_port_offloads(struct rte_eth_dev *dev)
2582 uint64_t tx_offload_capa;
2583 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2586 DEV_TX_OFFLOAD_VLAN_INSERT |
2587 DEV_TX_OFFLOAD_IPV4_CKSUM |
2588 DEV_TX_OFFLOAD_UDP_CKSUM |
2589 DEV_TX_OFFLOAD_TCP_CKSUM |
2590 DEV_TX_OFFLOAD_SCTP_CKSUM |
2591 DEV_TX_OFFLOAD_TCP_TSO |
2592 DEV_TX_OFFLOAD_MULTI_SEGS;
2594 if (hw->mac.type == ixgbe_mac_82599EB ||
2595 hw->mac.type == ixgbe_mac_X540)
2596 tx_offload_capa |= DEV_TX_OFFLOAD_MACSEC_INSERT;
2598 if (hw->mac.type == ixgbe_mac_X550 ||
2599 hw->mac.type == ixgbe_mac_X550EM_x ||
2600 hw->mac.type == ixgbe_mac_X550EM_a)
2601 tx_offload_capa |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM;
2603 #ifdef RTE_LIB_SECURITY
2604 if (dev->security_ctx)
2605 tx_offload_capa |= DEV_TX_OFFLOAD_SECURITY;
2607 return tx_offload_capa;
2611 ixgbe_dev_tx_queue_setup(struct rte_eth_dev *dev,
2614 unsigned int socket_id,
2615 const struct rte_eth_txconf *tx_conf)
2617 const struct rte_memzone *tz;
2618 struct ixgbe_tx_queue *txq;
2619 struct ixgbe_hw *hw;
2620 uint16_t tx_rs_thresh, tx_free_thresh;
2623 PMD_INIT_FUNC_TRACE();
2624 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2626 offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads;
2629 * Validate number of transmit descriptors.
2630 * It must not exceed hardware maximum, and must be multiple
2633 if (nb_desc % IXGBE_TXD_ALIGN != 0 ||
2634 (nb_desc > IXGBE_MAX_RING_DESC) ||
2635 (nb_desc < IXGBE_MIN_RING_DESC)) {
2640 * The following two parameters control the setting of the RS bit on
2641 * transmit descriptors.
2642 * TX descriptors will have their RS bit set after txq->tx_rs_thresh
2643 * descriptors have been used.
2644 * The TX descriptor ring will be cleaned after txq->tx_free_thresh
2645 * descriptors are used or if the number of descriptors required
2646 * to transmit a packet is greater than the number of free TX
2648 * The following constraints must be satisfied:
2649 * tx_rs_thresh must be greater than 0.
2650 * tx_rs_thresh must be less than the size of the ring minus 2.
2651 * tx_rs_thresh must be less than or equal to tx_free_thresh.
2652 * tx_rs_thresh must be a divisor of the ring size.
2653 * tx_free_thresh must be greater than 0.
2654 * tx_free_thresh must be less than the size of the ring minus 3.
2655 * tx_free_thresh + tx_rs_thresh must not exceed nb_desc.
2656 * One descriptor in the TX ring is used as a sentinel to avoid a
2657 * H/W race condition, hence the maximum threshold constraints.
2658 * When set to zero use default values.
2660 tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ?
2661 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH);
2662 /* force tx_rs_thresh to adapt an aggresive tx_free_thresh */
2663 tx_rs_thresh = (DEFAULT_TX_RS_THRESH + tx_free_thresh > nb_desc) ?
2664 nb_desc - tx_free_thresh : DEFAULT_TX_RS_THRESH;
2665 if (tx_conf->tx_rs_thresh > 0)
2666 tx_rs_thresh = tx_conf->tx_rs_thresh;
2667 if (tx_rs_thresh + tx_free_thresh > nb_desc) {
2668 PMD_INIT_LOG(ERR, "tx_rs_thresh + tx_free_thresh must not "
2669 "exceed nb_desc. (tx_rs_thresh=%u "
2670 "tx_free_thresh=%u nb_desc=%u port = %d queue=%d)",
2671 (unsigned int)tx_rs_thresh,
2672 (unsigned int)tx_free_thresh,
2673 (unsigned int)nb_desc,
2674 (int)dev->data->port_id,
2678 if (tx_rs_thresh >= (nb_desc - 2)) {
2679 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than the number "
2680 "of TX descriptors minus 2. (tx_rs_thresh=%u "
2681 "port=%d queue=%d)", (unsigned int)tx_rs_thresh,
2682 (int)dev->data->port_id, (int)queue_idx);
2685 if (tx_rs_thresh > DEFAULT_TX_RS_THRESH) {
2686 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less or equal than %u. "
2687 "(tx_rs_thresh=%u port=%d queue=%d)",
2688 DEFAULT_TX_RS_THRESH, (unsigned int)tx_rs_thresh,
2689 (int)dev->data->port_id, (int)queue_idx);
2692 if (tx_free_thresh >= (nb_desc - 3)) {
2693 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than the "
2694 "tx_free_thresh must be less than the number of "
2695 "TX descriptors minus 3. (tx_free_thresh=%u "
2696 "port=%d queue=%d)",
2697 (unsigned int)tx_free_thresh,
2698 (int)dev->data->port_id, (int)queue_idx);
2701 if (tx_rs_thresh > tx_free_thresh) {
2702 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than or equal to "
2703 "tx_free_thresh. (tx_free_thresh=%u "
2704 "tx_rs_thresh=%u port=%d queue=%d)",
2705 (unsigned int)tx_free_thresh,
2706 (unsigned int)tx_rs_thresh,
2707 (int)dev->data->port_id,
2711 if ((nb_desc % tx_rs_thresh) != 0) {
2712 PMD_INIT_LOG(ERR, "tx_rs_thresh must be a divisor of the "
2713 "number of TX descriptors. (tx_rs_thresh=%u "
2714 "port=%d queue=%d)", (unsigned int)tx_rs_thresh,
2715 (int)dev->data->port_id, (int)queue_idx);
2720 * If rs_bit_thresh is greater than 1, then TX WTHRESH should be
2721 * set to 0. If WTHRESH is greater than zero, the RS bit is ignored
2722 * by the NIC and all descriptors are written back after the NIC
2723 * accumulates WTHRESH descriptors.
2725 if ((tx_rs_thresh > 1) && (tx_conf->tx_thresh.wthresh != 0)) {
2726 PMD_INIT_LOG(ERR, "TX WTHRESH must be set to 0 if "
2727 "tx_rs_thresh is greater than 1. (tx_rs_thresh=%u "
2728 "port=%d queue=%d)", (unsigned int)tx_rs_thresh,
2729 (int)dev->data->port_id, (int)queue_idx);
2733 /* Free memory prior to re-allocation if needed... */
2734 if (dev->data->tx_queues[queue_idx] != NULL) {
2735 ixgbe_tx_queue_release(dev->data->tx_queues[queue_idx]);
2736 dev->data->tx_queues[queue_idx] = NULL;
2739 /* First allocate the tx queue data structure */
2740 txq = rte_zmalloc_socket("ethdev TX queue", sizeof(struct ixgbe_tx_queue),
2741 RTE_CACHE_LINE_SIZE, socket_id);
2746 * Allocate TX ring hardware descriptors. A memzone large enough to
2747 * handle the maximum ring size is allocated in order to allow for
2748 * resizing in later calls to the queue setup function.
2750 tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx,
2751 sizeof(union ixgbe_adv_tx_desc) * IXGBE_MAX_RING_DESC,
2752 IXGBE_ALIGN, socket_id);
2754 ixgbe_tx_queue_release(txq);
2758 txq->nb_tx_desc = nb_desc;
2759 txq->tx_rs_thresh = tx_rs_thresh;
2760 txq->tx_free_thresh = tx_free_thresh;
2761 txq->pthresh = tx_conf->tx_thresh.pthresh;
2762 txq->hthresh = tx_conf->tx_thresh.hthresh;
2763 txq->wthresh = tx_conf->tx_thresh.wthresh;
2764 txq->queue_id = queue_idx;
2765 txq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ?
2766 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx);
2767 txq->port_id = dev->data->port_id;
2768 txq->offloads = offloads;
2769 txq->ops = &def_txq_ops;
2770 txq->tx_deferred_start = tx_conf->tx_deferred_start;
2771 #ifdef RTE_LIB_SECURITY
2772 txq->using_ipsec = !!(dev->data->dev_conf.txmode.offloads &
2773 DEV_TX_OFFLOAD_SECURITY);
2777 * Modification to set VFTDT for virtual function if vf is detected
2779 if (hw->mac.type == ixgbe_mac_82599_vf ||
2780 hw->mac.type == ixgbe_mac_X540_vf ||
2781 hw->mac.type == ixgbe_mac_X550_vf ||
2782 hw->mac.type == ixgbe_mac_X550EM_x_vf ||
2783 hw->mac.type == ixgbe_mac_X550EM_a_vf)
2784 txq->tdt_reg_addr = IXGBE_PCI_REG_ADDR(hw, IXGBE_VFTDT(queue_idx));
2786 txq->tdt_reg_addr = IXGBE_PCI_REG_ADDR(hw, IXGBE_TDT(txq->reg_idx));
2788 txq->tx_ring_phys_addr = tz->iova;
2789 txq->tx_ring = (union ixgbe_adv_tx_desc *) tz->addr;
2791 /* Allocate software ring */
2792 txq->sw_ring = rte_zmalloc_socket("txq->sw_ring",
2793 sizeof(struct ixgbe_tx_entry) * nb_desc,
2794 RTE_CACHE_LINE_SIZE, socket_id);
2795 if (txq->sw_ring == NULL) {
2796 ixgbe_tx_queue_release(txq);
2799 PMD_INIT_LOG(DEBUG, "sw_ring=%p hw_ring=%p dma_addr=0x%"PRIx64,
2800 txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr);
2802 /* set up vector or scalar TX function as appropriate */
2803 ixgbe_set_tx_function(dev, txq);
2805 txq->ops->reset(txq);
2807 dev->data->tx_queues[queue_idx] = txq;
2814 * ixgbe_free_sc_cluster - free the not-yet-completed scattered cluster
2816 * The "next" pointer of the last segment of (not-yet-completed) RSC clusters
2817 * in the sw_rsc_ring is not set to NULL but rather points to the next
2818 * mbuf of this RSC aggregation (that has not been completed yet and still
2819 * resides on the HW ring). So, instead of calling for rte_pktmbuf_free() we
2820 * will just free first "nb_segs" segments of the cluster explicitly by calling
2821 * an rte_pktmbuf_free_seg().
2823 * @m scattered cluster head
2825 static void __rte_cold
2826 ixgbe_free_sc_cluster(struct rte_mbuf *m)
2828 uint16_t i, nb_segs = m->nb_segs;
2829 struct rte_mbuf *next_seg;
2831 for (i = 0; i < nb_segs; i++) {
2833 rte_pktmbuf_free_seg(m);
2838 static void __rte_cold
2839 ixgbe_rx_queue_release_mbufs(struct ixgbe_rx_queue *rxq)
2843 /* SSE Vector driver has a different way of releasing mbufs. */
2844 if (rxq->rx_using_sse) {
2845 ixgbe_rx_queue_release_mbufs_vec(rxq);
2849 if (rxq->sw_ring != NULL) {
2850 for (i = 0; i < rxq->nb_rx_desc; i++) {
2851 if (rxq->sw_ring[i].mbuf != NULL) {
2852 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
2853 rxq->sw_ring[i].mbuf = NULL;
2856 if (rxq->rx_nb_avail) {
2857 for (i = 0; i < rxq->rx_nb_avail; ++i) {
2858 struct rte_mbuf *mb;
2860 mb = rxq->rx_stage[rxq->rx_next_avail + i];
2861 rte_pktmbuf_free_seg(mb);
2863 rxq->rx_nb_avail = 0;
2867 if (rxq->sw_sc_ring)
2868 for (i = 0; i < rxq->nb_rx_desc; i++)
2869 if (rxq->sw_sc_ring[i].fbuf) {
2870 ixgbe_free_sc_cluster(rxq->sw_sc_ring[i].fbuf);
2871 rxq->sw_sc_ring[i].fbuf = NULL;
2875 static void __rte_cold
2876 ixgbe_rx_queue_release(struct ixgbe_rx_queue *rxq)
2879 ixgbe_rx_queue_release_mbufs(rxq);
2880 rte_free(rxq->sw_ring);
2881 rte_free(rxq->sw_sc_ring);
2887 ixgbe_dev_rx_queue_release(void *rxq)
2889 ixgbe_rx_queue_release(rxq);
2893 * Check if Rx Burst Bulk Alloc function can be used.
2895 * 0: the preconditions are satisfied and the bulk allocation function
2897 * -EINVAL: the preconditions are NOT satisfied and the default Rx burst
2898 * function must be used.
2900 static inline int __rte_cold
2901 check_rx_burst_bulk_alloc_preconditions(struct ixgbe_rx_queue *rxq)
2906 * Make sure the following pre-conditions are satisfied:
2907 * rxq->rx_free_thresh >= RTE_PMD_IXGBE_RX_MAX_BURST
2908 * rxq->rx_free_thresh < rxq->nb_rx_desc
2909 * (rxq->nb_rx_desc % rxq->rx_free_thresh) == 0
2910 * Scattered packets are not supported. This should be checked
2911 * outside of this function.
2913 if (!(rxq->rx_free_thresh >= RTE_PMD_IXGBE_RX_MAX_BURST)) {
2914 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
2915 "rxq->rx_free_thresh=%d, "
2916 "RTE_PMD_IXGBE_RX_MAX_BURST=%d",
2917 rxq->rx_free_thresh, RTE_PMD_IXGBE_RX_MAX_BURST);
2919 } else if (!(rxq->rx_free_thresh < rxq->nb_rx_desc)) {
2920 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
2921 "rxq->rx_free_thresh=%d, "
2922 "rxq->nb_rx_desc=%d",
2923 rxq->rx_free_thresh, rxq->nb_rx_desc);
2925 } else if (!((rxq->nb_rx_desc % rxq->rx_free_thresh) == 0)) {
2926 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
2927 "rxq->nb_rx_desc=%d, "
2928 "rxq->rx_free_thresh=%d",
2929 rxq->nb_rx_desc, rxq->rx_free_thresh);
2936 /* Reset dynamic ixgbe_rx_queue fields back to defaults */
2937 static void __rte_cold
2938 ixgbe_reset_rx_queue(struct ixgbe_adapter *adapter, struct ixgbe_rx_queue *rxq)
2940 static const union ixgbe_adv_rx_desc zeroed_desc = {{0}};
2942 uint16_t len = rxq->nb_rx_desc;
2945 * By default, the Rx queue setup function allocates enough memory for
2946 * IXGBE_MAX_RING_DESC. The Rx Burst bulk allocation function requires
2947 * extra memory at the end of the descriptor ring to be zero'd out.
2949 if (adapter->rx_bulk_alloc_allowed)
2950 /* zero out extra memory */
2951 len += RTE_PMD_IXGBE_RX_MAX_BURST;
2954 * Zero out HW ring memory. Zero out extra memory at the end of
2955 * the H/W ring so look-ahead logic in Rx Burst bulk alloc function
2956 * reads extra memory as zeros.
2958 for (i = 0; i < len; i++) {
2959 rxq->rx_ring[i] = zeroed_desc;
2963 * initialize extra software ring entries. Space for these extra
2964 * entries is always allocated
2966 memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf));
2967 for (i = rxq->nb_rx_desc; i < len; ++i) {
2968 rxq->sw_ring[i].mbuf = &rxq->fake_mbuf;
2971 rxq->rx_nb_avail = 0;
2972 rxq->rx_next_avail = 0;
2973 rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1);
2975 rxq->nb_rx_hold = 0;
2976 rxq->pkt_first_seg = NULL;
2977 rxq->pkt_last_seg = NULL;
2979 #if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64)
2980 rxq->rxrearm_start = 0;
2981 rxq->rxrearm_nb = 0;
2986 ixgbe_is_vf(struct rte_eth_dev *dev)
2988 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2990 switch (hw->mac.type) {
2991 case ixgbe_mac_82599_vf:
2992 case ixgbe_mac_X540_vf:
2993 case ixgbe_mac_X550_vf:
2994 case ixgbe_mac_X550EM_x_vf:
2995 case ixgbe_mac_X550EM_a_vf:
3003 ixgbe_get_rx_queue_offloads(struct rte_eth_dev *dev)
3005 uint64_t offloads = 0;
3006 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3008 if (hw->mac.type != ixgbe_mac_82598EB)
3009 offloads |= DEV_RX_OFFLOAD_VLAN_STRIP;
3015 ixgbe_get_rx_port_offloads(struct rte_eth_dev *dev)
3018 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3020 offloads = DEV_RX_OFFLOAD_IPV4_CKSUM |
3021 DEV_RX_OFFLOAD_UDP_CKSUM |
3022 DEV_RX_OFFLOAD_TCP_CKSUM |
3023 DEV_RX_OFFLOAD_KEEP_CRC |
3024 DEV_RX_OFFLOAD_JUMBO_FRAME |
3025 DEV_RX_OFFLOAD_VLAN_FILTER |
3026 DEV_RX_OFFLOAD_SCATTER |
3027 DEV_RX_OFFLOAD_RSS_HASH;
3029 if (hw->mac.type == ixgbe_mac_82598EB)
3030 offloads |= DEV_RX_OFFLOAD_VLAN_STRIP;
3032 if (ixgbe_is_vf(dev) == 0)
3033 offloads |= DEV_RX_OFFLOAD_VLAN_EXTEND;
3036 * RSC is only supported by 82599 and x540 PF devices in a non-SR-IOV
3039 if ((hw->mac.type == ixgbe_mac_82599EB ||
3040 hw->mac.type == ixgbe_mac_X540 ||
3041 hw->mac.type == ixgbe_mac_X550) &&
3042 !RTE_ETH_DEV_SRIOV(dev).active)
3043 offloads |= DEV_RX_OFFLOAD_TCP_LRO;
3045 if (hw->mac.type == ixgbe_mac_82599EB ||
3046 hw->mac.type == ixgbe_mac_X540)
3047 offloads |= DEV_RX_OFFLOAD_MACSEC_STRIP;
3049 if (hw->mac.type == ixgbe_mac_X550 ||
3050 hw->mac.type == ixgbe_mac_X550EM_x ||
3051 hw->mac.type == ixgbe_mac_X550EM_a)
3052 offloads |= DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM;
3054 #ifdef RTE_LIB_SECURITY
3055 if (dev->security_ctx)
3056 offloads |= DEV_RX_OFFLOAD_SECURITY;
3063 ixgbe_dev_rx_queue_setup(struct rte_eth_dev *dev,
3066 unsigned int socket_id,
3067 const struct rte_eth_rxconf *rx_conf,
3068 struct rte_mempool *mp)
3070 const struct rte_memzone *rz;
3071 struct ixgbe_rx_queue *rxq;
3072 struct ixgbe_hw *hw;
3074 struct ixgbe_adapter *adapter = dev->data->dev_private;
3077 PMD_INIT_FUNC_TRACE();
3078 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3080 offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads;
3083 * Validate number of receive descriptors.
3084 * It must not exceed hardware maximum, and must be multiple
3087 if (nb_desc % IXGBE_RXD_ALIGN != 0 ||
3088 (nb_desc > IXGBE_MAX_RING_DESC) ||
3089 (nb_desc < IXGBE_MIN_RING_DESC)) {
3093 /* Free memory prior to re-allocation if needed... */
3094 if (dev->data->rx_queues[queue_idx] != NULL) {
3095 ixgbe_rx_queue_release(dev->data->rx_queues[queue_idx]);
3096 dev->data->rx_queues[queue_idx] = NULL;
3099 /* First allocate the rx queue data structure */
3100 rxq = rte_zmalloc_socket("ethdev RX queue", sizeof(struct ixgbe_rx_queue),
3101 RTE_CACHE_LINE_SIZE, socket_id);
3105 rxq->nb_rx_desc = nb_desc;
3106 rxq->rx_free_thresh = rx_conf->rx_free_thresh;
3107 rxq->queue_id = queue_idx;
3108 rxq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ?
3109 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx);
3110 rxq->port_id = dev->data->port_id;
3111 if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC)
3112 rxq->crc_len = RTE_ETHER_CRC_LEN;
3115 rxq->drop_en = rx_conf->rx_drop_en;
3116 rxq->rx_deferred_start = rx_conf->rx_deferred_start;
3117 rxq->offloads = offloads;
3120 * The packet type in RX descriptor is different for different NICs.
3121 * Some bits are used for x550 but reserved for other NICS.
3122 * So set different masks for different NICs.
3124 if (hw->mac.type == ixgbe_mac_X550 ||
3125 hw->mac.type == ixgbe_mac_X550EM_x ||
3126 hw->mac.type == ixgbe_mac_X550EM_a ||
3127 hw->mac.type == ixgbe_mac_X550_vf ||
3128 hw->mac.type == ixgbe_mac_X550EM_x_vf ||
3129 hw->mac.type == ixgbe_mac_X550EM_a_vf)
3130 rxq->pkt_type_mask = IXGBE_PACKET_TYPE_MASK_X550;
3132 rxq->pkt_type_mask = IXGBE_PACKET_TYPE_MASK_82599;
3135 * 82599 errata, UDP frames with a 0 checksum can be marked as checksum
3138 if (hw->mac.type == ixgbe_mac_82599EB)
3139 rxq->rx_udp_csum_zero_err = 1;
3142 * Allocate RX ring hardware descriptors. A memzone large enough to
3143 * handle the maximum ring size is allocated in order to allow for
3144 * resizing in later calls to the queue setup function.
3146 rz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx,
3147 RX_RING_SZ, IXGBE_ALIGN, socket_id);
3149 ixgbe_rx_queue_release(rxq);
3154 * Zero init all the descriptors in the ring.
3156 memset(rz->addr, 0, RX_RING_SZ);
3159 * Modified to setup VFRDT for Virtual Function
3161 if (hw->mac.type == ixgbe_mac_82599_vf ||
3162 hw->mac.type == ixgbe_mac_X540_vf ||
3163 hw->mac.type == ixgbe_mac_X550_vf ||
3164 hw->mac.type == ixgbe_mac_X550EM_x_vf ||
3165 hw->mac.type == ixgbe_mac_X550EM_a_vf) {
3167 IXGBE_PCI_REG_ADDR(hw, IXGBE_VFRDT(queue_idx));
3169 IXGBE_PCI_REG_ADDR(hw, IXGBE_VFRDH(queue_idx));
3172 IXGBE_PCI_REG_ADDR(hw, IXGBE_RDT(rxq->reg_idx));
3174 IXGBE_PCI_REG_ADDR(hw, IXGBE_RDH(rxq->reg_idx));
3177 rxq->rx_ring_phys_addr = rz->iova;
3178 rxq->rx_ring = (union ixgbe_adv_rx_desc *) rz->addr;
3181 * Certain constraints must be met in order to use the bulk buffer
3182 * allocation Rx burst function. If any of Rx queues doesn't meet them
3183 * the feature should be disabled for the whole port.
3185 if (check_rx_burst_bulk_alloc_preconditions(rxq)) {
3186 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Rx Bulk Alloc "
3187 "preconditions - canceling the feature for "
3188 "the whole port[%d]",
3189 rxq->queue_id, rxq->port_id);
3190 adapter->rx_bulk_alloc_allowed = false;
3194 * Allocate software ring. Allow for space at the end of the
3195 * S/W ring to make sure look-ahead logic in bulk alloc Rx burst
3196 * function does not access an invalid memory region.
3199 if (adapter->rx_bulk_alloc_allowed)
3200 len += RTE_PMD_IXGBE_RX_MAX_BURST;
3202 rxq->sw_ring = rte_zmalloc_socket("rxq->sw_ring",
3203 sizeof(struct ixgbe_rx_entry) * len,
3204 RTE_CACHE_LINE_SIZE, socket_id);
3205 if (!rxq->sw_ring) {
3206 ixgbe_rx_queue_release(rxq);
3211 * Always allocate even if it's not going to be needed in order to
3212 * simplify the code.
3214 * This ring is used in LRO and Scattered Rx cases and Scattered Rx may
3215 * be requested in ixgbe_dev_rx_init(), which is called later from
3219 rte_zmalloc_socket("rxq->sw_sc_ring",
3220 sizeof(struct ixgbe_scattered_rx_entry) * len,
3221 RTE_CACHE_LINE_SIZE, socket_id);
3222 if (!rxq->sw_sc_ring) {
3223 ixgbe_rx_queue_release(rxq);
3227 PMD_INIT_LOG(DEBUG, "sw_ring=%p sw_sc_ring=%p hw_ring=%p "
3228 "dma_addr=0x%"PRIx64,
3229 rxq->sw_ring, rxq->sw_sc_ring, rxq->rx_ring,
3230 rxq->rx_ring_phys_addr);
3232 if (!rte_is_power_of_2(nb_desc)) {
3233 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
3234 "preconditions - canceling the feature for "
3235 "the whole port[%d]",
3236 rxq->queue_id, rxq->port_id);
3237 adapter->rx_vec_allowed = false;
3239 ixgbe_rxq_vec_setup(rxq);
3241 dev->data->rx_queues[queue_idx] = rxq;
3243 ixgbe_reset_rx_queue(adapter, rxq);
3249 ixgbe_dev_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id)
3251 #define IXGBE_RXQ_SCAN_INTERVAL 4
3252 volatile union ixgbe_adv_rx_desc *rxdp;
3253 struct ixgbe_rx_queue *rxq;
3256 rxq = dev->data->rx_queues[rx_queue_id];
3257 rxdp = &(rxq->rx_ring[rxq->rx_tail]);
3259 while ((desc < rxq->nb_rx_desc) &&
3260 (rxdp->wb.upper.status_error &
3261 rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD))) {
3262 desc += IXGBE_RXQ_SCAN_INTERVAL;
3263 rxdp += IXGBE_RXQ_SCAN_INTERVAL;
3264 if (rxq->rx_tail + desc >= rxq->nb_rx_desc)
3265 rxdp = &(rxq->rx_ring[rxq->rx_tail +
3266 desc - rxq->nb_rx_desc]);
3273 ixgbe_dev_rx_descriptor_done(void *rx_queue, uint16_t offset)
3275 volatile union ixgbe_adv_rx_desc *rxdp;
3276 struct ixgbe_rx_queue *rxq = rx_queue;
3279 if (unlikely(offset >= rxq->nb_rx_desc))
3281 desc = rxq->rx_tail + offset;
3282 if (desc >= rxq->nb_rx_desc)
3283 desc -= rxq->nb_rx_desc;
3285 rxdp = &rxq->rx_ring[desc];
3286 return !!(rxdp->wb.upper.status_error &
3287 rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD));
3291 ixgbe_dev_rx_descriptor_status(void *rx_queue, uint16_t offset)
3293 struct ixgbe_rx_queue *rxq = rx_queue;
3294 volatile uint32_t *status;
3295 uint32_t nb_hold, desc;
3297 if (unlikely(offset >= rxq->nb_rx_desc))
3300 #if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64)
3301 if (rxq->rx_using_sse)
3302 nb_hold = rxq->rxrearm_nb;
3305 nb_hold = rxq->nb_rx_hold;
3306 if (offset >= rxq->nb_rx_desc - nb_hold)
3307 return RTE_ETH_RX_DESC_UNAVAIL;
3309 desc = rxq->rx_tail + offset;
3310 if (desc >= rxq->nb_rx_desc)
3311 desc -= rxq->nb_rx_desc;
3313 status = &rxq->rx_ring[desc].wb.upper.status_error;
3314 if (*status & rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD))
3315 return RTE_ETH_RX_DESC_DONE;
3317 return RTE_ETH_RX_DESC_AVAIL;
3321 ixgbe_dev_tx_descriptor_status(void *tx_queue, uint16_t offset)
3323 struct ixgbe_tx_queue *txq = tx_queue;
3324 volatile uint32_t *status;
3327 if (unlikely(offset >= txq->nb_tx_desc))
3330 desc = txq->tx_tail + offset;
3331 /* go to next desc that has the RS bit */
3332 desc = ((desc + txq->tx_rs_thresh - 1) / txq->tx_rs_thresh) *
3334 if (desc >= txq->nb_tx_desc) {
3335 desc -= txq->nb_tx_desc;
3336 if (desc >= txq->nb_tx_desc)
3337 desc -= txq->nb_tx_desc;
3340 status = &txq->tx_ring[desc].wb.status;
3341 if (*status & rte_cpu_to_le_32(IXGBE_ADVTXD_STAT_DD))
3342 return RTE_ETH_TX_DESC_DONE;
3344 return RTE_ETH_TX_DESC_FULL;
3348 * Set up link loopback for X540/X550 mode Tx->Rx.
3350 static inline void __rte_cold
3351 ixgbe_setup_loopback_link_x540_x550(struct ixgbe_hw *hw, bool enable)
3354 PMD_INIT_FUNC_TRACE();
3356 u16 autoneg_reg = IXGBE_MII_AUTONEG_REG;
3358 hw->phy.ops.read_reg(hw, IXGBE_MDIO_AUTO_NEG_CONTROL,
3359 IXGBE_MDIO_AUTO_NEG_DEV_TYPE, &autoneg_reg);
3360 macc = IXGBE_READ_REG(hw, IXGBE_MACC);
3363 /* datasheet 15.2.1: disable AUTONEG (PHY Bit 7.0.C) */
3364 autoneg_reg |= IXGBE_MII_AUTONEG_ENABLE;
3365 /* datasheet 15.2.1: MACC.FLU = 1 (force link up) */
3366 macc |= IXGBE_MACC_FLU;
3368 autoneg_reg &= ~IXGBE_MII_AUTONEG_ENABLE;
3369 macc &= ~IXGBE_MACC_FLU;
3372 hw->phy.ops.write_reg(hw, IXGBE_MDIO_AUTO_NEG_CONTROL,
3373 IXGBE_MDIO_AUTO_NEG_DEV_TYPE, autoneg_reg);
3375 IXGBE_WRITE_REG(hw, IXGBE_MACC, macc);
3379 ixgbe_dev_clear_queues(struct rte_eth_dev *dev)
3382 struct ixgbe_adapter *adapter = dev->data->dev_private;
3383 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3385 PMD_INIT_FUNC_TRACE();
3387 for (i = 0; i < dev->data->nb_tx_queues; i++) {
3388 struct ixgbe_tx_queue *txq = dev->data->tx_queues[i];
3391 txq->ops->release_mbufs(txq);
3392 txq->ops->reset(txq);
3396 for (i = 0; i < dev->data->nb_rx_queues; i++) {
3397 struct ixgbe_rx_queue *rxq = dev->data->rx_queues[i];
3400 ixgbe_rx_queue_release_mbufs(rxq);
3401 ixgbe_reset_rx_queue(adapter, rxq);
3404 /* If loopback mode was enabled, reconfigure the link accordingly */
3405 if (dev->data->dev_conf.lpbk_mode != 0) {
3406 if (hw->mac.type == ixgbe_mac_X540 ||
3407 hw->mac.type == ixgbe_mac_X550 ||
3408 hw->mac.type == ixgbe_mac_X550EM_x ||
3409 hw->mac.type == ixgbe_mac_X550EM_a)
3410 ixgbe_setup_loopback_link_x540_x550(hw, false);
3415 ixgbe_dev_free_queues(struct rte_eth_dev *dev)
3419 PMD_INIT_FUNC_TRACE();
3421 for (i = 0; i < dev->data->nb_rx_queues; i++) {
3422 ixgbe_dev_rx_queue_release(dev->data->rx_queues[i]);
3423 dev->data->rx_queues[i] = NULL;
3424 rte_eth_dma_zone_free(dev, "rx_ring", i);
3426 dev->data->nb_rx_queues = 0;
3428 for (i = 0; i < dev->data->nb_tx_queues; i++) {
3429 ixgbe_dev_tx_queue_release(dev->data->tx_queues[i]);
3430 dev->data->tx_queues[i] = NULL;
3431 rte_eth_dma_zone_free(dev, "tx_ring", i);
3433 dev->data->nb_tx_queues = 0;
3436 /*********************************************************************
3438 * Device RX/TX init functions
3440 **********************************************************************/
3443 * Receive Side Scaling (RSS)
3444 * See section 7.1.2.8 in the following document:
3445 * "Intel 82599 10 GbE Controller Datasheet" - Revision 2.1 October 2009
3448 * The source and destination IP addresses of the IP header and the source
3449 * and destination ports of TCP/UDP headers, if any, of received packets are
3450 * hashed against a configurable random key to compute a 32-bit RSS hash result.
3451 * The seven (7) LSBs of the 32-bit hash result are used as an index into a
3452 * 128-entry redirection table (RETA). Each entry of the RETA provides a 3-bit
3453 * RSS output index which is used as the RX queue index where to store the
3455 * The following output is supplied in the RX write-back descriptor:
3456 * - 32-bit result of the Microsoft RSS hash function,
3457 * - 4-bit RSS type field.
3461 * RSS random key supplied in section 7.1.2.8.3 of the Intel 82599 datasheet.
3462 * Used as the default key.
3464 static uint8_t rss_intel_key[40] = {
3465 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
3466 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
3467 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
3468 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
3469 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
3473 ixgbe_rss_disable(struct rte_eth_dev *dev)
3475 struct ixgbe_hw *hw;
3479 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3480 mrqc_reg = ixgbe_mrqc_reg_get(hw->mac.type);
3481 mrqc = IXGBE_READ_REG(hw, mrqc_reg);
3482 mrqc &= ~IXGBE_MRQC_RSSEN;
3483 IXGBE_WRITE_REG(hw, mrqc_reg, mrqc);
3487 ixgbe_hw_rss_hash_set(struct ixgbe_hw *hw, struct rte_eth_rss_conf *rss_conf)
3497 mrqc_reg = ixgbe_mrqc_reg_get(hw->mac.type);
3498 rssrk_reg = ixgbe_rssrk_reg_get(hw->mac.type, 0);
3500 hash_key = rss_conf->rss_key;
3501 if (hash_key != NULL) {
3502 /* Fill in RSS hash key */
3503 for (i = 0; i < 10; i++) {
3504 rss_key = hash_key[(i * 4)];
3505 rss_key |= hash_key[(i * 4) + 1] << 8;
3506 rss_key |= hash_key[(i * 4) + 2] << 16;
3507 rss_key |= hash_key[(i * 4) + 3] << 24;
3508 IXGBE_WRITE_REG_ARRAY(hw, rssrk_reg, i, rss_key);
3512 /* Set configured hashing protocols in MRQC register */
3513 rss_hf = rss_conf->rss_hf;
3514 mrqc = IXGBE_MRQC_RSSEN; /* Enable RSS */
3515 if (rss_hf & ETH_RSS_IPV4)
3516 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4;
3517 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
3518 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP;
3519 if (rss_hf & ETH_RSS_IPV6)
3520 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6;
3521 if (rss_hf & ETH_RSS_IPV6_EX)
3522 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX;
3523 if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP)
3524 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP;
3525 if (rss_hf & ETH_RSS_IPV6_TCP_EX)
3526 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP;
3527 if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP)
3528 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP;
3529 if (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP)
3530 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP;
3531 if (rss_hf & ETH_RSS_IPV6_UDP_EX)
3532 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
3533 IXGBE_WRITE_REG(hw, mrqc_reg, mrqc);
3537 ixgbe_dev_rss_hash_update(struct rte_eth_dev *dev,
3538 struct rte_eth_rss_conf *rss_conf)
3540 struct ixgbe_hw *hw;
3545 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3547 if (!ixgbe_rss_update_sp(hw->mac.type)) {
3548 PMD_DRV_LOG(ERR, "RSS hash update is not supported on this "
3552 mrqc_reg = ixgbe_mrqc_reg_get(hw->mac.type);
3555 * Excerpt from section 7.1.2.8 Receive-Side Scaling (RSS):
3556 * "RSS enabling cannot be done dynamically while it must be
3557 * preceded by a software reset"
3558 * Before changing anything, first check that the update RSS operation
3559 * does not attempt to disable RSS, if RSS was enabled at
3560 * initialization time, or does not attempt to enable RSS, if RSS was
3561 * disabled at initialization time.
3563 rss_hf = rss_conf->rss_hf & IXGBE_RSS_OFFLOAD_ALL;
3564 mrqc = IXGBE_READ_REG(hw, mrqc_reg);
3565 if (!(mrqc & IXGBE_MRQC_RSSEN)) { /* RSS disabled */
3566 if (rss_hf != 0) /* Enable RSS */
3568 return 0; /* Nothing to do */
3571 if (rss_hf == 0) /* Disable RSS */
3573 ixgbe_hw_rss_hash_set(hw, rss_conf);
3578 ixgbe_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
3579 struct rte_eth_rss_conf *rss_conf)
3581 struct ixgbe_hw *hw;
3590 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3591 mrqc_reg = ixgbe_mrqc_reg_get(hw->mac.type);
3592 rssrk_reg = ixgbe_rssrk_reg_get(hw->mac.type, 0);
3593 hash_key = rss_conf->rss_key;
3594 if (hash_key != NULL) {
3595 /* Return RSS hash key */
3596 for (i = 0; i < 10; i++) {
3597 rss_key = IXGBE_READ_REG_ARRAY(hw, rssrk_reg, i);
3598 hash_key[(i * 4)] = rss_key & 0x000000FF;
3599 hash_key[(i * 4) + 1] = (rss_key >> 8) & 0x000000FF;
3600 hash_key[(i * 4) + 2] = (rss_key >> 16) & 0x000000FF;
3601 hash_key[(i * 4) + 3] = (rss_key >> 24) & 0x000000FF;
3605 /* Get RSS functions configured in MRQC register */
3606 mrqc = IXGBE_READ_REG(hw, mrqc_reg);
3607 if ((mrqc & IXGBE_MRQC_RSSEN) == 0) { /* RSS is disabled */
3608 rss_conf->rss_hf = 0;
3612 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV4)
3613 rss_hf |= ETH_RSS_IPV4;
3614 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV4_TCP)
3615 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP;
3616 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6)
3617 rss_hf |= ETH_RSS_IPV6;
3618 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_EX)
3619 rss_hf |= ETH_RSS_IPV6_EX;
3620 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_TCP)
3621 rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP;
3622 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP)
3623 rss_hf |= ETH_RSS_IPV6_TCP_EX;
3624 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV4_UDP)
3625 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP;
3626 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_UDP)
3627 rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP;
3628 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP)
3629 rss_hf |= ETH_RSS_IPV6_UDP_EX;
3630 rss_conf->rss_hf = rss_hf;
3635 ixgbe_rss_configure(struct rte_eth_dev *dev)
3637 struct rte_eth_rss_conf rss_conf;
3638 struct ixgbe_adapter *adapter;
3639 struct ixgbe_hw *hw;
3643 uint16_t sp_reta_size;
3646 PMD_INIT_FUNC_TRACE();
3647 adapter = dev->data->dev_private;
3648 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3650 sp_reta_size = ixgbe_reta_size_get(hw->mac.type);
3653 * Fill in redirection table
3654 * The byte-swap is needed because NIC registers are in
3655 * little-endian order.
3657 if (adapter->rss_reta_updated == 0) {
3659 for (i = 0, j = 0; i < sp_reta_size; i++, j++) {
3660 reta_reg = ixgbe_reta_reg_get(hw->mac.type, i);
3662 if (j == dev->data->nb_rx_queues)
3664 reta = (reta << 8) | j;
3666 IXGBE_WRITE_REG(hw, reta_reg,
3672 * Configure the RSS key and the RSS protocols used to compute
3673 * the RSS hash of input packets.
3675 rss_conf = dev->data->dev_conf.rx_adv_conf.rss_conf;
3676 if ((rss_conf.rss_hf & IXGBE_RSS_OFFLOAD_ALL) == 0) {
3677 ixgbe_rss_disable(dev);
3680 if (rss_conf.rss_key == NULL)
3681 rss_conf.rss_key = rss_intel_key; /* Default hash key */
3682 ixgbe_hw_rss_hash_set(hw, &rss_conf);
3685 #define NUM_VFTA_REGISTERS 128
3686 #define NIC_RX_BUFFER_SIZE 0x200
3687 #define X550_RX_BUFFER_SIZE 0x180
3690 ixgbe_vmdq_dcb_configure(struct rte_eth_dev *dev)
3692 struct rte_eth_vmdq_dcb_conf *cfg;
3693 struct ixgbe_hw *hw;
3694 enum rte_eth_nb_pools num_pools;
3695 uint32_t mrqc, vt_ctl, queue_mapping, vlanctrl;
3697 uint8_t nb_tcs; /* number of traffic classes */
3700 PMD_INIT_FUNC_TRACE();
3701 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3702 cfg = &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf;
3703 num_pools = cfg->nb_queue_pools;
3704 /* Check we have a valid number of pools */
3705 if (num_pools != ETH_16_POOLS && num_pools != ETH_32_POOLS) {
3706 ixgbe_rss_disable(dev);
3709 /* 16 pools -> 8 traffic classes, 32 pools -> 4 traffic classes */
3710 nb_tcs = (uint8_t)(ETH_VMDQ_DCB_NUM_QUEUES / (int)num_pools);
3714 * split rx buffer up into sections, each for 1 traffic class
3716 switch (hw->mac.type) {
3717 case ixgbe_mac_X550:
3718 case ixgbe_mac_X550EM_x:
3719 case ixgbe_mac_X550EM_a:
3720 pbsize = (uint16_t)(X550_RX_BUFFER_SIZE / nb_tcs);
3723 pbsize = (uint16_t)(NIC_RX_BUFFER_SIZE / nb_tcs);
3726 for (i = 0; i < nb_tcs; i++) {
3727 uint32_t rxpbsize = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(i));
3729 rxpbsize &= (~(0x3FF << IXGBE_RXPBSIZE_SHIFT));
3730 /* clear 10 bits. */
3731 rxpbsize |= (pbsize << IXGBE_RXPBSIZE_SHIFT); /* set value */
3732 IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), rxpbsize);
3734 /* zero alloc all unused TCs */
3735 for (i = nb_tcs; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3736 uint32_t rxpbsize = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(i));
3738 rxpbsize &= (~(0x3FF << IXGBE_RXPBSIZE_SHIFT));
3739 /* clear 10 bits. */
3740 IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), rxpbsize);
3743 /* MRQC: enable vmdq and dcb */
3744 mrqc = (num_pools == ETH_16_POOLS) ?
3745 IXGBE_MRQC_VMDQRT8TCEN : IXGBE_MRQC_VMDQRT4TCEN;
3746 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
3748 /* PFVTCTL: turn on virtualisation and set the default pool */
3749 vt_ctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
3750 if (cfg->enable_default_pool) {
3751 vt_ctl |= (cfg->default_pool << IXGBE_VT_CTL_POOL_SHIFT);
3753 vt_ctl |= IXGBE_VT_CTL_DIS_DEFPL;
3756 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vt_ctl);
3758 /* RTRUP2TC: mapping user priorities to traffic classes (TCs) */
3760 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++)
3762 * mapping is done with 3 bits per priority,
3763 * so shift by i*3 each time
3765 queue_mapping |= ((cfg->dcb_tc[i] & 0x07) << (i * 3));
3767 IXGBE_WRITE_REG(hw, IXGBE_RTRUP2TC, queue_mapping);
3769 /* RTRPCS: DCB related */
3770 IXGBE_WRITE_REG(hw, IXGBE_RTRPCS, IXGBE_RMCS_RRM);
3772 /* VLNCTRL: enable vlan filtering and allow all vlan tags through */
3773 vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
3774 vlanctrl |= IXGBE_VLNCTRL_VFE; /* enable vlan filters */
3775 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl);
3777 /* VFTA - enable all vlan filters */
3778 for (i = 0; i < NUM_VFTA_REGISTERS; i++) {
3779 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), 0xFFFFFFFF);
3782 /* VFRE: pool enabling for receive - 16 or 32 */
3783 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0),
3784 num_pools == ETH_16_POOLS ? 0xFFFF : 0xFFFFFFFF);
3787 * MPSAR - allow pools to read specific mac addresses
3788 * In this case, all pools should be able to read from mac addr 0
3790 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(0), 0xFFFFFFFF);
3791 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(0), 0xFFFFFFFF);
3793 /* PFVLVF, PFVLVFB: set up filters for vlan tags as configured */
3794 for (i = 0; i < cfg->nb_pool_maps; i++) {
3795 /* set vlan id in VF register and set the valid bit */
3796 IXGBE_WRITE_REG(hw, IXGBE_VLVF(i), (IXGBE_VLVF_VIEN |
3797 (cfg->pool_map[i].vlan_id & 0xFFF)));
3799 * Put the allowed pools in VFB reg. As we only have 16 or 32
3800 * pools, we only need to use the first half of the register
3803 IXGBE_WRITE_REG(hw, IXGBE_VLVFB(i*2), cfg->pool_map[i].pools);
3808 * ixgbe_dcb_config_tx_hw_config - Configure general DCB TX parameters
3809 * @dev: pointer to eth_dev structure
3810 * @dcb_config: pointer to ixgbe_dcb_config structure
3813 ixgbe_dcb_tx_hw_config(struct rte_eth_dev *dev,
3814 struct ixgbe_dcb_config *dcb_config)
3817 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3819 PMD_INIT_FUNC_TRACE();
3820 if (hw->mac.type != ixgbe_mac_82598EB) {
3821 /* Disable the Tx desc arbiter so that MTQC can be changed */
3822 reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3823 reg |= IXGBE_RTTDCS_ARBDIS;
3824 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg);
3826 /* Enable DCB for Tx with 8 TCs */
3827 if (dcb_config->num_tcs.pg_tcs == 8) {
3828 reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;
3830 reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
3832 if (dcb_config->vt_mode)
3833 reg |= IXGBE_MTQC_VT_ENA;
3834 IXGBE_WRITE_REG(hw, IXGBE_MTQC, reg);
3836 /* Enable the Tx desc arbiter */
3837 reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3838 reg &= ~IXGBE_RTTDCS_ARBDIS;
3839 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg);
3841 /* Enable Security TX Buffer IFG for DCB */
3842 reg = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG);
3843 reg |= IXGBE_SECTX_DCB;
3844 IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, reg);
3849 * ixgbe_vmdq_dcb_hw_tx_config - Configure general VMDQ+DCB TX parameters
3850 * @dev: pointer to rte_eth_dev structure
3851 * @dcb_config: pointer to ixgbe_dcb_config structure
3854 ixgbe_vmdq_dcb_hw_tx_config(struct rte_eth_dev *dev,
3855 struct ixgbe_dcb_config *dcb_config)
3857 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3858 &dev->data->dev_conf.tx_adv_conf.vmdq_dcb_tx_conf;
3859 struct ixgbe_hw *hw =
3860 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3862 PMD_INIT_FUNC_TRACE();
3863 if (hw->mac.type != ixgbe_mac_82598EB)
3864 /*PF VF Transmit Enable*/
3865 IXGBE_WRITE_REG(hw, IXGBE_VFTE(0),
3866 vmdq_tx_conf->nb_queue_pools == ETH_16_POOLS ? 0xFFFF : 0xFFFFFFFF);
3868 /*Configure general DCB TX parameters*/
3869 ixgbe_dcb_tx_hw_config(dev, dcb_config);
3873 ixgbe_vmdq_dcb_rx_config(struct rte_eth_dev *dev,
3874 struct ixgbe_dcb_config *dcb_config)
3876 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3877 &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf;
3878 struct ixgbe_dcb_tc_config *tc;
3881 /* convert rte_eth_conf.rx_adv_conf to struct ixgbe_dcb_config */
3882 if (vmdq_rx_conf->nb_queue_pools == ETH_16_POOLS) {
3883 dcb_config->num_tcs.pg_tcs = ETH_8_TCS;
3884 dcb_config->num_tcs.pfc_tcs = ETH_8_TCS;
3886 dcb_config->num_tcs.pg_tcs = ETH_4_TCS;
3887 dcb_config->num_tcs.pfc_tcs = ETH_4_TCS;
3890 /* Initialize User Priority to Traffic Class mapping */
3891 for (j = 0; j < IXGBE_DCB_MAX_TRAFFIC_CLASS; j++) {
3892 tc = &dcb_config->tc_config[j];
3893 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0;
3896 /* User Priority to Traffic Class mapping */
3897 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3898 j = vmdq_rx_conf->dcb_tc[i];
3899 tc = &dcb_config->tc_config[j];
3900 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap |=
3906 ixgbe_dcb_vt_tx_config(struct rte_eth_dev *dev,
3907 struct ixgbe_dcb_config *dcb_config)
3909 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3910 &dev->data->dev_conf.tx_adv_conf.vmdq_dcb_tx_conf;
3911 struct ixgbe_dcb_tc_config *tc;
3914 /* convert rte_eth_conf.rx_adv_conf to struct ixgbe_dcb_config */
3915 if (vmdq_tx_conf->nb_queue_pools == ETH_16_POOLS) {
3916 dcb_config->num_tcs.pg_tcs = ETH_8_TCS;
3917 dcb_config->num_tcs.pfc_tcs = ETH_8_TCS;
3919 dcb_config->num_tcs.pg_tcs = ETH_4_TCS;
3920 dcb_config->num_tcs.pfc_tcs = ETH_4_TCS;
3923 /* Initialize User Priority to Traffic Class mapping */
3924 for (j = 0; j < IXGBE_DCB_MAX_TRAFFIC_CLASS; j++) {
3925 tc = &dcb_config->tc_config[j];
3926 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0;
3929 /* User Priority to Traffic Class mapping */
3930 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3931 j = vmdq_tx_conf->dcb_tc[i];
3932 tc = &dcb_config->tc_config[j];
3933 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap |=
3939 ixgbe_dcb_rx_config(struct rte_eth_dev *dev,
3940 struct ixgbe_dcb_config *dcb_config)
3942 struct rte_eth_dcb_rx_conf *rx_conf =
3943 &dev->data->dev_conf.rx_adv_conf.dcb_rx_conf;
3944 struct ixgbe_dcb_tc_config *tc;
3947 dcb_config->num_tcs.pg_tcs = (uint8_t)rx_conf->nb_tcs;
3948 dcb_config->num_tcs.pfc_tcs = (uint8_t)rx_conf->nb_tcs;
3950 /* Initialize User Priority to Traffic Class mapping */
3951 for (j = 0; j < IXGBE_DCB_MAX_TRAFFIC_CLASS; j++) {
3952 tc = &dcb_config->tc_config[j];
3953 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0;
3956 /* User Priority to Traffic Class mapping */
3957 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3958 j = rx_conf->dcb_tc[i];
3959 tc = &dcb_config->tc_config[j];
3960 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap |=
3966 ixgbe_dcb_tx_config(struct rte_eth_dev *dev,
3967 struct ixgbe_dcb_config *dcb_config)
3969 struct rte_eth_dcb_tx_conf *tx_conf =
3970 &dev->data->dev_conf.tx_adv_conf.dcb_tx_conf;
3971 struct ixgbe_dcb_tc_config *tc;
3974 dcb_config->num_tcs.pg_tcs = (uint8_t)tx_conf->nb_tcs;
3975 dcb_config->num_tcs.pfc_tcs = (uint8_t)tx_conf->nb_tcs;
3977 /* Initialize User Priority to Traffic Class mapping */
3978 for (j = 0; j < IXGBE_DCB_MAX_TRAFFIC_CLASS; j++) {
3979 tc = &dcb_config->tc_config[j];
3980 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0;
3983 /* User Priority to Traffic Class mapping */
3984 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3985 j = tx_conf->dcb_tc[i];
3986 tc = &dcb_config->tc_config[j];
3987 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap |=
3993 * ixgbe_dcb_rx_hw_config - Configure general DCB RX HW parameters
3994 * @dev: pointer to eth_dev structure
3995 * @dcb_config: pointer to ixgbe_dcb_config structure
3998 ixgbe_dcb_rx_hw_config(struct rte_eth_dev *dev,
3999 struct ixgbe_dcb_config *dcb_config)
4005 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4007 PMD_INIT_FUNC_TRACE();
4009 * Disable the arbiter before changing parameters
4010 * (always enable recycle mode; WSP)
4012 reg = IXGBE_RTRPCS_RRM | IXGBE_RTRPCS_RAC | IXGBE_RTRPCS_ARBDIS;
4013 IXGBE_WRITE_REG(hw, IXGBE_RTRPCS, reg);
4015 if (hw->mac.type != ixgbe_mac_82598EB) {
4016 reg = IXGBE_READ_REG(hw, IXGBE_MRQC);
4017 if (dcb_config->num_tcs.pg_tcs == 4) {
4018 if (dcb_config->vt_mode)
4019 reg = (reg & ~IXGBE_MRQC_MRQE_MASK) |
4020 IXGBE_MRQC_VMDQRT4TCEN;
4022 /* no matter the mode is DCB or DCB_RSS, just
4023 * set the MRQE to RSSXTCEN. RSS is controlled
4026 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, 0);
4027 reg = (reg & ~IXGBE_MRQC_MRQE_MASK) |
4028 IXGBE_MRQC_RTRSS4TCEN;
4031 if (dcb_config->num_tcs.pg_tcs == 8) {
4032 if (dcb_config->vt_mode)
4033 reg = (reg & ~IXGBE_MRQC_MRQE_MASK) |
4034 IXGBE_MRQC_VMDQRT8TCEN;
4036 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, 0);
4037 reg = (reg & ~IXGBE_MRQC_MRQE_MASK) |
4038 IXGBE_MRQC_RTRSS8TCEN;
4042 IXGBE_WRITE_REG(hw, IXGBE_MRQC, reg);
4044 if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
4045 /* Disable drop for all queues in VMDQ mode*/
4046 for (q = 0; q < IXGBE_MAX_RX_QUEUE_NUM; q++)
4047 IXGBE_WRITE_REG(hw, IXGBE_QDE,
4049 (q << IXGBE_QDE_IDX_SHIFT)));
4051 /* Enable drop for all queues in SRIOV mode */
4052 for (q = 0; q < IXGBE_MAX_RX_QUEUE_NUM; q++)
4053 IXGBE_WRITE_REG(hw, IXGBE_QDE,
4055 (q << IXGBE_QDE_IDX_SHIFT) |
4060 /* VLNCTRL: enable vlan filtering and allow all vlan tags through */
4061 vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4062 vlanctrl |= IXGBE_VLNCTRL_VFE; /* enable vlan filters */
4063 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl);
4065 /* VFTA - enable all vlan filters */
4066 for (i = 0; i < NUM_VFTA_REGISTERS; i++) {
4067 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), 0xFFFFFFFF);
4071 * Configure Rx packet plane (recycle mode; WSP) and
4074 reg = IXGBE_RTRPCS_RRM | IXGBE_RTRPCS_RAC;
4075 IXGBE_WRITE_REG(hw, IXGBE_RTRPCS, reg);
4079 ixgbe_dcb_hw_arbite_rx_config(struct ixgbe_hw *hw, uint16_t *refill,
4080 uint16_t *max, uint8_t *bwg_id, uint8_t *tsa, uint8_t *map)
4082 switch (hw->mac.type) {
4083 case ixgbe_mac_82598EB:
4084 ixgbe_dcb_config_rx_arbiter_82598(hw, refill, max, tsa);
4086 case ixgbe_mac_82599EB:
4087 case ixgbe_mac_X540:
4088 case ixgbe_mac_X550:
4089 case ixgbe_mac_X550EM_x:
4090 case ixgbe_mac_X550EM_a:
4091 ixgbe_dcb_config_rx_arbiter_82599(hw, refill, max, bwg_id,
4100 ixgbe_dcb_hw_arbite_tx_config(struct ixgbe_hw *hw, uint16_t *refill, uint16_t *max,
4101 uint8_t *bwg_id, uint8_t *tsa, uint8_t *map)
4103 switch (hw->mac.type) {
4104 case ixgbe_mac_82598EB:
4105 ixgbe_dcb_config_tx_desc_arbiter_82598(hw, refill, max, bwg_id, tsa);
4106 ixgbe_dcb_config_tx_data_arbiter_82598(hw, refill, max, bwg_id, tsa);
4108 case ixgbe_mac_82599EB:
4109 case ixgbe_mac_X540:
4110 case ixgbe_mac_X550:
4111 case ixgbe_mac_X550EM_x:
4112 case ixgbe_mac_X550EM_a:
4113 ixgbe_dcb_config_tx_desc_arbiter_82599(hw, refill, max, bwg_id, tsa);
4114 ixgbe_dcb_config_tx_data_arbiter_82599(hw, refill, max, bwg_id, tsa, map);
4121 #define DCB_RX_CONFIG 1
4122 #define DCB_TX_CONFIG 1
4123 #define DCB_TX_PB 1024
4125 * ixgbe_dcb_hw_configure - Enable DCB and configure
4126 * general DCB in VT mode and non-VT mode parameters
4127 * @dev: pointer to rte_eth_dev structure
4128 * @dcb_config: pointer to ixgbe_dcb_config structure
4131 ixgbe_dcb_hw_configure(struct rte_eth_dev *dev,
4132 struct ixgbe_dcb_config *dcb_config)
4135 uint8_t i, pfc_en, nb_tcs;
4136 uint16_t pbsize, rx_buffer_size;
4137 uint8_t config_dcb_rx = 0;
4138 uint8_t config_dcb_tx = 0;
4139 uint8_t tsa[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0};
4140 uint8_t bwgid[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0};
4141 uint16_t refill[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0};
4142 uint16_t max[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0};
4143 uint8_t map[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0};
4144 struct ixgbe_dcb_tc_config *tc;
4145 uint32_t max_frame = dev->data->mtu + RTE_ETHER_HDR_LEN +
4147 struct ixgbe_hw *hw =
4148 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4149 struct ixgbe_bw_conf *bw_conf =
4150 IXGBE_DEV_PRIVATE_TO_BW_CONF(dev->data->dev_private);
4152 switch (dev->data->dev_conf.rxmode.mq_mode) {
4153 case ETH_MQ_RX_VMDQ_DCB:
4154 dcb_config->vt_mode = true;
4155 if (hw->mac.type != ixgbe_mac_82598EB) {
4156 config_dcb_rx = DCB_RX_CONFIG;
4158 *get dcb and VT rx configuration parameters
4161 ixgbe_vmdq_dcb_rx_config(dev, dcb_config);
4162 /*Configure general VMDQ and DCB RX parameters*/
4163 ixgbe_vmdq_dcb_configure(dev);
4167 case ETH_MQ_RX_DCB_RSS:
4168 dcb_config->vt_mode = false;
4169 config_dcb_rx = DCB_RX_CONFIG;
4170 /* Get dcb TX configuration parameters from rte_eth_conf */
4171 ixgbe_dcb_rx_config(dev, dcb_config);
4172 /*Configure general DCB RX parameters*/
4173 ixgbe_dcb_rx_hw_config(dev, dcb_config);
4176 PMD_INIT_LOG(ERR, "Incorrect DCB RX mode configuration");
4179 switch (dev->data->dev_conf.txmode.mq_mode) {
4180 case ETH_MQ_TX_VMDQ_DCB:
4181 dcb_config->vt_mode = true;
4182 config_dcb_tx = DCB_TX_CONFIG;
4183 /* get DCB and VT TX configuration parameters
4186 ixgbe_dcb_vt_tx_config(dev, dcb_config);
4187 /*Configure general VMDQ and DCB TX parameters*/
4188 ixgbe_vmdq_dcb_hw_tx_config(dev, dcb_config);
4192 dcb_config->vt_mode = false;
4193 config_dcb_tx = DCB_TX_CONFIG;
4194 /*get DCB TX configuration parameters from rte_eth_conf*/
4195 ixgbe_dcb_tx_config(dev, dcb_config);
4196 /*Configure general DCB TX parameters*/
4197 ixgbe_dcb_tx_hw_config(dev, dcb_config);
4200 PMD_INIT_LOG(ERR, "Incorrect DCB TX mode configuration");
4204 nb_tcs = dcb_config->num_tcs.pfc_tcs;
4206 ixgbe_dcb_unpack_map_cee(dcb_config, IXGBE_DCB_RX_CONFIG, map);
4207 if (nb_tcs == ETH_4_TCS) {
4208 /* Avoid un-configured priority mapping to TC0 */
4210 uint8_t mask = 0xFF;
4212 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES - 4; i++)
4213 mask = (uint8_t)(mask & (~(1 << map[i])));
4214 for (i = 0; mask && (i < IXGBE_DCB_MAX_TRAFFIC_CLASS); i++) {
4215 if ((mask & 0x1) && (j < ETH_DCB_NUM_USER_PRIORITIES))
4219 /* Re-configure 4 TCs BW */
4220 for (i = 0; i < nb_tcs; i++) {
4221 tc = &dcb_config->tc_config[i];
4222 if (bw_conf->tc_num != nb_tcs)
4223 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent =
4224 (uint8_t)(100 / nb_tcs);
4225 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent =
4226 (uint8_t)(100 / nb_tcs);
4228 for (; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
4229 tc = &dcb_config->tc_config[i];
4230 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent = 0;
4231 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent = 0;
4234 /* Re-configure 8 TCs BW */
4235 for (i = 0; i < nb_tcs; i++) {
4236 tc = &dcb_config->tc_config[i];
4237 if (bw_conf->tc_num != nb_tcs)
4238 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent =
4239 (uint8_t)(100 / nb_tcs + (i & 1));
4240 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent =
4241 (uint8_t)(100 / nb_tcs + (i & 1));
4245 switch (hw->mac.type) {
4246 case ixgbe_mac_X550:
4247 case ixgbe_mac_X550EM_x:
4248 case ixgbe_mac_X550EM_a:
4249 rx_buffer_size = X550_RX_BUFFER_SIZE;
4252 rx_buffer_size = NIC_RX_BUFFER_SIZE;
4256 if (config_dcb_rx) {
4257 /* Set RX buffer size */
4258 pbsize = (uint16_t)(rx_buffer_size / nb_tcs);
4259 uint32_t rxpbsize = pbsize << IXGBE_RXPBSIZE_SHIFT;
4261 for (i = 0; i < nb_tcs; i++) {
4262 IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), rxpbsize);
4264 /* zero alloc all unused TCs */
4265 for (; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
4266 IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), 0);
4269 if (config_dcb_tx) {
4270 /* Only support an equally distributed
4271 * Tx packet buffer strategy.
4273 uint32_t txpktsize = IXGBE_TXPBSIZE_MAX / nb_tcs;
4274 uint32_t txpbthresh = (txpktsize / DCB_TX_PB) - IXGBE_TXPKT_SIZE_MAX;
4276 for (i = 0; i < nb_tcs; i++) {
4277 IXGBE_WRITE_REG(hw, IXGBE_TXPBSIZE(i), txpktsize);
4278 IXGBE_WRITE_REG(hw, IXGBE_TXPBTHRESH(i), txpbthresh);
4280 /* Clear unused TCs, if any, to zero buffer size*/
4281 for (; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
4282 IXGBE_WRITE_REG(hw, IXGBE_TXPBSIZE(i), 0);
4283 IXGBE_WRITE_REG(hw, IXGBE_TXPBTHRESH(i), 0);
4287 /*Calculates traffic class credits*/
4288 ixgbe_dcb_calculate_tc_credits_cee(hw, dcb_config, max_frame,
4289 IXGBE_DCB_TX_CONFIG);
4290 ixgbe_dcb_calculate_tc_credits_cee(hw, dcb_config, max_frame,
4291 IXGBE_DCB_RX_CONFIG);
4293 if (config_dcb_rx) {
4294 /* Unpack CEE standard containers */
4295 ixgbe_dcb_unpack_refill_cee(dcb_config, IXGBE_DCB_RX_CONFIG, refill);
4296 ixgbe_dcb_unpack_max_cee(dcb_config, max);
4297 ixgbe_dcb_unpack_bwgid_cee(dcb_config, IXGBE_DCB_RX_CONFIG, bwgid);
4298 ixgbe_dcb_unpack_tsa_cee(dcb_config, IXGBE_DCB_RX_CONFIG, tsa);
4299 /* Configure PG(ETS) RX */
4300 ixgbe_dcb_hw_arbite_rx_config(hw, refill, max, bwgid, tsa, map);
4303 if (config_dcb_tx) {
4304 /* Unpack CEE standard containers */
4305 ixgbe_dcb_unpack_refill_cee(dcb_config, IXGBE_DCB_TX_CONFIG, refill);
4306 ixgbe_dcb_unpack_max_cee(dcb_config, max);
4307 ixgbe_dcb_unpack_bwgid_cee(dcb_config, IXGBE_DCB_TX_CONFIG, bwgid);
4308 ixgbe_dcb_unpack_tsa_cee(dcb_config, IXGBE_DCB_TX_CONFIG, tsa);
4309 /* Configure PG(ETS) TX */
4310 ixgbe_dcb_hw_arbite_tx_config(hw, refill, max, bwgid, tsa, map);
4313 /*Configure queue statistics registers*/
4314 ixgbe_dcb_config_tc_stats_82599(hw, dcb_config);
4316 /* Check if the PFC is supported */
4317 if (dev->data->dev_conf.dcb_capability_en & ETH_DCB_PFC_SUPPORT) {
4318 pbsize = (uint16_t)(rx_buffer_size / nb_tcs);
4319 for (i = 0; i < nb_tcs; i++) {
4321 * If the TC count is 8,and the default high_water is 48,
4322 * the low_water is 16 as default.
4324 hw->fc.high_water[i] = (pbsize * 3) / 4;
4325 hw->fc.low_water[i] = pbsize / 4;
4326 /* Enable pfc for this TC */
4327 tc = &dcb_config->tc_config[i];
4328 tc->pfc = ixgbe_dcb_pfc_enabled;
4330 ixgbe_dcb_unpack_pfc_cee(dcb_config, map, &pfc_en);
4331 if (dcb_config->num_tcs.pfc_tcs == ETH_4_TCS)
4333 ret = ixgbe_dcb_config_pfc(hw, pfc_en, map);
4340 * ixgbe_configure_dcb - Configure DCB Hardware
4341 * @dev: pointer to rte_eth_dev
4343 void ixgbe_configure_dcb(struct rte_eth_dev *dev)
4345 struct ixgbe_dcb_config *dcb_cfg =
4346 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private);
4347 struct rte_eth_conf *dev_conf = &(dev->data->dev_conf);
4349 PMD_INIT_FUNC_TRACE();
4351 /* check support mq_mode for DCB */
4352 if ((dev_conf->rxmode.mq_mode != ETH_MQ_RX_VMDQ_DCB) &&
4353 (dev_conf->rxmode.mq_mode != ETH_MQ_RX_DCB) &&
4354 (dev_conf->rxmode.mq_mode != ETH_MQ_RX_DCB_RSS))
4357 if (dev->data->nb_rx_queues > ETH_DCB_NUM_QUEUES)
4360 /** Configure DCB hardware **/
4361 ixgbe_dcb_hw_configure(dev, dcb_cfg);
4365 * VMDq only support for 10 GbE NIC.
4368 ixgbe_vmdq_rx_hw_configure(struct rte_eth_dev *dev)
4370 struct rte_eth_vmdq_rx_conf *cfg;
4371 struct ixgbe_hw *hw;
4372 enum rte_eth_nb_pools num_pools;
4373 uint32_t mrqc, vt_ctl, vlanctrl;
4377 PMD_INIT_FUNC_TRACE();
4378 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4379 cfg = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
4380 num_pools = cfg->nb_queue_pools;
4382 ixgbe_rss_disable(dev);
4384 /* MRQC: enable vmdq */
4385 mrqc = IXGBE_MRQC_VMDQEN;
4386 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4388 /* PFVTCTL: turn on virtualisation and set the default pool */
4389 vt_ctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
4390 if (cfg->enable_default_pool)
4391 vt_ctl |= (cfg->default_pool << IXGBE_VT_CTL_POOL_SHIFT);
4393 vt_ctl |= IXGBE_VT_CTL_DIS_DEFPL;
4395 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vt_ctl);
4397 for (i = 0; i < (int)num_pools; i++) {
4398 vmolr = ixgbe_convert_vm_rx_mask_to_val(cfg->rx_mode, vmolr);
4399 IXGBE_WRITE_REG(hw, IXGBE_VMOLR(i), vmolr);
4402 /* VLNCTRL: enable vlan filtering and allow all vlan tags through */
4403 vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4404 vlanctrl |= IXGBE_VLNCTRL_VFE; /* enable vlan filters */
4405 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl);
4407 /* VFTA - enable all vlan filters */
4408 for (i = 0; i < NUM_VFTA_REGISTERS; i++)
4409 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), UINT32_MAX);
4411 /* VFRE: pool enabling for receive - 64 */
4412 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), UINT32_MAX);
4413 if (num_pools == ETH_64_POOLS)
4414 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), UINT32_MAX);
4417 * MPSAR - allow pools to read specific mac addresses
4418 * In this case, all pools should be able to read from mac addr 0
4420 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(0), UINT32_MAX);
4421 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(0), UINT32_MAX);
4423 /* PFVLVF, PFVLVFB: set up filters for vlan tags as configured */
4424 for (i = 0; i < cfg->nb_pool_maps; i++) {
4425 /* set vlan id in VF register and set the valid bit */
4426 IXGBE_WRITE_REG(hw, IXGBE_VLVF(i), (IXGBE_VLVF_VIEN |
4427 (cfg->pool_map[i].vlan_id & IXGBE_RXD_VLAN_ID_MASK)));
4429 * Put the allowed pools in VFB reg. As we only have 16 or 64
4430 * pools, we only need to use the first half of the register
4433 if (((cfg->pool_map[i].pools >> 32) & UINT32_MAX) == 0)
4434 IXGBE_WRITE_REG(hw, IXGBE_VLVFB(i * 2),
4435 (cfg->pool_map[i].pools & UINT32_MAX));
4437 IXGBE_WRITE_REG(hw, IXGBE_VLVFB((i * 2 + 1)),
4438 ((cfg->pool_map[i].pools >> 32) & UINT32_MAX));
4442 /* PFDMA Tx General Switch Control Enables VMDQ loopback */
4443 if (cfg->enable_loop_back) {
4444 IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN);
4445 for (i = 0; i < RTE_IXGBE_VMTXSW_REGISTER_COUNT; i++)
4446 IXGBE_WRITE_REG(hw, IXGBE_VMTXSW(i), UINT32_MAX);
4449 IXGBE_WRITE_FLUSH(hw);
4453 * ixgbe_dcb_config_tx_hw_config - Configure general VMDq TX parameters
4454 * @hw: pointer to hardware structure
4457 ixgbe_vmdq_tx_hw_configure(struct ixgbe_hw *hw)
4462 PMD_INIT_FUNC_TRACE();
4463 /*PF VF Transmit Enable*/
4464 IXGBE_WRITE_REG(hw, IXGBE_VFTE(0), UINT32_MAX);
4465 IXGBE_WRITE_REG(hw, IXGBE_VFTE(1), UINT32_MAX);
4467 /* Disable the Tx desc arbiter so that MTQC can be changed */
4468 reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
4469 reg |= IXGBE_RTTDCS_ARBDIS;
4470 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg);
4472 reg = IXGBE_MTQC_VT_ENA | IXGBE_MTQC_64VF;
4473 IXGBE_WRITE_REG(hw, IXGBE_MTQC, reg);
4475 /* Disable drop for all queues */
4476 for (q = 0; q < IXGBE_MAX_RX_QUEUE_NUM; q++)
4477 IXGBE_WRITE_REG(hw, IXGBE_QDE,
4478 (IXGBE_QDE_WRITE | (q << IXGBE_QDE_IDX_SHIFT)));
4480 /* Enable the Tx desc arbiter */
4481 reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
4482 reg &= ~IXGBE_RTTDCS_ARBDIS;
4483 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg);
4485 IXGBE_WRITE_FLUSH(hw);
4488 static int __rte_cold
4489 ixgbe_alloc_rx_queue_mbufs(struct ixgbe_rx_queue *rxq)
4491 struct ixgbe_rx_entry *rxe = rxq->sw_ring;
4495 /* Initialize software ring entries */
4496 for (i = 0; i < rxq->nb_rx_desc; i++) {
4497 volatile union ixgbe_adv_rx_desc *rxd;
4498 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mb_pool);
4501 PMD_INIT_LOG(ERR, "RX mbuf alloc failed queue_id=%u",
4502 (unsigned) rxq->queue_id);
4506 mbuf->data_off = RTE_PKTMBUF_HEADROOM;
4507 mbuf->port = rxq->port_id;
4510 rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf));
4511 rxd = &rxq->rx_ring[i];
4512 rxd->read.hdr_addr = 0;
4513 rxd->read.pkt_addr = dma_addr;
4521 ixgbe_config_vf_rss(struct rte_eth_dev *dev)
4523 struct ixgbe_hw *hw;
4526 ixgbe_rss_configure(dev);
4528 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4530 /* MRQC: enable VF RSS */
4531 mrqc = IXGBE_READ_REG(hw, IXGBE_MRQC);
4532 mrqc &= ~IXGBE_MRQC_MRQE_MASK;
4533 switch (RTE_ETH_DEV_SRIOV(dev).active) {
4535 mrqc |= IXGBE_MRQC_VMDQRSS64EN;
4539 mrqc |= IXGBE_MRQC_VMDQRSS32EN;
4543 PMD_INIT_LOG(ERR, "Invalid pool number in IOV mode with VMDQ RSS");
4547 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4553 ixgbe_config_vf_default(struct rte_eth_dev *dev)
4555 struct ixgbe_hw *hw =
4556 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4558 switch (RTE_ETH_DEV_SRIOV(dev).active) {
4560 IXGBE_WRITE_REG(hw, IXGBE_MRQC,
4565 IXGBE_WRITE_REG(hw, IXGBE_MRQC,
4566 IXGBE_MRQC_VMDQRT4TCEN);
4570 IXGBE_WRITE_REG(hw, IXGBE_MRQC,
4571 IXGBE_MRQC_VMDQRT8TCEN);
4575 "invalid pool number in IOV mode");
4582 ixgbe_dev_mq_rx_configure(struct rte_eth_dev *dev)
4584 struct ixgbe_hw *hw =
4585 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4587 if (hw->mac.type == ixgbe_mac_82598EB)
4590 if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
4592 * SRIOV inactive scheme
4593 * any DCB/RSS w/o VMDq multi-queue setting
4595 switch (dev->data->dev_conf.rxmode.mq_mode) {
4597 case ETH_MQ_RX_DCB_RSS:
4598 case ETH_MQ_RX_VMDQ_RSS:
4599 ixgbe_rss_configure(dev);
4602 case ETH_MQ_RX_VMDQ_DCB:
4603 ixgbe_vmdq_dcb_configure(dev);
4606 case ETH_MQ_RX_VMDQ_ONLY:
4607 ixgbe_vmdq_rx_hw_configure(dev);
4610 case ETH_MQ_RX_NONE:
4612 /* if mq_mode is none, disable rss mode.*/
4613 ixgbe_rss_disable(dev);
4617 /* SRIOV active scheme
4618 * Support RSS together with SRIOV.
4620 switch (dev->data->dev_conf.rxmode.mq_mode) {
4622 case ETH_MQ_RX_VMDQ_RSS:
4623 ixgbe_config_vf_rss(dev);
4625 case ETH_MQ_RX_VMDQ_DCB:
4627 /* In SRIOV, the configuration is the same as VMDq case */
4628 ixgbe_vmdq_dcb_configure(dev);
4630 /* DCB/RSS together with SRIOV is not supported */
4631 case ETH_MQ_RX_VMDQ_DCB_RSS:
4632 case ETH_MQ_RX_DCB_RSS:
4634 "Could not support DCB/RSS with VMDq & SRIOV");
4637 ixgbe_config_vf_default(dev);
4646 ixgbe_dev_mq_tx_configure(struct rte_eth_dev *dev)
4648 struct ixgbe_hw *hw =
4649 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4653 if (hw->mac.type == ixgbe_mac_82598EB)
4656 /* disable arbiter before setting MTQC */
4657 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
4658 rttdcs |= IXGBE_RTTDCS_ARBDIS;
4659 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
4661 if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
4663 * SRIOV inactive scheme
4664 * any DCB w/o VMDq multi-queue setting
4666 if (dev->data->dev_conf.txmode.mq_mode == ETH_MQ_TX_VMDQ_ONLY)
4667 ixgbe_vmdq_tx_hw_configure(hw);
4669 mtqc = IXGBE_MTQC_64Q_1PB;
4670 IXGBE_WRITE_REG(hw, IXGBE_MTQC, mtqc);
4673 switch (RTE_ETH_DEV_SRIOV(dev).active) {
4676 * SRIOV active scheme
4677 * FIXME if support DCB together with VMDq & SRIOV
4680 mtqc = IXGBE_MTQC_VT_ENA | IXGBE_MTQC_64VF;
4683 mtqc = IXGBE_MTQC_VT_ENA | IXGBE_MTQC_32VF;
4686 mtqc = IXGBE_MTQC_VT_ENA | IXGBE_MTQC_RT_ENA |
4690 mtqc = IXGBE_MTQC_64Q_1PB;
4691 PMD_INIT_LOG(ERR, "invalid pool number in IOV mode");
4693 IXGBE_WRITE_REG(hw, IXGBE_MTQC, mtqc);
4696 /* re-enable arbiter */
4697 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
4698 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
4704 * ixgbe_get_rscctl_maxdesc - Calculate the RSCCTL[n].MAXDESC for PF
4706 * Return the RSCCTL[n].MAXDESC for 82599 and x540 PF devices according to the
4707 * spec rev. 3.0 chapter 8.2.3.8.13.
4709 * @pool Memory pool of the Rx queue
4711 static inline uint32_t
4712 ixgbe_get_rscctl_maxdesc(struct rte_mempool *pool)
4714 struct rte_pktmbuf_pool_private *mp_priv = rte_mempool_get_priv(pool);
4716 /* MAXDESC * SRRCTL.BSIZEPKT must not exceed 64 KB minus one */
4718 RTE_IPV4_MAX_PKT_LEN /
4719 (mp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM);
4722 return IXGBE_RSCCTL_MAXDESC_16;
4723 else if (maxdesc >= 8)
4724 return IXGBE_RSCCTL_MAXDESC_8;
4725 else if (maxdesc >= 4)
4726 return IXGBE_RSCCTL_MAXDESC_4;
4728 return IXGBE_RSCCTL_MAXDESC_1;
4732 * ixgbe_set_ivar - Setup the correct IVAR register for a particular MSIX
4735 * (Taken from FreeBSD tree)
4736 * (yes this is all very magic and confusing :)
4739 * @entry the register array entry
4740 * @vector the MSIX vector for this queue
4744 ixgbe_set_ivar(struct rte_eth_dev *dev, u8 entry, u8 vector, s8 type)
4746 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4749 vector |= IXGBE_IVAR_ALLOC_VAL;
4751 switch (hw->mac.type) {
4753 case ixgbe_mac_82598EB:
4755 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
4757 entry += (type * 64);
4758 index = (entry >> 2) & 0x1F;
4759 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4760 ivar &= ~(0xFF << (8 * (entry & 0x3)));
4761 ivar |= (vector << (8 * (entry & 0x3)));
4762 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
4765 case ixgbe_mac_82599EB:
4766 case ixgbe_mac_X540:
4767 if (type == -1) { /* MISC IVAR */
4768 index = (entry & 1) * 8;
4769 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4770 ivar &= ~(0xFF << index);
4771 ivar |= (vector << index);
4772 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4773 } else { /* RX/TX IVARS */
4774 index = (16 * (entry & 1)) + (8 * type);
4775 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
4776 ivar &= ~(0xFF << index);
4777 ivar |= (vector << index);
4778 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
4789 ixgbe_set_rx_function(struct rte_eth_dev *dev)
4791 uint16_t i, rx_using_sse;
4792 struct ixgbe_adapter *adapter = dev->data->dev_private;
4795 * In order to allow Vector Rx there are a few configuration
4796 * conditions to be met and Rx Bulk Allocation should be allowed.
4798 if (ixgbe_rx_vec_dev_conf_condition_check(dev) ||
4799 !adapter->rx_bulk_alloc_allowed ||
4800 rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128) {
4801 PMD_INIT_LOG(DEBUG, "Port[%d] doesn't meet Vector Rx "
4803 dev->data->port_id);
4805 adapter->rx_vec_allowed = false;
4809 * Initialize the appropriate LRO callback.
4811 * If all queues satisfy the bulk allocation preconditions
4812 * (hw->rx_bulk_alloc_allowed is TRUE) then we may use bulk allocation.
4813 * Otherwise use a single allocation version.
4815 if (dev->data->lro) {
4816 if (adapter->rx_bulk_alloc_allowed) {
4817 PMD_INIT_LOG(DEBUG, "LRO is requested. Using a bulk "
4818 "allocation version");
4819 dev->rx_pkt_burst = ixgbe_recv_pkts_lro_bulk_alloc;
4821 PMD_INIT_LOG(DEBUG, "LRO is requested. Using a single "
4822 "allocation version");
4823 dev->rx_pkt_burst = ixgbe_recv_pkts_lro_single_alloc;
4825 } else if (dev->data->scattered_rx) {
4827 * Set the non-LRO scattered callback: there are Vector and
4828 * single allocation versions.
4830 if (adapter->rx_vec_allowed) {
4831 PMD_INIT_LOG(DEBUG, "Using Vector Scattered Rx "
4832 "callback (port=%d).",
4833 dev->data->port_id);
4835 dev->rx_pkt_burst = ixgbe_recv_scattered_pkts_vec;
4836 } else if (adapter->rx_bulk_alloc_allowed) {
4837 PMD_INIT_LOG(DEBUG, "Using a Scattered with bulk "
4838 "allocation callback (port=%d).",
4839 dev->data->port_id);
4840 dev->rx_pkt_burst = ixgbe_recv_pkts_lro_bulk_alloc;
4842 PMD_INIT_LOG(DEBUG, "Using Regualr (non-vector, "
4843 "single allocation) "
4844 "Scattered Rx callback "
4846 dev->data->port_id);
4848 dev->rx_pkt_burst = ixgbe_recv_pkts_lro_single_alloc;
4851 * Below we set "simple" callbacks according to port/queues parameters.
4852 * If parameters allow we are going to choose between the following
4856 * - Single buffer allocation (the simplest one)
4858 } else if (adapter->rx_vec_allowed) {
4859 PMD_INIT_LOG(DEBUG, "Vector rx enabled, please make sure RX "
4860 "burst size no less than %d (port=%d).",
4861 RTE_IXGBE_DESCS_PER_LOOP,
4862 dev->data->port_id);
4864 dev->rx_pkt_burst = ixgbe_recv_pkts_vec;
4865 } else if (adapter->rx_bulk_alloc_allowed) {
4866 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are "
4867 "satisfied. Rx Burst Bulk Alloc function "
4868 "will be used on port=%d.",
4869 dev->data->port_id);
4871 dev->rx_pkt_burst = ixgbe_recv_pkts_bulk_alloc;
4873 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are not "
4874 "satisfied, or Scattered Rx is requested "
4876 dev->data->port_id);
4878 dev->rx_pkt_burst = ixgbe_recv_pkts;
4881 /* Propagate information about RX function choice through all queues. */
4884 (dev->rx_pkt_burst == ixgbe_recv_scattered_pkts_vec ||
4885 dev->rx_pkt_burst == ixgbe_recv_pkts_vec);
4887 for (i = 0; i < dev->data->nb_rx_queues; i++) {
4888 struct ixgbe_rx_queue *rxq = dev->data->rx_queues[i];
4890 rxq->rx_using_sse = rx_using_sse;
4891 #ifdef RTE_LIB_SECURITY
4892 rxq->using_ipsec = !!(dev->data->dev_conf.rxmode.offloads &
4893 DEV_RX_OFFLOAD_SECURITY);
4899 * ixgbe_set_rsc - configure RSC related port HW registers
4901 * Configures the port's RSC related registers according to the 4.6.7.2 chapter
4902 * of 82599 Spec (x540 configuration is virtually the same).
4906 * Returns 0 in case of success or a non-zero error code
4909 ixgbe_set_rsc(struct rte_eth_dev *dev)
4911 struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode;
4912 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4913 struct rte_eth_dev_info dev_info = { 0 };
4914 bool rsc_capable = false;
4920 dev->dev_ops->dev_infos_get(dev, &dev_info);
4921 if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TCP_LRO)
4924 if (!rsc_capable && (rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO)) {
4925 PMD_INIT_LOG(CRIT, "LRO is requested on HW that doesn't "
4930 /* RSC global configuration (chapter 4.6.7.2.1 of 82599 Spec) */
4932 if ((rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC) &&
4933 (rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO)) {
4935 * According to chapter of 4.6.7.2.1 of the Spec Rev.
4936 * 3.0 RSC configuration requires HW CRC stripping being
4937 * enabled. If user requested both HW CRC stripping off
4938 * and RSC on - return an error.
4940 PMD_INIT_LOG(CRIT, "LRO can't be enabled when HW CRC "
4945 /* RFCTL configuration */
4946 rfctl = IXGBE_READ_REG(hw, IXGBE_RFCTL);
4947 if ((rsc_capable) && (rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO))
4948 rfctl &= ~IXGBE_RFCTL_RSC_DIS;
4950 rfctl |= IXGBE_RFCTL_RSC_DIS;
4951 /* disable NFS filtering */
4952 rfctl |= IXGBE_RFCTL_NFSW_DIS | IXGBE_RFCTL_NFSR_DIS;
4953 IXGBE_WRITE_REG(hw, IXGBE_RFCTL, rfctl);
4955 /* If LRO hasn't been requested - we are done here. */
4956 if (!(rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO))
4959 /* Set RDRXCTL.RSCACKC bit */
4960 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
4961 rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
4962 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
4964 /* Per-queue RSC configuration (chapter 4.6.7.2.2 of 82599 Spec) */
4965 for (i = 0; i < dev->data->nb_rx_queues; i++) {
4966 struct ixgbe_rx_queue *rxq = dev->data->rx_queues[i];
4968 IXGBE_READ_REG(hw, IXGBE_SRRCTL(rxq->reg_idx));
4970 IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxq->reg_idx));
4972 IXGBE_READ_REG(hw, IXGBE_PSRTYPE(rxq->reg_idx));
4974 IXGBE_READ_REG(hw, IXGBE_EITR(rxq->reg_idx));
4977 * ixgbe PMD doesn't support header-split at the moment.
4979 * Following the 4.6.7.2.1 chapter of the 82599/x540
4980 * Spec if RSC is enabled the SRRCTL[n].BSIZEHEADER
4981 * should be configured even if header split is not
4982 * enabled. We will configure it 128 bytes following the
4983 * recommendation in the spec.
4985 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
4986 srrctl |= (128 << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) &
4987 IXGBE_SRRCTL_BSIZEHDR_MASK;
4990 * TODO: Consider setting the Receive Descriptor Minimum
4991 * Threshold Size for an RSC case. This is not an obviously
4992 * beneficiary option but the one worth considering...
4995 rscctl |= IXGBE_RSCCTL_RSCEN;
4996 rscctl |= ixgbe_get_rscctl_maxdesc(rxq->mb_pool);
4997 psrtype |= IXGBE_PSRTYPE_TCPHDR;
5000 * RSC: Set ITR interval corresponding to 2K ints/s.
5002 * Full-sized RSC aggregations for a 10Gb/s link will
5003 * arrive at about 20K aggregation/s rate.
5005 * 2K inst/s rate will make only 10% of the
5006 * aggregations to be closed due to the interrupt timer
5007 * expiration for a streaming at wire-speed case.
5009 * For a sparse streaming case this setting will yield
5010 * at most 500us latency for a single RSC aggregation.
5012 eitr &= ~IXGBE_EITR_ITR_INT_MASK;
5013 eitr |= IXGBE_EITR_INTERVAL_US(IXGBE_QUEUE_ITR_INTERVAL_DEFAULT);
5014 eitr |= IXGBE_EITR_CNT_WDIS;
5016 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxq->reg_idx), srrctl);
5017 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxq->reg_idx), rscctl);
5018 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(rxq->reg_idx), psrtype);
5019 IXGBE_WRITE_REG(hw, IXGBE_EITR(rxq->reg_idx), eitr);
5022 * RSC requires the mapping of the queue to the
5025 ixgbe_set_ivar(dev, rxq->reg_idx, i, 0);
5030 PMD_INIT_LOG(DEBUG, "enabling LRO mode");
5036 * Initializes Receive Unit.
5039 ixgbe_dev_rx_init(struct rte_eth_dev *dev)
5041 struct ixgbe_hw *hw;
5042 struct ixgbe_rx_queue *rxq;
5053 struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode;
5056 PMD_INIT_FUNC_TRACE();
5057 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5060 * Make sure receives are disabled while setting
5061 * up the RX context (registers, descriptor rings, etc.).
5063 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
5064 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl & ~IXGBE_RXCTRL_RXEN);
5066 /* Enable receipt of broadcasted frames */
5067 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
5068 fctrl |= IXGBE_FCTRL_BAM;
5069 fctrl |= IXGBE_FCTRL_DPF;
5070 fctrl |= IXGBE_FCTRL_PMCF;
5071 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
5074 * Configure CRC stripping, if any.
5076 hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
5077 if (rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC)
5078 hlreg0 &= ~IXGBE_HLREG0_RXCRCSTRP;
5080 hlreg0 |= IXGBE_HLREG0_RXCRCSTRP;
5083 * Configure jumbo frame support, if any.
5085 if (rx_conf->offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) {
5086 hlreg0 |= IXGBE_HLREG0_JUMBOEN;
5087 maxfrs = IXGBE_READ_REG(hw, IXGBE_MAXFRS);
5088 maxfrs &= 0x0000FFFF;
5089 maxfrs |= (rx_conf->max_rx_pkt_len << 16);
5090 IXGBE_WRITE_REG(hw, IXGBE_MAXFRS, maxfrs);
5092 hlreg0 &= ~IXGBE_HLREG0_JUMBOEN;
5095 * If loopback mode is configured, set LPBK bit.
5097 if (dev->data->dev_conf.lpbk_mode != 0) {
5098 rc = ixgbe_check_supported_loopback_mode(dev);
5100 PMD_INIT_LOG(ERR, "Unsupported loopback mode");
5103 hlreg0 |= IXGBE_HLREG0_LPBK;
5105 hlreg0 &= ~IXGBE_HLREG0_LPBK;
5108 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);
5111 * Assume no header split and no VLAN strip support
5112 * on any Rx queue first .
5114 rx_conf->offloads &= ~DEV_RX_OFFLOAD_VLAN_STRIP;
5115 /* Setup RX queues */
5116 for (i = 0; i < dev->data->nb_rx_queues; i++) {
5117 rxq = dev->data->rx_queues[i];
5120 * Reset crc_len in case it was changed after queue setup by a
5121 * call to configure.
5123 if (rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC)
5124 rxq->crc_len = RTE_ETHER_CRC_LEN;
5128 /* Setup the Base and Length of the Rx Descriptor Rings */
5129 bus_addr = rxq->rx_ring_phys_addr;
5130 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(rxq->reg_idx),
5131 (uint32_t)(bus_addr & 0x00000000ffffffffULL));
5132 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(rxq->reg_idx),
5133 (uint32_t)(bus_addr >> 32));
5134 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(rxq->reg_idx),
5135 rxq->nb_rx_desc * sizeof(union ixgbe_adv_rx_desc));
5136 IXGBE_WRITE_REG(hw, IXGBE_RDH(rxq->reg_idx), 0);
5137 IXGBE_WRITE_REG(hw, IXGBE_RDT(rxq->reg_idx), 0);
5139 /* Configure the SRRCTL register */
5140 srrctl = IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
5142 /* Set if packets are dropped when no descriptors available */
5144 srrctl |= IXGBE_SRRCTL_DROP_EN;
5147 * Configure the RX buffer size in the BSIZEPACKET field of
5148 * the SRRCTL register of the queue.
5149 * The value is in 1 KB resolution. Valid values can be from
5152 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) -
5153 RTE_PKTMBUF_HEADROOM);
5154 srrctl |= ((buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) &
5155 IXGBE_SRRCTL_BSIZEPKT_MASK);
5157 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxq->reg_idx), srrctl);
5159 buf_size = (uint16_t) ((srrctl & IXGBE_SRRCTL_BSIZEPKT_MASK) <<
5160 IXGBE_SRRCTL_BSIZEPKT_SHIFT);
5162 /* It adds dual VLAN length for supporting dual VLAN */
5163 if (dev->data->dev_conf.rxmode.max_rx_pkt_len +
5164 2 * IXGBE_VLAN_TAG_SIZE > buf_size)
5165 dev->data->scattered_rx = 1;
5166 if (rxq->offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
5167 rx_conf->offloads |= DEV_RX_OFFLOAD_VLAN_STRIP;
5170 if (rx_conf->offloads & DEV_RX_OFFLOAD_SCATTER)
5171 dev->data->scattered_rx = 1;
5174 * Device configured with multiple RX queues.
5176 ixgbe_dev_mq_rx_configure(dev);
5179 * Setup the Checksum Register.
5180 * Disable Full-Packet Checksum which is mutually exclusive with RSS.
5181 * Enable IP/L4 checkum computation by hardware if requested to do so.
5183 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
5184 rxcsum |= IXGBE_RXCSUM_PCSD;
5185 if (rx_conf->offloads & DEV_RX_OFFLOAD_CHECKSUM)
5186 rxcsum |= IXGBE_RXCSUM_IPPCSE;
5188 rxcsum &= ~IXGBE_RXCSUM_IPPCSE;
5190 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
5192 if (hw->mac.type == ixgbe_mac_82599EB ||
5193 hw->mac.type == ixgbe_mac_X540) {
5194 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
5195 if (rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC)
5196 rdrxctl &= ~IXGBE_RDRXCTL_CRCSTRIP;
5198 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
5199 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
5200 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
5203 rc = ixgbe_set_rsc(dev);
5207 ixgbe_set_rx_function(dev);
5213 * Initializes Transmit Unit.
5216 ixgbe_dev_tx_init(struct rte_eth_dev *dev)
5218 struct ixgbe_hw *hw;
5219 struct ixgbe_tx_queue *txq;
5225 PMD_INIT_FUNC_TRACE();
5226 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5228 /* Enable TX CRC (checksum offload requirement) and hw padding
5231 hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
5232 hlreg0 |= (IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN);
5233 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);
5235 /* Setup the Base and Length of the Tx Descriptor Rings */
5236 for (i = 0; i < dev->data->nb_tx_queues; i++) {
5237 txq = dev->data->tx_queues[i];
5239 bus_addr = txq->tx_ring_phys_addr;
5240 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(txq->reg_idx),
5241 (uint32_t)(bus_addr & 0x00000000ffffffffULL));
5242 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(txq->reg_idx),
5243 (uint32_t)(bus_addr >> 32));
5244 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(txq->reg_idx),
5245 txq->nb_tx_desc * sizeof(union ixgbe_adv_tx_desc));
5246 /* Setup the HW Tx Head and TX Tail descriptor pointers */
5247 IXGBE_WRITE_REG(hw, IXGBE_TDH(txq->reg_idx), 0);
5248 IXGBE_WRITE_REG(hw, IXGBE_TDT(txq->reg_idx), 0);
5251 * Disable Tx Head Writeback RO bit, since this hoses
5252 * bookkeeping if things aren't delivered in order.
5254 switch (hw->mac.type) {
5255 case ixgbe_mac_82598EB:
5256 txctrl = IXGBE_READ_REG(hw,
5257 IXGBE_DCA_TXCTRL(txq->reg_idx));
5258 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
5259 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(txq->reg_idx),
5263 case ixgbe_mac_82599EB:
5264 case ixgbe_mac_X540:
5265 case ixgbe_mac_X550:
5266 case ixgbe_mac_X550EM_x:
5267 case ixgbe_mac_X550EM_a:
5269 txctrl = IXGBE_READ_REG(hw,
5270 IXGBE_DCA_TXCTRL_82599(txq->reg_idx));
5271 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
5272 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(txq->reg_idx),
5278 /* Device configured with multiple TX queues. */
5279 ixgbe_dev_mq_tx_configure(dev);
5283 * Check if requested loopback mode is supported
5286 ixgbe_check_supported_loopback_mode(struct rte_eth_dev *dev)
5288 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5290 if (dev->data->dev_conf.lpbk_mode == IXGBE_LPBK_TX_RX)
5291 if (hw->mac.type == ixgbe_mac_82599EB ||
5292 hw->mac.type == ixgbe_mac_X540 ||
5293 hw->mac.type == ixgbe_mac_X550 ||
5294 hw->mac.type == ixgbe_mac_X550EM_x ||
5295 hw->mac.type == ixgbe_mac_X550EM_a)
5302 * Set up link for 82599 loopback mode Tx->Rx.
5304 static inline void __rte_cold
5305 ixgbe_setup_loopback_link_82599(struct ixgbe_hw *hw)
5307 PMD_INIT_FUNC_TRACE();
5309 if (ixgbe_verify_lesm_fw_enabled_82599(hw)) {
5310 if (hw->mac.ops.acquire_swfw_sync(hw, IXGBE_GSSR_MAC_CSR_SM) !=
5312 PMD_INIT_LOG(ERR, "Could not enable loopback mode");
5321 IXGBE_AUTOC_LMS_10G_LINK_NO_AN | IXGBE_AUTOC_FLU);
5322 ixgbe_reset_pipeline_82599(hw);
5324 hw->mac.ops.release_swfw_sync(hw, IXGBE_GSSR_MAC_CSR_SM);
5330 * Start Transmit and Receive Units.
5333 ixgbe_dev_rxtx_start(struct rte_eth_dev *dev)
5335 struct ixgbe_hw *hw;
5336 struct ixgbe_tx_queue *txq;
5337 struct ixgbe_rx_queue *rxq;
5344 PMD_INIT_FUNC_TRACE();
5345 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5347 for (i = 0; i < dev->data->nb_tx_queues; i++) {
5348 txq = dev->data->tx_queues[i];
5349 /* Setup Transmit Threshold Registers */
5350 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txq->reg_idx));
5351 txdctl |= txq->pthresh & 0x7F;
5352 txdctl |= ((txq->hthresh & 0x7F) << 8);
5353 txdctl |= ((txq->wthresh & 0x7F) << 16);
5354 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txq->reg_idx), txdctl);
5357 if (hw->mac.type != ixgbe_mac_82598EB) {
5358 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
5359 dmatxctl |= IXGBE_DMATXCTL_TE;
5360 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
5363 for (i = 0; i < dev->data->nb_tx_queues; i++) {
5364 txq = dev->data->tx_queues[i];
5365 if (!txq->tx_deferred_start) {
5366 ret = ixgbe_dev_tx_queue_start(dev, i);
5372 for (i = 0; i < dev->data->nb_rx_queues; i++) {
5373 rxq = dev->data->rx_queues[i];
5374 if (!rxq->rx_deferred_start) {
5375 ret = ixgbe_dev_rx_queue_start(dev, i);
5381 /* Enable Receive engine */
5382 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
5383 if (hw->mac.type == ixgbe_mac_82598EB)
5384 rxctrl |= IXGBE_RXCTRL_DMBYPS;
5385 rxctrl |= IXGBE_RXCTRL_RXEN;
5386 hw->mac.ops.enable_rx_dma(hw, rxctrl);
5388 /* If loopback mode is enabled, set up the link accordingly */
5389 if (dev->data->dev_conf.lpbk_mode != 0) {
5390 if (hw->mac.type == ixgbe_mac_82599EB)
5391 ixgbe_setup_loopback_link_82599(hw);
5392 else if (hw->mac.type == ixgbe_mac_X540 ||
5393 hw->mac.type == ixgbe_mac_X550 ||
5394 hw->mac.type == ixgbe_mac_X550EM_x ||
5395 hw->mac.type == ixgbe_mac_X550EM_a)
5396 ixgbe_setup_loopback_link_x540_x550(hw, true);
5399 #ifdef RTE_LIB_SECURITY
5400 if ((dev->data->dev_conf.rxmode.offloads &
5401 DEV_RX_OFFLOAD_SECURITY) ||
5402 (dev->data->dev_conf.txmode.offloads &
5403 DEV_TX_OFFLOAD_SECURITY)) {
5404 ret = ixgbe_crypto_enable_ipsec(dev);
5407 "ixgbe_crypto_enable_ipsec fails with %d.",
5418 * Start Receive Units for specified queue.
5421 ixgbe_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
5423 struct ixgbe_hw *hw;
5424 struct ixgbe_rx_queue *rxq;
5428 PMD_INIT_FUNC_TRACE();
5429 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5431 rxq = dev->data->rx_queues[rx_queue_id];
5433 /* Allocate buffers for descriptor rings */
5434 if (ixgbe_alloc_rx_queue_mbufs(rxq) != 0) {
5435 PMD_INIT_LOG(ERR, "Could not alloc mbuf for queue:%d",
5439 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
5440 rxdctl |= IXGBE_RXDCTL_ENABLE;
5441 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxq->reg_idx), rxdctl);
5443 /* Wait until RX Enable ready */
5444 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
5447 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
5448 } while (--poll_ms && !(rxdctl & IXGBE_RXDCTL_ENABLE));
5450 PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", rx_queue_id);
5452 IXGBE_WRITE_REG(hw, IXGBE_RDH(rxq->reg_idx), 0);
5453 IXGBE_WRITE_REG(hw, IXGBE_RDT(rxq->reg_idx), rxq->nb_rx_desc - 1);
5454 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
5460 * Stop Receive Units for specified queue.
5463 ixgbe_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
5465 struct ixgbe_hw *hw;
5466 struct ixgbe_adapter *adapter = dev->data->dev_private;
5467 struct ixgbe_rx_queue *rxq;
5471 PMD_INIT_FUNC_TRACE();
5472 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5474 rxq = dev->data->rx_queues[rx_queue_id];
5476 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
5477 rxdctl &= ~IXGBE_RXDCTL_ENABLE;
5478 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxq->reg_idx), rxdctl);
5480 /* Wait until RX Enable bit clear */
5481 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
5484 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
5485 } while (--poll_ms && (rxdctl & IXGBE_RXDCTL_ENABLE));
5487 PMD_INIT_LOG(ERR, "Could not disable Rx Queue %d", rx_queue_id);
5489 rte_delay_us(RTE_IXGBE_WAIT_100_US);
5491 ixgbe_rx_queue_release_mbufs(rxq);
5492 ixgbe_reset_rx_queue(adapter, rxq);
5493 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
5500 * Start Transmit Units for specified queue.
5503 ixgbe_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
5505 struct ixgbe_hw *hw;
5506 struct ixgbe_tx_queue *txq;
5510 PMD_INIT_FUNC_TRACE();
5511 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5513 txq = dev->data->tx_queues[tx_queue_id];
5514 IXGBE_WRITE_REG(hw, IXGBE_TDH(txq->reg_idx), 0);
5515 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txq->reg_idx));
5516 txdctl |= IXGBE_TXDCTL_ENABLE;
5517 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txq->reg_idx), txdctl);
5519 /* Wait until TX Enable ready */
5520 if (hw->mac.type == ixgbe_mac_82599EB) {
5521 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
5524 txdctl = IXGBE_READ_REG(hw,
5525 IXGBE_TXDCTL(txq->reg_idx));
5526 } while (--poll_ms && !(txdctl & IXGBE_TXDCTL_ENABLE));
5528 PMD_INIT_LOG(ERR, "Could not enable Tx Queue %d",
5532 IXGBE_WRITE_REG(hw, IXGBE_TDT(txq->reg_idx), 0);
5533 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
5539 * Stop Transmit Units for specified queue.
5542 ixgbe_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
5544 struct ixgbe_hw *hw;
5545 struct ixgbe_tx_queue *txq;
5547 uint32_t txtdh, txtdt;
5550 PMD_INIT_FUNC_TRACE();
5551 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5553 txq = dev->data->tx_queues[tx_queue_id];
5555 /* Wait until TX queue is empty */
5556 if (hw->mac.type == ixgbe_mac_82599EB) {
5557 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
5559 rte_delay_us(RTE_IXGBE_WAIT_100_US);
5560 txtdh = IXGBE_READ_REG(hw,
5561 IXGBE_TDH(txq->reg_idx));
5562 txtdt = IXGBE_READ_REG(hw,
5563 IXGBE_TDT(txq->reg_idx));
5564 } while (--poll_ms && (txtdh != txtdt));
5567 "Tx Queue %d is not empty when stopping.",
5571 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txq->reg_idx));
5572 txdctl &= ~IXGBE_TXDCTL_ENABLE;
5573 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txq->reg_idx), txdctl);
5575 /* Wait until TX Enable bit clear */
5576 if (hw->mac.type == ixgbe_mac_82599EB) {
5577 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
5580 txdctl = IXGBE_READ_REG(hw,
5581 IXGBE_TXDCTL(txq->reg_idx));
5582 } while (--poll_ms && (txdctl & IXGBE_TXDCTL_ENABLE));
5584 PMD_INIT_LOG(ERR, "Could not disable Tx Queue %d",
5588 if (txq->ops != NULL) {
5589 txq->ops->release_mbufs(txq);
5590 txq->ops->reset(txq);
5592 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
5598 ixgbe_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
5599 struct rte_eth_rxq_info *qinfo)
5601 struct ixgbe_rx_queue *rxq;
5603 rxq = dev->data->rx_queues[queue_id];
5605 qinfo->mp = rxq->mb_pool;
5606 qinfo->scattered_rx = dev->data->scattered_rx;
5607 qinfo->nb_desc = rxq->nb_rx_desc;
5609 qinfo->conf.rx_free_thresh = rxq->rx_free_thresh;
5610 qinfo->conf.rx_drop_en = rxq->drop_en;
5611 qinfo->conf.rx_deferred_start = rxq->rx_deferred_start;
5612 qinfo->conf.offloads = rxq->offloads;
5616 ixgbe_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
5617 struct rte_eth_txq_info *qinfo)
5619 struct ixgbe_tx_queue *txq;
5621 txq = dev->data->tx_queues[queue_id];
5623 qinfo->nb_desc = txq->nb_tx_desc;
5625 qinfo->conf.tx_thresh.pthresh = txq->pthresh;
5626 qinfo->conf.tx_thresh.hthresh = txq->hthresh;
5627 qinfo->conf.tx_thresh.wthresh = txq->wthresh;
5629 qinfo->conf.tx_free_thresh = txq->tx_free_thresh;
5630 qinfo->conf.tx_rs_thresh = txq->tx_rs_thresh;
5631 qinfo->conf.offloads = txq->offloads;
5632 qinfo->conf.tx_deferred_start = txq->tx_deferred_start;
5636 * [VF] Initializes Receive Unit.
5639 ixgbevf_dev_rx_init(struct rte_eth_dev *dev)
5641 struct ixgbe_hw *hw;
5642 struct ixgbe_rx_queue *rxq;
5643 struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
5645 uint32_t srrctl, psrtype = 0;
5650 PMD_INIT_FUNC_TRACE();
5651 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5653 if (rte_is_power_of_2(dev->data->nb_rx_queues) == 0) {
5654 PMD_INIT_LOG(ERR, "The number of Rx queue invalid, "
5655 "it should be power of 2");
5659 if (dev->data->nb_rx_queues > hw->mac.max_rx_queues) {
5660 PMD_INIT_LOG(ERR, "The number of Rx queue invalid, "
5661 "it should be equal to or less than %d",
5662 hw->mac.max_rx_queues);
5667 * When the VF driver issues a IXGBE_VF_RESET request, the PF driver
5668 * disables the VF receipt of packets if the PF MTU is > 1500.
5669 * This is done to deal with 82599 limitations that imposes
5670 * the PF and all VFs to share the same MTU.
5671 * Then, the PF driver enables again the VF receipt of packet when
5672 * the VF driver issues a IXGBE_VF_SET_LPE request.
5673 * In the meantime, the VF device cannot be used, even if the VF driver
5674 * and the Guest VM network stack are ready to accept packets with a
5675 * size up to the PF MTU.
5676 * As a work-around to this PF behaviour, force the call to
5677 * ixgbevf_rlpml_set_vf even if jumbo frames are not used. This way,
5678 * VF packets received can work in all cases.
5680 if (ixgbevf_rlpml_set_vf(hw,
5681 (uint16_t)dev->data->dev_conf.rxmode.max_rx_pkt_len)) {
5682 PMD_INIT_LOG(ERR, "Set max packet length to %d failed.",
5683 dev->data->dev_conf.rxmode.max_rx_pkt_len);
5688 * Assume no header split and no VLAN strip support
5689 * on any Rx queue first .
5691 rxmode->offloads &= ~DEV_RX_OFFLOAD_VLAN_STRIP;
5692 /* Setup RX queues */
5693 for (i = 0; i < dev->data->nb_rx_queues; i++) {
5694 rxq = dev->data->rx_queues[i];
5696 /* Allocate buffers for descriptor rings */
5697 ret = ixgbe_alloc_rx_queue_mbufs(rxq);
5701 /* Setup the Base and Length of the Rx Descriptor Rings */
5702 bus_addr = rxq->rx_ring_phys_addr;
5704 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(i),
5705 (uint32_t)(bus_addr & 0x00000000ffffffffULL));
5706 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(i),
5707 (uint32_t)(bus_addr >> 32));
5708 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(i),
5709 rxq->nb_rx_desc * sizeof(union ixgbe_adv_rx_desc));
5710 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(i), 0);
5711 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(i), 0);
5714 /* Configure the SRRCTL register */
5715 srrctl = IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
5717 /* Set if packets are dropped when no descriptors available */
5719 srrctl |= IXGBE_SRRCTL_DROP_EN;
5722 * Configure the RX buffer size in the BSIZEPACKET field of
5723 * the SRRCTL register of the queue.
5724 * The value is in 1 KB resolution. Valid values can be from
5727 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) -
5728 RTE_PKTMBUF_HEADROOM);
5729 srrctl |= ((buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) &
5730 IXGBE_SRRCTL_BSIZEPKT_MASK);
5733 * VF modification to write virtual function SRRCTL register
5735 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(i), srrctl);
5737 buf_size = (uint16_t) ((srrctl & IXGBE_SRRCTL_BSIZEPKT_MASK) <<
5738 IXGBE_SRRCTL_BSIZEPKT_SHIFT);
5740 if (rxmode->offloads & DEV_RX_OFFLOAD_SCATTER ||
5741 /* It adds dual VLAN length for supporting dual VLAN */
5742 (rxmode->max_rx_pkt_len +
5743 2 * IXGBE_VLAN_TAG_SIZE) > buf_size) {
5744 if (!dev->data->scattered_rx)
5745 PMD_INIT_LOG(DEBUG, "forcing scatter mode");
5746 dev->data->scattered_rx = 1;
5749 if (rxq->offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
5750 rxmode->offloads |= DEV_RX_OFFLOAD_VLAN_STRIP;
5753 /* Set RQPL for VF RSS according to max Rx queue */
5754 psrtype |= (dev->data->nb_rx_queues >> 1) <<
5755 IXGBE_PSRTYPE_RQPL_SHIFT;
5756 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype);
5758 ixgbe_set_rx_function(dev);
5764 * [VF] Initializes Transmit Unit.
5767 ixgbevf_dev_tx_init(struct rte_eth_dev *dev)
5769 struct ixgbe_hw *hw;
5770 struct ixgbe_tx_queue *txq;
5775 PMD_INIT_FUNC_TRACE();
5776 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5778 /* Setup the Base and Length of the Tx Descriptor Rings */
5779 for (i = 0; i < dev->data->nb_tx_queues; i++) {
5780 txq = dev->data->tx_queues[i];
5781 bus_addr = txq->tx_ring_phys_addr;
5782 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(i),
5783 (uint32_t)(bus_addr & 0x00000000ffffffffULL));
5784 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(i),
5785 (uint32_t)(bus_addr >> 32));
5786 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(i),
5787 txq->nb_tx_desc * sizeof(union ixgbe_adv_tx_desc));
5788 /* Setup the HW Tx Head and TX Tail descriptor pointers */
5789 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(i), 0);
5790 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(i), 0);
5793 * Disable Tx Head Writeback RO bit, since this hoses
5794 * bookkeeping if things aren't delivered in order.
5796 txctrl = IXGBE_READ_REG(hw,
5797 IXGBE_VFDCA_TXCTRL(i));
5798 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
5799 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(i),
5805 * [VF] Start Transmit and Receive Units.
5808 ixgbevf_dev_rxtx_start(struct rte_eth_dev *dev)
5810 struct ixgbe_hw *hw;
5811 struct ixgbe_tx_queue *txq;
5812 struct ixgbe_rx_queue *rxq;
5818 PMD_INIT_FUNC_TRACE();
5819 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5821 for (i = 0; i < dev->data->nb_tx_queues; i++) {
5822 txq = dev->data->tx_queues[i];
5823 /* Setup Transmit Threshold Registers */
5824 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i));
5825 txdctl |= txq->pthresh & 0x7F;
5826 txdctl |= ((txq->hthresh & 0x7F) << 8);
5827 txdctl |= ((txq->wthresh & 0x7F) << 16);
5828 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl);
5831 for (i = 0; i < dev->data->nb_tx_queues; i++) {
5833 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i));
5834 txdctl |= IXGBE_TXDCTL_ENABLE;
5835 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl);
5838 /* Wait until TX Enable ready */
5841 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i));
5842 } while (--poll_ms && !(txdctl & IXGBE_TXDCTL_ENABLE));
5844 PMD_INIT_LOG(ERR, "Could not enable Tx Queue %d", i);
5846 for (i = 0; i < dev->data->nb_rx_queues; i++) {
5848 rxq = dev->data->rx_queues[i];
5850 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i));
5851 rxdctl |= IXGBE_RXDCTL_ENABLE;
5852 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), rxdctl);
5854 /* Wait until RX Enable ready */
5858 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i));
5859 } while (--poll_ms && !(rxdctl & IXGBE_RXDCTL_ENABLE));
5861 PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", i);
5863 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(i), rxq->nb_rx_desc - 1);
5869 ixgbe_rss_conf_init(struct ixgbe_rte_flow_rss_conf *out,
5870 const struct rte_flow_action_rss *in)
5872 if (in->key_len > RTE_DIM(out->key) ||
5873 in->queue_num > RTE_DIM(out->queue))
5875 out->conf = (struct rte_flow_action_rss){
5879 .key_len = in->key_len,
5880 .queue_num = in->queue_num,
5881 .key = memcpy(out->key, in->key, in->key_len),
5882 .queue = memcpy(out->queue, in->queue,
5883 sizeof(*in->queue) * in->queue_num),
5889 ixgbe_action_rss_same(const struct rte_flow_action_rss *comp,
5890 const struct rte_flow_action_rss *with)
5892 return (comp->func == with->func &&
5893 comp->level == with->level &&
5894 comp->types == with->types &&
5895 comp->key_len == with->key_len &&
5896 comp->queue_num == with->queue_num &&
5897 !memcmp(comp->key, with->key, with->key_len) &&
5898 !memcmp(comp->queue, with->queue,
5899 sizeof(*with->queue) * with->queue_num));
5903 ixgbe_config_rss_filter(struct rte_eth_dev *dev,
5904 struct ixgbe_rte_flow_rss_conf *conf, bool add)
5906 struct ixgbe_hw *hw;
5910 uint16_t sp_reta_size;
5912 struct rte_eth_rss_conf rss_conf = {
5913 .rss_key = conf->conf.key_len ?
5914 (void *)(uintptr_t)conf->conf.key : NULL,
5915 .rss_key_len = conf->conf.key_len,
5916 .rss_hf = conf->conf.types,
5918 struct ixgbe_filter_info *filter_info =
5919 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
5921 PMD_INIT_FUNC_TRACE();
5922 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5924 sp_reta_size = ixgbe_reta_size_get(hw->mac.type);
5927 if (ixgbe_action_rss_same(&filter_info->rss_info.conf,
5929 ixgbe_rss_disable(dev);
5930 memset(&filter_info->rss_info, 0,
5931 sizeof(struct ixgbe_rte_flow_rss_conf));
5937 if (filter_info->rss_info.conf.queue_num)
5939 /* Fill in redirection table
5940 * The byte-swap is needed because NIC registers are in
5941 * little-endian order.
5944 for (i = 0, j = 0; i < sp_reta_size; i++, j++) {
5945 reta_reg = ixgbe_reta_reg_get(hw->mac.type, i);
5947 if (j == conf->conf.queue_num)
5949 reta = (reta << 8) | conf->conf.queue[j];
5951 IXGBE_WRITE_REG(hw, reta_reg,
5955 /* Configure the RSS key and the RSS protocols used to compute
5956 * the RSS hash of input packets.
5958 if ((rss_conf.rss_hf & IXGBE_RSS_OFFLOAD_ALL) == 0) {
5959 ixgbe_rss_disable(dev);
5962 if (rss_conf.rss_key == NULL)
5963 rss_conf.rss_key = rss_intel_key; /* Default hash key */
5964 ixgbe_hw_rss_hash_set(hw, &rss_conf);
5966 if (ixgbe_rss_conf_init(&filter_info->rss_info, &conf->conf))
5972 /* Stubs needed for linkage when RTE_ARCH_PPC_64 is set */
5973 #if defined(RTE_ARCH_PPC_64)
5975 ixgbe_rx_vec_dev_conf_condition_check(struct rte_eth_dev __rte_unused *dev)
5981 ixgbe_recv_pkts_vec(
5982 void __rte_unused *rx_queue,
5983 struct rte_mbuf __rte_unused **rx_pkts,
5984 uint16_t __rte_unused nb_pkts)
5990 ixgbe_recv_scattered_pkts_vec(
5991 void __rte_unused *rx_queue,
5992 struct rte_mbuf __rte_unused **rx_pkts,
5993 uint16_t __rte_unused nb_pkts)
5999 ixgbe_rxq_vec_setup(struct ixgbe_rx_queue __rte_unused *rxq)
6005 ixgbe_xmit_fixed_burst_vec(void __rte_unused *tx_queue,
6006 struct rte_mbuf __rte_unused **tx_pkts,
6007 uint16_t __rte_unused nb_pkts)
6013 ixgbe_txq_vec_setup(struct ixgbe_tx_queue __rte_unused *txq)
6019 ixgbe_rx_queue_release_mbufs_vec(struct ixgbe_rx_queue __rte_unused *rxq)