1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2016 Intel Corporation.
3 * Copyright 2014 6WIND S.A.
17 #include <rte_byteorder.h>
18 #include <rte_common.h>
19 #include <rte_cycles.h>
21 #include <rte_debug.h>
22 #include <rte_interrupts.h>
24 #include <rte_memory.h>
25 #include <rte_memzone.h>
26 #include <rte_launch.h>
28 #include <rte_per_lcore.h>
29 #include <rte_lcore.h>
30 #include <rte_atomic.h>
31 #include <rte_branch_prediction.h>
32 #include <rte_mempool.h>
33 #include <rte_malloc.h>
35 #include <rte_ether.h>
36 #include <ethdev_driver.h>
37 #include <rte_security_driver.h>
38 #include <rte_prefetch.h>
42 #include <rte_string_fns.h>
43 #include <rte_errno.h>
48 #include "ixgbe_logs.h"
49 #include "base/ixgbe_api.h"
50 #include "base/ixgbe_vf.h"
51 #include "ixgbe_ethdev.h"
52 #include "base/ixgbe_dcb.h"
53 #include "base/ixgbe_common.h"
54 #include "ixgbe_rxtx.h"
56 #ifdef RTE_LIBRTE_IEEE1588
57 #define IXGBE_TX_IEEE1588_TMST PKT_TX_IEEE1588_TMST
59 #define IXGBE_TX_IEEE1588_TMST 0
61 /* Bit Mask to indicate what bits required for building TX context */
62 #define IXGBE_TX_OFFLOAD_MASK ( \
72 PKT_TX_OUTER_IP_CKSUM | \
73 PKT_TX_SEC_OFFLOAD | \
74 IXGBE_TX_IEEE1588_TMST)
76 #define IXGBE_TX_OFFLOAD_NOTSUP_MASK \
77 (PKT_TX_OFFLOAD_MASK ^ IXGBE_TX_OFFLOAD_MASK)
80 #define RTE_PMD_USE_PREFETCH
83 #ifdef RTE_PMD_USE_PREFETCH
85 * Prefetch a cache line into all cache levels.
87 #define rte_ixgbe_prefetch(p) rte_prefetch0(p)
89 #define rte_ixgbe_prefetch(p) do {} while (0)
92 /*********************************************************************
96 **********************************************************************/
99 * Check for descriptors with their DD bit set and free mbufs.
100 * Return the total number of buffers freed.
102 static __rte_always_inline int
103 ixgbe_tx_free_bufs(struct ixgbe_tx_queue *txq)
105 struct ixgbe_tx_entry *txep;
108 struct rte_mbuf *m, *free[RTE_IXGBE_TX_MAX_FREE_BUF_SZ];
110 /* check DD bit on threshold descriptor */
111 status = txq->tx_ring[txq->tx_next_dd].wb.status;
112 if (!(status & rte_cpu_to_le_32(IXGBE_ADVTXD_STAT_DD)))
116 * first buffer to free from S/W ring is at index
117 * tx_next_dd - (tx_rs_thresh-1)
119 txep = &(txq->sw_ring[txq->tx_next_dd - (txq->tx_rs_thresh - 1)]);
121 for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) {
122 /* free buffers one at a time */
123 m = rte_pktmbuf_prefree_seg(txep->mbuf);
126 if (unlikely(m == NULL))
129 if (nb_free >= RTE_IXGBE_TX_MAX_FREE_BUF_SZ ||
130 (nb_free > 0 && m->pool != free[0]->pool)) {
131 rte_mempool_put_bulk(free[0]->pool,
132 (void **)free, nb_free);
140 rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free);
142 /* buffers were freed, update counters */
143 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh);
144 txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh);
145 if (txq->tx_next_dd >= txq->nb_tx_desc)
146 txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
148 return txq->tx_rs_thresh;
151 /* Populate 4 descriptors with data from 4 mbufs */
153 tx4(volatile union ixgbe_adv_tx_desc *txdp, struct rte_mbuf **pkts)
155 uint64_t buf_dma_addr;
159 for (i = 0; i < 4; ++i, ++txdp, ++pkts) {
160 buf_dma_addr = rte_mbuf_data_iova(*pkts);
161 pkt_len = (*pkts)->data_len;
163 /* write data to descriptor */
164 txdp->read.buffer_addr = rte_cpu_to_le_64(buf_dma_addr);
166 txdp->read.cmd_type_len =
167 rte_cpu_to_le_32((uint32_t)DCMD_DTYP_FLAGS | pkt_len);
169 txdp->read.olinfo_status =
170 rte_cpu_to_le_32(pkt_len << IXGBE_ADVTXD_PAYLEN_SHIFT);
172 rte_prefetch0(&(*pkts)->pool);
176 /* Populate 1 descriptor with data from 1 mbuf */
178 tx1(volatile union ixgbe_adv_tx_desc *txdp, struct rte_mbuf **pkts)
180 uint64_t buf_dma_addr;
183 buf_dma_addr = rte_mbuf_data_iova(*pkts);
184 pkt_len = (*pkts)->data_len;
186 /* write data to descriptor */
187 txdp->read.buffer_addr = rte_cpu_to_le_64(buf_dma_addr);
188 txdp->read.cmd_type_len =
189 rte_cpu_to_le_32((uint32_t)DCMD_DTYP_FLAGS | pkt_len);
190 txdp->read.olinfo_status =
191 rte_cpu_to_le_32(pkt_len << IXGBE_ADVTXD_PAYLEN_SHIFT);
192 rte_prefetch0(&(*pkts)->pool);
196 * Fill H/W descriptor ring with mbuf data.
197 * Copy mbuf pointers to the S/W ring.
200 ixgbe_tx_fill_hw_ring(struct ixgbe_tx_queue *txq, struct rte_mbuf **pkts,
203 volatile union ixgbe_adv_tx_desc *txdp = &(txq->tx_ring[txq->tx_tail]);
204 struct ixgbe_tx_entry *txep = &(txq->sw_ring[txq->tx_tail]);
205 const int N_PER_LOOP = 4;
206 const int N_PER_LOOP_MASK = N_PER_LOOP-1;
207 int mainpart, leftover;
211 * Process most of the packets in chunks of N pkts. Any
212 * leftover packets will get processed one at a time.
214 mainpart = (nb_pkts & ((uint32_t) ~N_PER_LOOP_MASK));
215 leftover = (nb_pkts & ((uint32_t) N_PER_LOOP_MASK));
216 for (i = 0; i < mainpart; i += N_PER_LOOP) {
217 /* Copy N mbuf pointers to the S/W ring */
218 for (j = 0; j < N_PER_LOOP; ++j) {
219 (txep + i + j)->mbuf = *(pkts + i + j);
221 tx4(txdp + i, pkts + i);
224 if (unlikely(leftover > 0)) {
225 for (i = 0; i < leftover; ++i) {
226 (txep + mainpart + i)->mbuf = *(pkts + mainpart + i);
227 tx1(txdp + mainpart + i, pkts + mainpart + i);
232 static inline uint16_t
233 tx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
236 struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue;
237 volatile union ixgbe_adv_tx_desc *tx_r = txq->tx_ring;
241 * Begin scanning the H/W ring for done descriptors when the
242 * number of available descriptors drops below tx_free_thresh. For
243 * each done descriptor, free the associated buffer.
245 if (txq->nb_tx_free < txq->tx_free_thresh)
246 ixgbe_tx_free_bufs(txq);
248 /* Only use descriptors that are available */
249 nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts);
250 if (unlikely(nb_pkts == 0))
253 /* Use exactly nb_pkts descriptors */
254 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts);
257 * At this point, we know there are enough descriptors in the
258 * ring to transmit all the packets. This assumes that each
259 * mbuf contains a single segment, and that no new offloads
260 * are expected, which would require a new context descriptor.
264 * See if we're going to wrap-around. If so, handle the top
265 * of the descriptor ring first, then do the bottom. If not,
266 * the processing looks just like the "bottom" part anyway...
268 if ((txq->tx_tail + nb_pkts) > txq->nb_tx_desc) {
269 n = (uint16_t)(txq->nb_tx_desc - txq->tx_tail);
270 ixgbe_tx_fill_hw_ring(txq, tx_pkts, n);
273 * We know that the last descriptor in the ring will need to
274 * have its RS bit set because tx_rs_thresh has to be
275 * a divisor of the ring size
277 tx_r[txq->tx_next_rs].read.cmd_type_len |=
278 rte_cpu_to_le_32(IXGBE_ADVTXD_DCMD_RS);
279 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
284 /* Fill H/W descriptor ring with mbuf data */
285 ixgbe_tx_fill_hw_ring(txq, tx_pkts + n, (uint16_t)(nb_pkts - n));
286 txq->tx_tail = (uint16_t)(txq->tx_tail + (nb_pkts - n));
289 * Determine if RS bit should be set
290 * This is what we actually want:
291 * if ((txq->tx_tail - 1) >= txq->tx_next_rs)
292 * but instead of subtracting 1 and doing >=, we can just do
293 * greater than without subtracting.
295 if (txq->tx_tail > txq->tx_next_rs) {
296 tx_r[txq->tx_next_rs].read.cmd_type_len |=
297 rte_cpu_to_le_32(IXGBE_ADVTXD_DCMD_RS);
298 txq->tx_next_rs = (uint16_t)(txq->tx_next_rs +
300 if (txq->tx_next_rs >= txq->nb_tx_desc)
301 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
305 * Check for wrap-around. This would only happen if we used
306 * up to the last descriptor in the ring, no more, no less.
308 if (txq->tx_tail >= txq->nb_tx_desc)
311 /* update tail pointer */
313 IXGBE_PCI_REG_WC_WRITE_RELAXED(txq->tdt_reg_addr, txq->tx_tail);
319 ixgbe_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts,
324 /* Try to transmit at least chunks of TX_MAX_BURST pkts */
325 if (likely(nb_pkts <= RTE_PMD_IXGBE_TX_MAX_BURST))
326 return tx_xmit_pkts(tx_queue, tx_pkts, nb_pkts);
328 /* transmit more than the max burst, in chunks of TX_MAX_BURST */
333 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_IXGBE_TX_MAX_BURST);
334 ret = tx_xmit_pkts(tx_queue, &(tx_pkts[nb_tx]), n);
335 nb_tx = (uint16_t)(nb_tx + ret);
336 nb_pkts = (uint16_t)(nb_pkts - ret);
345 ixgbe_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
349 struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue;
354 num = (uint16_t)RTE_MIN(nb_pkts, txq->tx_rs_thresh);
355 ret = ixgbe_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
367 ixgbe_set_xmit_ctx(struct ixgbe_tx_queue *txq,
368 volatile struct ixgbe_adv_tx_context_desc *ctx_txd,
369 uint64_t ol_flags, union ixgbe_tx_offload tx_offload,
370 __rte_unused uint64_t *mdata)
372 uint32_t type_tucmd_mlhl;
373 uint32_t mss_l4len_idx = 0;
375 uint32_t vlan_macip_lens;
376 union ixgbe_tx_offload tx_offload_mask;
377 uint32_t seqnum_seed = 0;
379 ctx_idx = txq->ctx_curr;
380 tx_offload_mask.data[0] = 0;
381 tx_offload_mask.data[1] = 0;
384 /* Specify which HW CTX to upload. */
385 mss_l4len_idx |= (ctx_idx << IXGBE_ADVTXD_IDX_SHIFT);
387 if (ol_flags & PKT_TX_VLAN_PKT) {
388 tx_offload_mask.vlan_tci |= ~0;
391 /* check if TCP segmentation required for this packet */
392 if (ol_flags & PKT_TX_TCP_SEG) {
393 /* implies IP cksum in IPv4 */
394 if (ol_flags & PKT_TX_IP_CKSUM)
395 type_tucmd_mlhl = IXGBE_ADVTXD_TUCMD_IPV4 |
396 IXGBE_ADVTXD_TUCMD_L4T_TCP |
397 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
399 type_tucmd_mlhl = IXGBE_ADVTXD_TUCMD_IPV6 |
400 IXGBE_ADVTXD_TUCMD_L4T_TCP |
401 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
403 tx_offload_mask.l2_len |= ~0;
404 tx_offload_mask.l3_len |= ~0;
405 tx_offload_mask.l4_len |= ~0;
406 tx_offload_mask.tso_segsz |= ~0;
407 mss_l4len_idx |= tx_offload.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT;
408 mss_l4len_idx |= tx_offload.l4_len << IXGBE_ADVTXD_L4LEN_SHIFT;
409 } else { /* no TSO, check if hardware checksum is needed */
410 if (ol_flags & PKT_TX_IP_CKSUM) {
411 type_tucmd_mlhl = IXGBE_ADVTXD_TUCMD_IPV4;
412 tx_offload_mask.l2_len |= ~0;
413 tx_offload_mask.l3_len |= ~0;
416 switch (ol_flags & PKT_TX_L4_MASK) {
417 case PKT_TX_UDP_CKSUM:
418 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP |
419 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
420 mss_l4len_idx |= sizeof(struct rte_udp_hdr)
421 << IXGBE_ADVTXD_L4LEN_SHIFT;
422 tx_offload_mask.l2_len |= ~0;
423 tx_offload_mask.l3_len |= ~0;
425 case PKT_TX_TCP_CKSUM:
426 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP |
427 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
428 mss_l4len_idx |= sizeof(struct rte_tcp_hdr)
429 << IXGBE_ADVTXD_L4LEN_SHIFT;
430 tx_offload_mask.l2_len |= ~0;
431 tx_offload_mask.l3_len |= ~0;
433 case PKT_TX_SCTP_CKSUM:
434 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP |
435 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
436 mss_l4len_idx |= sizeof(struct rte_sctp_hdr)
437 << IXGBE_ADVTXD_L4LEN_SHIFT;
438 tx_offload_mask.l2_len |= ~0;
439 tx_offload_mask.l3_len |= ~0;
442 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_RSV |
443 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
448 if (ol_flags & PKT_TX_OUTER_IP_CKSUM) {
449 tx_offload_mask.outer_l2_len |= ~0;
450 tx_offload_mask.outer_l3_len |= ~0;
451 tx_offload_mask.l2_len |= ~0;
452 seqnum_seed |= tx_offload.outer_l3_len
453 << IXGBE_ADVTXD_OUTER_IPLEN;
454 seqnum_seed |= tx_offload.l2_len
455 << IXGBE_ADVTXD_TUNNEL_LEN;
457 #ifdef RTE_LIB_SECURITY
458 if (ol_flags & PKT_TX_SEC_OFFLOAD) {
459 union ixgbe_crypto_tx_desc_md *md =
460 (union ixgbe_crypto_tx_desc_md *)mdata;
462 (IXGBE_ADVTXD_IPSEC_SA_INDEX_MASK & md->sa_idx);
463 type_tucmd_mlhl |= md->enc ?
464 (IXGBE_ADVTXD_TUCMD_IPSEC_TYPE_ESP |
465 IXGBE_ADVTXD_TUCMD_IPSEC_ENCRYPT_EN) : 0;
467 (md->pad_len & IXGBE_ADVTXD_IPSEC_ESP_LEN_MASK);
468 tx_offload_mask.sa_idx |= ~0;
469 tx_offload_mask.sec_pad_len |= ~0;
473 txq->ctx_cache[ctx_idx].flags = ol_flags;
474 txq->ctx_cache[ctx_idx].tx_offload.data[0] =
475 tx_offload_mask.data[0] & tx_offload.data[0];
476 txq->ctx_cache[ctx_idx].tx_offload.data[1] =
477 tx_offload_mask.data[1] & tx_offload.data[1];
478 txq->ctx_cache[ctx_idx].tx_offload_mask = tx_offload_mask;
480 ctx_txd->type_tucmd_mlhl = rte_cpu_to_le_32(type_tucmd_mlhl);
481 vlan_macip_lens = tx_offload.l3_len;
482 if (ol_flags & PKT_TX_OUTER_IP_CKSUM)
483 vlan_macip_lens |= (tx_offload.outer_l2_len <<
484 IXGBE_ADVTXD_MACLEN_SHIFT);
486 vlan_macip_lens |= (tx_offload.l2_len <<
487 IXGBE_ADVTXD_MACLEN_SHIFT);
488 vlan_macip_lens |= ((uint32_t)tx_offload.vlan_tci << IXGBE_ADVTXD_VLAN_SHIFT);
489 ctx_txd->vlan_macip_lens = rte_cpu_to_le_32(vlan_macip_lens);
490 ctx_txd->mss_l4len_idx = rte_cpu_to_le_32(mss_l4len_idx);
491 ctx_txd->seqnum_seed = seqnum_seed;
495 * Check which hardware context can be used. Use the existing match
496 * or create a new context descriptor.
498 static inline uint32_t
499 what_advctx_update(struct ixgbe_tx_queue *txq, uint64_t flags,
500 union ixgbe_tx_offload tx_offload)
502 /* If match with the current used context */
503 if (likely((txq->ctx_cache[txq->ctx_curr].flags == flags) &&
504 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] ==
505 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0]
506 & tx_offload.data[0])) &&
507 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] ==
508 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1]
509 & tx_offload.data[1]))))
510 return txq->ctx_curr;
512 /* What if match with the next context */
514 if (likely((txq->ctx_cache[txq->ctx_curr].flags == flags) &&
515 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] ==
516 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0]
517 & tx_offload.data[0])) &&
518 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] ==
519 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1]
520 & tx_offload.data[1]))))
521 return txq->ctx_curr;
523 /* Mismatch, use the previous context */
524 return IXGBE_CTX_NUM;
527 static inline uint32_t
528 tx_desc_cksum_flags_to_olinfo(uint64_t ol_flags)
532 if ((ol_flags & PKT_TX_L4_MASK) != PKT_TX_L4_NO_CKSUM)
533 tmp |= IXGBE_ADVTXD_POPTS_TXSM;
534 if (ol_flags & PKT_TX_IP_CKSUM)
535 tmp |= IXGBE_ADVTXD_POPTS_IXSM;
536 if (ol_flags & PKT_TX_TCP_SEG)
537 tmp |= IXGBE_ADVTXD_POPTS_TXSM;
541 static inline uint32_t
542 tx_desc_ol_flags_to_cmdtype(uint64_t ol_flags)
544 uint32_t cmdtype = 0;
546 if (ol_flags & PKT_TX_VLAN_PKT)
547 cmdtype |= IXGBE_ADVTXD_DCMD_VLE;
548 if (ol_flags & PKT_TX_TCP_SEG)
549 cmdtype |= IXGBE_ADVTXD_DCMD_TSE;
550 if (ol_flags & PKT_TX_OUTER_IP_CKSUM)
551 cmdtype |= (1 << IXGBE_ADVTXD_OUTERIPCS_SHIFT);
552 if (ol_flags & PKT_TX_MACSEC)
553 cmdtype |= IXGBE_ADVTXD_MAC_LINKSEC;
557 /* Default RS bit threshold values */
558 #ifndef DEFAULT_TX_RS_THRESH
559 #define DEFAULT_TX_RS_THRESH 32
561 #ifndef DEFAULT_TX_FREE_THRESH
562 #define DEFAULT_TX_FREE_THRESH 32
565 /* Reset transmit descriptors after they have been used */
567 ixgbe_xmit_cleanup(struct ixgbe_tx_queue *txq)
569 struct ixgbe_tx_entry *sw_ring = txq->sw_ring;
570 volatile union ixgbe_adv_tx_desc *txr = txq->tx_ring;
571 uint16_t last_desc_cleaned = txq->last_desc_cleaned;
572 uint16_t nb_tx_desc = txq->nb_tx_desc;
573 uint16_t desc_to_clean_to;
574 uint16_t nb_tx_to_clean;
577 /* Determine the last descriptor needing to be cleaned */
578 desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->tx_rs_thresh);
579 if (desc_to_clean_to >= nb_tx_desc)
580 desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc);
582 /* Check to make sure the last descriptor to clean is done */
583 desc_to_clean_to = sw_ring[desc_to_clean_to].last_id;
584 status = txr[desc_to_clean_to].wb.status;
585 if (!(status & rte_cpu_to_le_32(IXGBE_TXD_STAT_DD))) {
586 PMD_TX_FREE_LOG(DEBUG,
587 "TX descriptor %4u is not done"
588 "(port=%d queue=%d)",
590 txq->port_id, txq->queue_id);
591 /* Failed to clean any descriptors, better luck next time */
595 /* Figure out how many descriptors will be cleaned */
596 if (last_desc_cleaned > desc_to_clean_to)
597 nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) +
600 nb_tx_to_clean = (uint16_t)(desc_to_clean_to -
603 PMD_TX_FREE_LOG(DEBUG,
604 "Cleaning %4u TX descriptors: %4u to %4u "
605 "(port=%d queue=%d)",
606 nb_tx_to_clean, last_desc_cleaned, desc_to_clean_to,
607 txq->port_id, txq->queue_id);
610 * The last descriptor to clean is done, so that means all the
611 * descriptors from the last descriptor that was cleaned
612 * up to the last descriptor with the RS bit set
613 * are done. Only reset the threshold descriptor.
615 txr[desc_to_clean_to].wb.status = 0;
617 /* Update the txq to reflect the last descriptor that was cleaned */
618 txq->last_desc_cleaned = desc_to_clean_to;
619 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + nb_tx_to_clean);
626 ixgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
629 struct ixgbe_tx_queue *txq;
630 struct ixgbe_tx_entry *sw_ring;
631 struct ixgbe_tx_entry *txe, *txn;
632 volatile union ixgbe_adv_tx_desc *txr;
633 volatile union ixgbe_adv_tx_desc *txd, *txp;
634 struct rte_mbuf *tx_pkt;
635 struct rte_mbuf *m_seg;
636 uint64_t buf_dma_addr;
637 uint32_t olinfo_status;
638 uint32_t cmd_type_len;
649 union ixgbe_tx_offload tx_offload;
650 #ifdef RTE_LIB_SECURITY
654 tx_offload.data[0] = 0;
655 tx_offload.data[1] = 0;
657 sw_ring = txq->sw_ring;
659 tx_id = txq->tx_tail;
660 txe = &sw_ring[tx_id];
663 /* Determine if the descriptor ring needs to be cleaned. */
664 if (txq->nb_tx_free < txq->tx_free_thresh)
665 ixgbe_xmit_cleanup(txq);
667 rte_prefetch0(&txe->mbuf->pool);
670 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
673 pkt_len = tx_pkt->pkt_len;
676 * Determine how many (if any) context descriptors
677 * are needed for offload functionality.
679 ol_flags = tx_pkt->ol_flags;
680 #ifdef RTE_LIB_SECURITY
681 use_ipsec = txq->using_ipsec && (ol_flags & PKT_TX_SEC_OFFLOAD);
684 /* If hardware offload required */
685 tx_ol_req = ol_flags & IXGBE_TX_OFFLOAD_MASK;
687 tx_offload.l2_len = tx_pkt->l2_len;
688 tx_offload.l3_len = tx_pkt->l3_len;
689 tx_offload.l4_len = tx_pkt->l4_len;
690 tx_offload.vlan_tci = tx_pkt->vlan_tci;
691 tx_offload.tso_segsz = tx_pkt->tso_segsz;
692 tx_offload.outer_l2_len = tx_pkt->outer_l2_len;
693 tx_offload.outer_l3_len = tx_pkt->outer_l3_len;
694 #ifdef RTE_LIB_SECURITY
696 union ixgbe_crypto_tx_desc_md *ipsec_mdata =
697 (union ixgbe_crypto_tx_desc_md *)
698 rte_security_dynfield(tx_pkt);
699 tx_offload.sa_idx = ipsec_mdata->sa_idx;
700 tx_offload.sec_pad_len = ipsec_mdata->pad_len;
704 /* If new context need be built or reuse the exist ctx. */
705 ctx = what_advctx_update(txq, tx_ol_req,
707 /* Only allocate context descriptor if required*/
708 new_ctx = (ctx == IXGBE_CTX_NUM);
713 * Keep track of how many descriptors are used this loop
714 * This will always be the number of segments + the number of
715 * Context descriptors required to transmit the packet
717 nb_used = (uint16_t)(tx_pkt->nb_segs + new_ctx);
720 nb_used + txq->nb_tx_used >= txq->tx_rs_thresh)
721 /* set RS on the previous packet in the burst */
722 txp->read.cmd_type_len |=
723 rte_cpu_to_le_32(IXGBE_TXD_CMD_RS);
726 * The number of descriptors that must be allocated for a
727 * packet is the number of segments of that packet, plus 1
728 * Context Descriptor for the hardware offload, if any.
729 * Determine the last TX descriptor to allocate in the TX ring
730 * for the packet, starting from the current position (tx_id)
733 tx_last = (uint16_t) (tx_id + nb_used - 1);
736 if (tx_last >= txq->nb_tx_desc)
737 tx_last = (uint16_t) (tx_last - txq->nb_tx_desc);
739 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u pktlen=%u"
740 " tx_first=%u tx_last=%u",
741 (unsigned) txq->port_id,
742 (unsigned) txq->queue_id,
748 * Make sure there are enough TX descriptors available to
749 * transmit the entire packet.
750 * nb_used better be less than or equal to txq->tx_rs_thresh
752 if (nb_used > txq->nb_tx_free) {
753 PMD_TX_FREE_LOG(DEBUG,
754 "Not enough free TX descriptors "
755 "nb_used=%4u nb_free=%4u "
756 "(port=%d queue=%d)",
757 nb_used, txq->nb_tx_free,
758 txq->port_id, txq->queue_id);
760 if (ixgbe_xmit_cleanup(txq) != 0) {
761 /* Could not clean any descriptors */
767 /* nb_used better be <= txq->tx_rs_thresh */
768 if (unlikely(nb_used > txq->tx_rs_thresh)) {
769 PMD_TX_FREE_LOG(DEBUG,
770 "The number of descriptors needed to "
771 "transmit the packet exceeds the "
772 "RS bit threshold. This will impact "
774 "nb_used=%4u nb_free=%4u "
776 "(port=%d queue=%d)",
777 nb_used, txq->nb_tx_free,
779 txq->port_id, txq->queue_id);
781 * Loop here until there are enough TX
782 * descriptors or until the ring cannot be
785 while (nb_used > txq->nb_tx_free) {
786 if (ixgbe_xmit_cleanup(txq) != 0) {
788 * Could not clean any
800 * By now there are enough free TX descriptors to transmit
805 * Set common flags of all TX Data Descriptors.
807 * The following bits must be set in all Data Descriptors:
808 * - IXGBE_ADVTXD_DTYP_DATA
809 * - IXGBE_ADVTXD_DCMD_DEXT
811 * The following bits must be set in the first Data Descriptor
812 * and are ignored in the other ones:
813 * - IXGBE_ADVTXD_DCMD_IFCS
814 * - IXGBE_ADVTXD_MAC_1588
815 * - IXGBE_ADVTXD_DCMD_VLE
817 * The following bits must only be set in the last Data
819 * - IXGBE_TXD_CMD_EOP
821 * The following bits can be set in any Data Descriptor, but
822 * are only set in the last Data Descriptor:
825 cmd_type_len = IXGBE_ADVTXD_DTYP_DATA |
826 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT;
828 #ifdef RTE_LIBRTE_IEEE1588
829 if (ol_flags & PKT_TX_IEEE1588_TMST)
830 cmd_type_len |= IXGBE_ADVTXD_MAC_1588;
836 if (ol_flags & PKT_TX_TCP_SEG) {
837 /* when TSO is on, paylen in descriptor is the
838 * not the packet len but the tcp payload len */
839 pkt_len -= (tx_offload.l2_len +
840 tx_offload.l3_len + tx_offload.l4_len);
844 * Setup the TX Advanced Context Descriptor if required
847 volatile struct ixgbe_adv_tx_context_desc *
850 ctx_txd = (volatile struct
851 ixgbe_adv_tx_context_desc *)
854 txn = &sw_ring[txe->next_id];
855 rte_prefetch0(&txn->mbuf->pool);
857 if (txe->mbuf != NULL) {
858 rte_pktmbuf_free_seg(txe->mbuf);
862 ixgbe_set_xmit_ctx(txq, ctx_txd, tx_ol_req,
864 rte_security_dynfield(tx_pkt));
866 txe->last_id = tx_last;
867 tx_id = txe->next_id;
872 * Setup the TX Advanced Data Descriptor,
873 * This path will go through
874 * whatever new/reuse the context descriptor
876 cmd_type_len |= tx_desc_ol_flags_to_cmdtype(ol_flags);
877 olinfo_status |= tx_desc_cksum_flags_to_olinfo(ol_flags);
878 olinfo_status |= ctx << IXGBE_ADVTXD_IDX_SHIFT;
881 olinfo_status |= (pkt_len << IXGBE_ADVTXD_PAYLEN_SHIFT);
882 #ifdef RTE_LIB_SECURITY
884 olinfo_status |= IXGBE_ADVTXD_POPTS_IPSEC;
890 txn = &sw_ring[txe->next_id];
891 rte_prefetch0(&txn->mbuf->pool);
893 if (txe->mbuf != NULL)
894 rte_pktmbuf_free_seg(txe->mbuf);
898 * Set up Transmit Data Descriptor.
900 slen = m_seg->data_len;
901 buf_dma_addr = rte_mbuf_data_iova(m_seg);
902 txd->read.buffer_addr =
903 rte_cpu_to_le_64(buf_dma_addr);
904 txd->read.cmd_type_len =
905 rte_cpu_to_le_32(cmd_type_len | slen);
906 txd->read.olinfo_status =
907 rte_cpu_to_le_32(olinfo_status);
908 txe->last_id = tx_last;
909 tx_id = txe->next_id;
912 } while (m_seg != NULL);
915 * The last packet data descriptor needs End Of Packet (EOP)
917 cmd_type_len |= IXGBE_TXD_CMD_EOP;
918 txq->nb_tx_used = (uint16_t)(txq->nb_tx_used + nb_used);
919 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_used);
921 /* Set RS bit only on threshold packets' last descriptor */
922 if (txq->nb_tx_used >= txq->tx_rs_thresh) {
923 PMD_TX_FREE_LOG(DEBUG,
924 "Setting RS bit on TXD id="
925 "%4u (port=%d queue=%d)",
926 tx_last, txq->port_id, txq->queue_id);
928 cmd_type_len |= IXGBE_TXD_CMD_RS;
930 /* Update txq RS bit counters */
936 txd->read.cmd_type_len |= rte_cpu_to_le_32(cmd_type_len);
940 /* set RS on last packet in the burst */
942 txp->read.cmd_type_len |= rte_cpu_to_le_32(IXGBE_TXD_CMD_RS);
947 * Set the Transmit Descriptor Tail (TDT)
949 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u",
950 (unsigned) txq->port_id, (unsigned) txq->queue_id,
951 (unsigned) tx_id, (unsigned) nb_tx);
952 IXGBE_PCI_REG_WC_WRITE_RELAXED(txq->tdt_reg_addr, tx_id);
953 txq->tx_tail = tx_id;
958 /*********************************************************************
962 **********************************************************************/
964 ixgbe_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
969 struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue;
971 for (i = 0; i < nb_pkts; i++) {
973 ol_flags = m->ol_flags;
976 * Check if packet meets requirements for number of segments
978 * NOTE: for ixgbe it's always (40 - WTHRESH) for both TSO and
982 if (m->nb_segs > IXGBE_TX_MAX_SEG - txq->wthresh) {
987 if (ol_flags & IXGBE_TX_OFFLOAD_NOTSUP_MASK) {
992 /* check the size of packet */
993 if (m->pkt_len < IXGBE_TX_MIN_PKT_LEN) {
998 #ifdef RTE_LIBRTE_ETHDEV_DEBUG
999 ret = rte_validate_tx_offload(m);
1005 ret = rte_net_intel_cksum_prepare(m);
1015 /*********************************************************************
1019 **********************************************************************/
1021 #define IXGBE_PACKET_TYPE_ETHER 0X00
1022 #define IXGBE_PACKET_TYPE_IPV4 0X01
1023 #define IXGBE_PACKET_TYPE_IPV4_TCP 0X11
1024 #define IXGBE_PACKET_TYPE_IPV4_UDP 0X21
1025 #define IXGBE_PACKET_TYPE_IPV4_SCTP 0X41
1026 #define IXGBE_PACKET_TYPE_IPV4_EXT 0X03
1027 #define IXGBE_PACKET_TYPE_IPV4_EXT_TCP 0X13
1028 #define IXGBE_PACKET_TYPE_IPV4_EXT_UDP 0X23
1029 #define IXGBE_PACKET_TYPE_IPV4_EXT_SCTP 0X43
1030 #define IXGBE_PACKET_TYPE_IPV6 0X04
1031 #define IXGBE_PACKET_TYPE_IPV6_TCP 0X14
1032 #define IXGBE_PACKET_TYPE_IPV6_UDP 0X24
1033 #define IXGBE_PACKET_TYPE_IPV6_SCTP 0X44
1034 #define IXGBE_PACKET_TYPE_IPV6_EXT 0X0C
1035 #define IXGBE_PACKET_TYPE_IPV6_EXT_TCP 0X1C
1036 #define IXGBE_PACKET_TYPE_IPV6_EXT_UDP 0X2C
1037 #define IXGBE_PACKET_TYPE_IPV6_EXT_SCTP 0X4C
1038 #define IXGBE_PACKET_TYPE_IPV4_IPV6 0X05
1039 #define IXGBE_PACKET_TYPE_IPV4_IPV6_TCP 0X15
1040 #define IXGBE_PACKET_TYPE_IPV4_IPV6_UDP 0X25
1041 #define IXGBE_PACKET_TYPE_IPV4_IPV6_SCTP 0X45
1042 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6 0X07
1043 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_TCP 0X17
1044 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_UDP 0X27
1045 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_SCTP 0X47
1046 #define IXGBE_PACKET_TYPE_IPV4_IPV6_EXT 0X0D
1047 #define IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_TCP 0X1D
1048 #define IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_UDP 0X2D
1049 #define IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_SCTP 0X4D
1050 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT 0X0F
1051 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_TCP 0X1F
1052 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_UDP 0X2F
1053 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_SCTP 0X4F
1055 #define IXGBE_PACKET_TYPE_NVGRE 0X00
1056 #define IXGBE_PACKET_TYPE_NVGRE_IPV4 0X01
1057 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_TCP 0X11
1058 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_UDP 0X21
1059 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_SCTP 0X41
1060 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT 0X03
1061 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_TCP 0X13
1062 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_UDP 0X23
1063 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_SCTP 0X43
1064 #define IXGBE_PACKET_TYPE_NVGRE_IPV6 0X04
1065 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_TCP 0X14
1066 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_UDP 0X24
1067 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_SCTP 0X44
1068 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT 0X0C
1069 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_TCP 0X1C
1070 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_UDP 0X2C
1071 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_SCTP 0X4C
1072 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6 0X05
1073 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_TCP 0X15
1074 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_UDP 0X25
1075 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT 0X0D
1076 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_TCP 0X1D
1077 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_UDP 0X2D
1079 #define IXGBE_PACKET_TYPE_VXLAN 0X80
1080 #define IXGBE_PACKET_TYPE_VXLAN_IPV4 0X81
1081 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_TCP 0x91
1082 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_UDP 0xA1
1083 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_SCTP 0xC1
1084 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT 0x83
1085 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_TCP 0X93
1086 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_UDP 0XA3
1087 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_SCTP 0XC3
1088 #define IXGBE_PACKET_TYPE_VXLAN_IPV6 0X84
1089 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_TCP 0X94
1090 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_UDP 0XA4
1091 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_SCTP 0XC4
1092 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT 0X8C
1093 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_TCP 0X9C
1094 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_UDP 0XAC
1095 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_SCTP 0XCC
1096 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6 0X85
1097 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_TCP 0X95
1098 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_UDP 0XA5
1099 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT 0X8D
1100 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_TCP 0X9D
1101 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_UDP 0XAD
1104 * Use 2 different table for normal packet and tunnel packet
1105 * to save the space.
1108 ptype_table[IXGBE_PACKET_TYPE_MAX] __rte_cache_aligned = {
1109 [IXGBE_PACKET_TYPE_ETHER] = RTE_PTYPE_L2_ETHER,
1110 [IXGBE_PACKET_TYPE_IPV4] = RTE_PTYPE_L2_ETHER |
1112 [IXGBE_PACKET_TYPE_IPV4_TCP] = RTE_PTYPE_L2_ETHER |
1113 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP,
1114 [IXGBE_PACKET_TYPE_IPV4_UDP] = RTE_PTYPE_L2_ETHER |
1115 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP,
1116 [IXGBE_PACKET_TYPE_IPV4_SCTP] = RTE_PTYPE_L2_ETHER |
1117 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_SCTP,
1118 [IXGBE_PACKET_TYPE_IPV4_EXT] = RTE_PTYPE_L2_ETHER |
1119 RTE_PTYPE_L3_IPV4_EXT,
1120 [IXGBE_PACKET_TYPE_IPV4_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1121 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_TCP,
1122 [IXGBE_PACKET_TYPE_IPV4_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1123 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_UDP,
1124 [IXGBE_PACKET_TYPE_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1125 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_SCTP,
1126 [IXGBE_PACKET_TYPE_IPV6] = RTE_PTYPE_L2_ETHER |
1128 [IXGBE_PACKET_TYPE_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1129 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP,
1130 [IXGBE_PACKET_TYPE_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1131 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP,
1132 [IXGBE_PACKET_TYPE_IPV6_SCTP] = RTE_PTYPE_L2_ETHER |
1133 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_SCTP,
1134 [IXGBE_PACKET_TYPE_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1135 RTE_PTYPE_L3_IPV6_EXT,
1136 [IXGBE_PACKET_TYPE_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1137 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_TCP,
1138 [IXGBE_PACKET_TYPE_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1139 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_UDP,
1140 [IXGBE_PACKET_TYPE_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1141 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_SCTP,
1142 [IXGBE_PACKET_TYPE_IPV4_IPV6] = RTE_PTYPE_L2_ETHER |
1143 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1144 RTE_PTYPE_INNER_L3_IPV6,
1145 [IXGBE_PACKET_TYPE_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1146 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1147 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP,
1148 [IXGBE_PACKET_TYPE_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1149 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1150 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP,
1151 [IXGBE_PACKET_TYPE_IPV4_IPV6_SCTP] = RTE_PTYPE_L2_ETHER |
1152 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1153 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_SCTP,
1154 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6] = RTE_PTYPE_L2_ETHER |
1155 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1156 RTE_PTYPE_INNER_L3_IPV6,
1157 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1158 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1159 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP,
1160 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1161 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1162 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP,
1163 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_SCTP] = RTE_PTYPE_L2_ETHER |
1164 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1165 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_SCTP,
1166 [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1167 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1168 RTE_PTYPE_INNER_L3_IPV6_EXT,
1169 [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1170 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1171 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP,
1172 [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1173 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1174 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP,
1175 [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1176 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1177 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_SCTP,
1178 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1179 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1180 RTE_PTYPE_INNER_L3_IPV6_EXT,
1181 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1182 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1183 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP,
1184 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1185 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1186 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP,
1187 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_SCTP] =
1188 RTE_PTYPE_L2_ETHER |
1189 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1190 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_SCTP,
1194 ptype_table_tn[IXGBE_PACKET_TYPE_TN_MAX] __rte_cache_aligned = {
1195 [IXGBE_PACKET_TYPE_NVGRE] = RTE_PTYPE_L2_ETHER |
1196 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1197 RTE_PTYPE_INNER_L2_ETHER,
1198 [IXGBE_PACKET_TYPE_NVGRE_IPV4] = RTE_PTYPE_L2_ETHER |
1199 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1200 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1201 [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT] = RTE_PTYPE_L2_ETHER |
1202 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1203 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT,
1204 [IXGBE_PACKET_TYPE_NVGRE_IPV6] = RTE_PTYPE_L2_ETHER |
1205 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1206 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6,
1207 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6] = RTE_PTYPE_L2_ETHER |
1208 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1209 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1210 [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1211 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1212 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT,
1213 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1214 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1215 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1216 [IXGBE_PACKET_TYPE_NVGRE_IPV4_TCP] = RTE_PTYPE_L2_ETHER |
1217 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1218 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4 |
1219 RTE_PTYPE_INNER_L4_TCP,
1220 [IXGBE_PACKET_TYPE_NVGRE_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1221 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1222 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6 |
1223 RTE_PTYPE_INNER_L4_TCP,
1224 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1225 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1226 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1227 [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1228 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1229 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT |
1230 RTE_PTYPE_INNER_L4_TCP,
1231 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_TCP] =
1232 RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
1233 RTE_PTYPE_TUNNEL_GRE | RTE_PTYPE_INNER_L2_ETHER |
1234 RTE_PTYPE_INNER_L3_IPV4,
1235 [IXGBE_PACKET_TYPE_NVGRE_IPV4_UDP] = RTE_PTYPE_L2_ETHER |
1236 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1237 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4 |
1238 RTE_PTYPE_INNER_L4_UDP,
1239 [IXGBE_PACKET_TYPE_NVGRE_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1240 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1241 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6 |
1242 RTE_PTYPE_INNER_L4_UDP,
1243 [IXGBE_PACKET_TYPE_NVGRE_IPV6_SCTP] = RTE_PTYPE_L2_ETHER |
1244 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1245 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6 |
1246 RTE_PTYPE_INNER_L4_SCTP,
1247 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1248 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1249 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1250 [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1251 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1252 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT |
1253 RTE_PTYPE_INNER_L4_UDP,
1254 [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1255 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1256 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT |
1257 RTE_PTYPE_INNER_L4_SCTP,
1258 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_UDP] =
1259 RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
1260 RTE_PTYPE_TUNNEL_GRE | RTE_PTYPE_INNER_L2_ETHER |
1261 RTE_PTYPE_INNER_L3_IPV4,
1262 [IXGBE_PACKET_TYPE_NVGRE_IPV4_SCTP] = RTE_PTYPE_L2_ETHER |
1263 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1264 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4 |
1265 RTE_PTYPE_INNER_L4_SCTP,
1266 [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1267 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1268 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT |
1269 RTE_PTYPE_INNER_L4_SCTP,
1270 [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1271 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1272 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT |
1273 RTE_PTYPE_INNER_L4_TCP,
1274 [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1275 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1276 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT |
1277 RTE_PTYPE_INNER_L4_UDP,
1279 [IXGBE_PACKET_TYPE_VXLAN] = RTE_PTYPE_L2_ETHER |
1280 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1281 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER,
1282 [IXGBE_PACKET_TYPE_VXLAN_IPV4] = RTE_PTYPE_L2_ETHER |
1283 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1284 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1285 RTE_PTYPE_INNER_L3_IPV4,
1286 [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT] = RTE_PTYPE_L2_ETHER |
1287 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1288 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1289 RTE_PTYPE_INNER_L3_IPV4_EXT,
1290 [IXGBE_PACKET_TYPE_VXLAN_IPV6] = RTE_PTYPE_L2_ETHER |
1291 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1292 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1293 RTE_PTYPE_INNER_L3_IPV6,
1294 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6] = RTE_PTYPE_L2_ETHER |
1295 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1296 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1297 RTE_PTYPE_INNER_L3_IPV4,
1298 [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1299 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1300 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1301 RTE_PTYPE_INNER_L3_IPV6_EXT,
1302 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1303 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1304 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1305 RTE_PTYPE_INNER_L3_IPV4,
1306 [IXGBE_PACKET_TYPE_VXLAN_IPV4_TCP] = RTE_PTYPE_L2_ETHER |
1307 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1308 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1309 RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L4_TCP,
1310 [IXGBE_PACKET_TYPE_VXLAN_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1311 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1312 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1313 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP,
1314 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1315 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1316 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1317 RTE_PTYPE_INNER_L3_IPV4,
1318 [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1319 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1320 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1321 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP,
1322 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_TCP] =
1323 RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
1324 RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_VXLAN |
1325 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1326 [IXGBE_PACKET_TYPE_VXLAN_IPV4_UDP] = RTE_PTYPE_L2_ETHER |
1327 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1328 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1329 RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L4_UDP,
1330 [IXGBE_PACKET_TYPE_VXLAN_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1331 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1332 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1333 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP,
1334 [IXGBE_PACKET_TYPE_VXLAN_IPV6_SCTP] = RTE_PTYPE_L2_ETHER |
1335 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1336 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1337 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_SCTP,
1338 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1339 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1340 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1341 RTE_PTYPE_INNER_L3_IPV4,
1342 [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1343 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1344 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1345 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP,
1346 [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1347 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1348 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1349 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_SCTP,
1350 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_UDP] =
1351 RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
1352 RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_VXLAN |
1353 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1354 [IXGBE_PACKET_TYPE_VXLAN_IPV4_SCTP] = RTE_PTYPE_L2_ETHER |
1355 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1356 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1357 RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L4_SCTP,
1358 [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1359 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1360 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1361 RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_SCTP,
1362 [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1363 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1364 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1365 RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_TCP,
1366 [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1367 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1368 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1369 RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_UDP,
1373 ixgbe_get_monitor_addr(void *rx_queue, struct rte_power_monitor_cond *pmc)
1375 volatile union ixgbe_adv_rx_desc *rxdp;
1376 struct ixgbe_rx_queue *rxq = rx_queue;
1379 desc = rxq->rx_tail;
1380 rxdp = &rxq->rx_ring[desc];
1381 /* watch for changes in status bit */
1382 pmc->addr = &rxdp->wb.upper.status_error;
1385 * we expect the DD bit to be set to 1 if this descriptor was already
1388 pmc->val = rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD);
1389 pmc->mask = rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD);
1391 /* the registers are 32-bit */
1392 pmc->size = sizeof(uint32_t);
1397 /* @note: fix ixgbe_dev_supported_ptypes_get() if any change here. */
1398 static inline uint32_t
1399 ixgbe_rxd_pkt_info_to_pkt_type(uint32_t pkt_info, uint16_t ptype_mask)
1402 if (unlikely(pkt_info & IXGBE_RXDADV_PKTTYPE_ETQF))
1403 return RTE_PTYPE_UNKNOWN;
1405 pkt_info = (pkt_info >> IXGBE_PACKET_TYPE_SHIFT) & ptype_mask;
1407 /* For tunnel packet */
1408 if (pkt_info & IXGBE_PACKET_TYPE_TUNNEL_BIT) {
1409 /* Remove the tunnel bit to save the space. */
1410 pkt_info &= IXGBE_PACKET_TYPE_MASK_TUNNEL;
1411 return ptype_table_tn[pkt_info];
1415 * For x550, if it's not tunnel,
1416 * tunnel type bit should be set to 0.
1417 * Reuse 82599's mask.
1419 pkt_info &= IXGBE_PACKET_TYPE_MASK_82599;
1421 return ptype_table[pkt_info];
1424 static inline uint64_t
1425 ixgbe_rxd_pkt_info_to_pkt_flags(uint16_t pkt_info)
1427 static uint64_t ip_rss_types_map[16] __rte_cache_aligned = {
1428 0, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH,
1429 0, PKT_RX_RSS_HASH, 0, PKT_RX_RSS_HASH,
1430 PKT_RX_RSS_HASH, 0, 0, 0,
1431 0, 0, 0, PKT_RX_FDIR,
1433 #ifdef RTE_LIBRTE_IEEE1588
1434 static uint64_t ip_pkt_etqf_map[8] = {
1435 0, 0, 0, PKT_RX_IEEE1588_PTP,
1439 if (likely(pkt_info & IXGBE_RXDADV_PKTTYPE_ETQF))
1440 return ip_pkt_etqf_map[(pkt_info >> 4) & 0X07] |
1441 ip_rss_types_map[pkt_info & 0XF];
1443 return ip_rss_types_map[pkt_info & 0XF];
1445 return ip_rss_types_map[pkt_info & 0XF];
1449 static inline uint64_t
1450 rx_desc_status_to_pkt_flags(uint32_t rx_status, uint64_t vlan_flags)
1455 * Check if VLAN present only.
1456 * Do not check whether L3/L4 rx checksum done by NIC or not,
1457 * That can be found from rte_eth_rxmode.offloads flag
1459 pkt_flags = (rx_status & IXGBE_RXD_STAT_VP) ? vlan_flags : 0;
1461 #ifdef RTE_LIBRTE_IEEE1588
1462 if (rx_status & IXGBE_RXD_STAT_TMST)
1463 pkt_flags = pkt_flags | PKT_RX_IEEE1588_TMST;
1468 static inline uint64_t
1469 rx_desc_error_to_pkt_flags(uint32_t rx_status)
1474 * Bit 31: IPE, IPv4 checksum error
1475 * Bit 30: L4I, L4I integrity error
1477 static uint64_t error_to_pkt_flags_map[4] = {
1478 PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD,
1479 PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD,
1480 PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_GOOD,
1481 PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD
1483 pkt_flags = error_to_pkt_flags_map[(rx_status >>
1484 IXGBE_RXDADV_ERR_CKSUM_BIT) & IXGBE_RXDADV_ERR_CKSUM_MSK];
1486 if ((rx_status & IXGBE_RXD_STAT_OUTERIPCS) &&
1487 (rx_status & IXGBE_RXDADV_ERR_OUTERIPER)) {
1488 pkt_flags |= PKT_RX_EIP_CKSUM_BAD;
1491 #ifdef RTE_LIB_SECURITY
1492 if (rx_status & IXGBE_RXD_STAT_SECP) {
1493 pkt_flags |= PKT_RX_SEC_OFFLOAD;
1494 if (rx_status & IXGBE_RXDADV_LNKSEC_ERROR_BAD_SIG)
1495 pkt_flags |= PKT_RX_SEC_OFFLOAD_FAILED;
1503 * LOOK_AHEAD defines how many desc statuses to check beyond the
1504 * current descriptor.
1505 * It must be a pound define for optimal performance.
1506 * Do not change the value of LOOK_AHEAD, as the ixgbe_rx_scan_hw_ring
1507 * function only works with LOOK_AHEAD=8.
1509 #define LOOK_AHEAD 8
1510 #if (LOOK_AHEAD != 8)
1511 #error "PMD IXGBE: LOOK_AHEAD must be 8\n"
1514 ixgbe_rx_scan_hw_ring(struct ixgbe_rx_queue *rxq)
1516 volatile union ixgbe_adv_rx_desc *rxdp;
1517 struct ixgbe_rx_entry *rxep;
1518 struct rte_mbuf *mb;
1522 uint32_t s[LOOK_AHEAD];
1523 uint32_t pkt_info[LOOK_AHEAD];
1524 int i, j, nb_rx = 0;
1526 uint64_t vlan_flags = rxq->vlan_flags;
1528 /* get references to current descriptor and S/W ring entry */
1529 rxdp = &rxq->rx_ring[rxq->rx_tail];
1530 rxep = &rxq->sw_ring[rxq->rx_tail];
1532 status = rxdp->wb.upper.status_error;
1533 /* check to make sure there is at least 1 packet to receive */
1534 if (!(status & rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD)))
1538 * Scan LOOK_AHEAD descriptors at a time to determine which descriptors
1539 * reference packets that are ready to be received.
1541 for (i = 0; i < RTE_PMD_IXGBE_RX_MAX_BURST;
1542 i += LOOK_AHEAD, rxdp += LOOK_AHEAD, rxep += LOOK_AHEAD) {
1543 /* Read desc statuses backwards to avoid race condition */
1544 for (j = 0; j < LOOK_AHEAD; j++)
1545 s[j] = rte_le_to_cpu_32(rxdp[j].wb.upper.status_error);
1549 /* Compute how many status bits were set */
1550 for (nb_dd = 0; nb_dd < LOOK_AHEAD &&
1551 (s[nb_dd] & IXGBE_RXDADV_STAT_DD); nb_dd++)
1554 for (j = 0; j < nb_dd; j++)
1555 pkt_info[j] = rte_le_to_cpu_32(rxdp[j].wb.lower.
1560 /* Translate descriptor info to mbuf format */
1561 for (j = 0; j < nb_dd; ++j) {
1563 pkt_len = rte_le_to_cpu_16(rxdp[j].wb.upper.length) -
1565 mb->data_len = pkt_len;
1566 mb->pkt_len = pkt_len;
1567 mb->vlan_tci = rte_le_to_cpu_16(rxdp[j].wb.upper.vlan);
1569 /* convert descriptor fields to rte mbuf flags */
1570 pkt_flags = rx_desc_status_to_pkt_flags(s[j],
1572 pkt_flags |= rx_desc_error_to_pkt_flags(s[j]);
1573 pkt_flags |= ixgbe_rxd_pkt_info_to_pkt_flags
1574 ((uint16_t)pkt_info[j]);
1575 mb->ol_flags = pkt_flags;
1577 ixgbe_rxd_pkt_info_to_pkt_type
1578 (pkt_info[j], rxq->pkt_type_mask);
1580 if (likely(pkt_flags & PKT_RX_RSS_HASH))
1581 mb->hash.rss = rte_le_to_cpu_32(
1582 rxdp[j].wb.lower.hi_dword.rss);
1583 else if (pkt_flags & PKT_RX_FDIR) {
1584 mb->hash.fdir.hash = rte_le_to_cpu_16(
1585 rxdp[j].wb.lower.hi_dword.csum_ip.csum) &
1586 IXGBE_ATR_HASH_MASK;
1587 mb->hash.fdir.id = rte_le_to_cpu_16(
1588 rxdp[j].wb.lower.hi_dword.csum_ip.ip_id);
1592 /* Move mbuf pointers from the S/W ring to the stage */
1593 for (j = 0; j < LOOK_AHEAD; ++j) {
1594 rxq->rx_stage[i + j] = rxep[j].mbuf;
1597 /* stop if all requested packets could not be received */
1598 if (nb_dd != LOOK_AHEAD)
1602 /* clear software ring entries so we can cleanup correctly */
1603 for (i = 0; i < nb_rx; ++i) {
1604 rxq->sw_ring[rxq->rx_tail + i].mbuf = NULL;
1612 ixgbe_rx_alloc_bufs(struct ixgbe_rx_queue *rxq, bool reset_mbuf)
1614 volatile union ixgbe_adv_rx_desc *rxdp;
1615 struct ixgbe_rx_entry *rxep;
1616 struct rte_mbuf *mb;
1621 /* allocate buffers in bulk directly into the S/W ring */
1622 alloc_idx = rxq->rx_free_trigger - (rxq->rx_free_thresh - 1);
1623 rxep = &rxq->sw_ring[alloc_idx];
1624 diag = rte_mempool_get_bulk(rxq->mb_pool, (void *)rxep,
1625 rxq->rx_free_thresh);
1626 if (unlikely(diag != 0))
1629 rxdp = &rxq->rx_ring[alloc_idx];
1630 for (i = 0; i < rxq->rx_free_thresh; ++i) {
1631 /* populate the static rte mbuf fields */
1634 mb->port = rxq->port_id;
1637 rte_mbuf_refcnt_set(mb, 1);
1638 mb->data_off = RTE_PKTMBUF_HEADROOM;
1640 /* populate the descriptors */
1641 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mb));
1642 rxdp[i].read.hdr_addr = 0;
1643 rxdp[i].read.pkt_addr = dma_addr;
1646 /* update state of internal queue structure */
1647 rxq->rx_free_trigger = rxq->rx_free_trigger + rxq->rx_free_thresh;
1648 if (rxq->rx_free_trigger >= rxq->nb_rx_desc)
1649 rxq->rx_free_trigger = rxq->rx_free_thresh - 1;
1655 static inline uint16_t
1656 ixgbe_rx_fill_from_stage(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
1659 struct rte_mbuf **stage = &rxq->rx_stage[rxq->rx_next_avail];
1662 /* how many packets are ready to return? */
1663 nb_pkts = (uint16_t)RTE_MIN(nb_pkts, rxq->rx_nb_avail);
1665 /* copy mbuf pointers to the application's packet list */
1666 for (i = 0; i < nb_pkts; ++i)
1667 rx_pkts[i] = stage[i];
1669 /* update internal queue state */
1670 rxq->rx_nb_avail = (uint16_t)(rxq->rx_nb_avail - nb_pkts);
1671 rxq->rx_next_avail = (uint16_t)(rxq->rx_next_avail + nb_pkts);
1676 static inline uint16_t
1677 rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
1680 struct ixgbe_rx_queue *rxq = (struct ixgbe_rx_queue *)rx_queue;
1683 /* Any previously recv'd pkts will be returned from the Rx stage */
1684 if (rxq->rx_nb_avail)
1685 return ixgbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
1687 /* Scan the H/W ring for packets to receive */
1688 nb_rx = (uint16_t)ixgbe_rx_scan_hw_ring(rxq);
1690 /* update internal queue state */
1691 rxq->rx_next_avail = 0;
1692 rxq->rx_nb_avail = nb_rx;
1693 rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_rx);
1695 /* if required, allocate new buffers to replenish descriptors */
1696 if (rxq->rx_tail > rxq->rx_free_trigger) {
1697 uint16_t cur_free_trigger = rxq->rx_free_trigger;
1699 if (ixgbe_rx_alloc_bufs(rxq, true) != 0) {
1702 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u "
1703 "queue_id=%u", (unsigned) rxq->port_id,
1704 (unsigned) rxq->queue_id);
1706 rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed +=
1707 rxq->rx_free_thresh;
1710 * Need to rewind any previous receives if we cannot
1711 * allocate new buffers to replenish the old ones.
1713 rxq->rx_nb_avail = 0;
1714 rxq->rx_tail = (uint16_t)(rxq->rx_tail - nb_rx);
1715 for (i = 0, j = rxq->rx_tail; i < nb_rx; ++i, ++j)
1716 rxq->sw_ring[j].mbuf = rxq->rx_stage[i];
1721 /* update tail pointer */
1723 IXGBE_PCI_REG_WC_WRITE_RELAXED(rxq->rdt_reg_addr,
1727 if (rxq->rx_tail >= rxq->nb_rx_desc)
1730 /* received any packets this loop? */
1731 if (rxq->rx_nb_avail)
1732 return ixgbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
1737 /* split requests into chunks of size RTE_PMD_IXGBE_RX_MAX_BURST */
1739 ixgbe_recv_pkts_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
1744 if (unlikely(nb_pkts == 0))
1747 if (likely(nb_pkts <= RTE_PMD_IXGBE_RX_MAX_BURST))
1748 return rx_recv_pkts(rx_queue, rx_pkts, nb_pkts);
1750 /* request is relatively large, chunk it up */
1755 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_IXGBE_RX_MAX_BURST);
1756 ret = rx_recv_pkts(rx_queue, &rx_pkts[nb_rx], n);
1757 nb_rx = (uint16_t)(nb_rx + ret);
1758 nb_pkts = (uint16_t)(nb_pkts - ret);
1767 ixgbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
1770 struct ixgbe_rx_queue *rxq;
1771 volatile union ixgbe_adv_rx_desc *rx_ring;
1772 volatile union ixgbe_adv_rx_desc *rxdp;
1773 struct ixgbe_rx_entry *sw_ring;
1774 struct ixgbe_rx_entry *rxe;
1775 struct rte_mbuf *rxm;
1776 struct rte_mbuf *nmb;
1777 union ixgbe_adv_rx_desc rxd;
1786 uint64_t vlan_flags;
1791 rx_id = rxq->rx_tail;
1792 rx_ring = rxq->rx_ring;
1793 sw_ring = rxq->sw_ring;
1794 vlan_flags = rxq->vlan_flags;
1795 while (nb_rx < nb_pkts) {
1797 * The order of operations here is important as the DD status
1798 * bit must not be read after any other descriptor fields.
1799 * rx_ring and rxdp are pointing to volatile data so the order
1800 * of accesses cannot be reordered by the compiler. If they were
1801 * not volatile, they could be reordered which could lead to
1802 * using invalid descriptor fields when read from rxd.
1804 rxdp = &rx_ring[rx_id];
1805 staterr = rxdp->wb.upper.status_error;
1806 if (!(staterr & rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD)))
1813 * If the IXGBE_RXDADV_STAT_EOP flag is not set, the RX packet
1814 * is likely to be invalid and to be dropped by the various
1815 * validation checks performed by the network stack.
1817 * Allocate a new mbuf to replenish the RX ring descriptor.
1818 * If the allocation fails:
1819 * - arrange for that RX descriptor to be the first one
1820 * being parsed the next time the receive function is
1821 * invoked [on the same queue].
1823 * - Stop parsing the RX ring and return immediately.
1825 * This policy do not drop the packet received in the RX
1826 * descriptor for which the allocation of a new mbuf failed.
1827 * Thus, it allows that packet to be later retrieved if
1828 * mbuf have been freed in the mean time.
1829 * As a side effect, holding RX descriptors instead of
1830 * systematically giving them back to the NIC may lead to
1831 * RX ring exhaustion situations.
1832 * However, the NIC can gracefully prevent such situations
1833 * to happen by sending specific "back-pressure" flow control
1834 * frames to its peer(s).
1836 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u "
1837 "ext_err_stat=0x%08x pkt_len=%u",
1838 (unsigned) rxq->port_id, (unsigned) rxq->queue_id,
1839 (unsigned) rx_id, (unsigned) staterr,
1840 (unsigned) rte_le_to_cpu_16(rxd.wb.upper.length));
1842 nmb = rte_mbuf_raw_alloc(rxq->mb_pool);
1844 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u "
1845 "queue_id=%u", (unsigned) rxq->port_id,
1846 (unsigned) rxq->queue_id);
1847 rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed++;
1852 rxe = &sw_ring[rx_id];
1854 if (rx_id == rxq->nb_rx_desc)
1857 /* Prefetch next mbuf while processing current one. */
1858 rte_ixgbe_prefetch(sw_ring[rx_id].mbuf);
1861 * When next RX descriptor is on a cache-line boundary,
1862 * prefetch the next 4 RX descriptors and the next 8 pointers
1865 if ((rx_id & 0x3) == 0) {
1866 rte_ixgbe_prefetch(&rx_ring[rx_id]);
1867 rte_ixgbe_prefetch(&sw_ring[rx_id]);
1873 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
1874 rxdp->read.hdr_addr = 0;
1875 rxdp->read.pkt_addr = dma_addr;
1878 * Initialize the returned mbuf.
1879 * 1) setup generic mbuf fields:
1880 * - number of segments,
1883 * - RX port identifier.
1884 * 2) integrate hardware offload data, if any:
1885 * - RSS flag & hash,
1886 * - IP checksum flag,
1887 * - VLAN TCI, if any,
1890 pkt_len = (uint16_t) (rte_le_to_cpu_16(rxd.wb.upper.length) -
1892 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1893 rte_packet_prefetch((char *)rxm->buf_addr + rxm->data_off);
1896 rxm->pkt_len = pkt_len;
1897 rxm->data_len = pkt_len;
1898 rxm->port = rxq->port_id;
1900 pkt_info = rte_le_to_cpu_32(rxd.wb.lower.lo_dword.data);
1901 /* Only valid if PKT_RX_VLAN set in pkt_flags */
1902 rxm->vlan_tci = rte_le_to_cpu_16(rxd.wb.upper.vlan);
1904 pkt_flags = rx_desc_status_to_pkt_flags(staterr, vlan_flags);
1905 pkt_flags = pkt_flags | rx_desc_error_to_pkt_flags(staterr);
1906 pkt_flags = pkt_flags |
1907 ixgbe_rxd_pkt_info_to_pkt_flags((uint16_t)pkt_info);
1908 rxm->ol_flags = pkt_flags;
1910 ixgbe_rxd_pkt_info_to_pkt_type(pkt_info,
1911 rxq->pkt_type_mask);
1913 if (likely(pkt_flags & PKT_RX_RSS_HASH))
1914 rxm->hash.rss = rte_le_to_cpu_32(
1915 rxd.wb.lower.hi_dword.rss);
1916 else if (pkt_flags & PKT_RX_FDIR) {
1917 rxm->hash.fdir.hash = rte_le_to_cpu_16(
1918 rxd.wb.lower.hi_dword.csum_ip.csum) &
1919 IXGBE_ATR_HASH_MASK;
1920 rxm->hash.fdir.id = rte_le_to_cpu_16(
1921 rxd.wb.lower.hi_dword.csum_ip.ip_id);
1924 * Store the mbuf address into the next entry of the array
1925 * of returned packets.
1927 rx_pkts[nb_rx++] = rxm;
1929 rxq->rx_tail = rx_id;
1932 * If the number of free RX descriptors is greater than the RX free
1933 * threshold of the queue, advance the Receive Descriptor Tail (RDT)
1935 * Update the RDT with the value of the last processed RX descriptor
1936 * minus 1, to guarantee that the RDT register is never equal to the
1937 * RDH register, which creates a "full" ring situtation from the
1938 * hardware point of view...
1940 nb_hold = (uint16_t) (nb_hold + rxq->nb_rx_hold);
1941 if (nb_hold > rxq->rx_free_thresh) {
1942 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u "
1943 "nb_hold=%u nb_rx=%u",
1944 (unsigned) rxq->port_id, (unsigned) rxq->queue_id,
1945 (unsigned) rx_id, (unsigned) nb_hold,
1947 rx_id = (uint16_t) ((rx_id == 0) ?
1948 (rxq->nb_rx_desc - 1) : (rx_id - 1));
1949 IXGBE_PCI_REG_WC_WRITE(rxq->rdt_reg_addr, rx_id);
1952 rxq->nb_rx_hold = nb_hold;
1957 * Detect an RSC descriptor.
1959 static inline uint32_t
1960 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
1962 return (rte_le_to_cpu_32(rx->wb.lower.lo_dword.data) &
1963 IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
1967 * ixgbe_fill_cluster_head_buf - fill the first mbuf of the returned packet
1969 * Fill the following info in the HEAD buffer of the Rx cluster:
1970 * - RX port identifier
1971 * - hardware offload data, if any:
1973 * - IP checksum flag
1974 * - VLAN TCI, if any
1976 * @head HEAD of the packet cluster
1977 * @desc HW descriptor to get data from
1978 * @rxq Pointer to the Rx queue
1981 ixgbe_fill_cluster_head_buf(
1982 struct rte_mbuf *head,
1983 union ixgbe_adv_rx_desc *desc,
1984 struct ixgbe_rx_queue *rxq,
1990 head->port = rxq->port_id;
1992 /* The vlan_tci field is only valid when PKT_RX_VLAN is
1993 * set in the pkt_flags field.
1995 head->vlan_tci = rte_le_to_cpu_16(desc->wb.upper.vlan);
1996 pkt_info = rte_le_to_cpu_32(desc->wb.lower.lo_dword.data);
1997 pkt_flags = rx_desc_status_to_pkt_flags(staterr, rxq->vlan_flags);
1998 pkt_flags |= rx_desc_error_to_pkt_flags(staterr);
1999 pkt_flags |= ixgbe_rxd_pkt_info_to_pkt_flags((uint16_t)pkt_info);
2000 head->ol_flags = pkt_flags;
2002 ixgbe_rxd_pkt_info_to_pkt_type(pkt_info, rxq->pkt_type_mask);
2004 if (likely(pkt_flags & PKT_RX_RSS_HASH))
2005 head->hash.rss = rte_le_to_cpu_32(desc->wb.lower.hi_dword.rss);
2006 else if (pkt_flags & PKT_RX_FDIR) {
2007 head->hash.fdir.hash =
2008 rte_le_to_cpu_16(desc->wb.lower.hi_dword.csum_ip.csum)
2009 & IXGBE_ATR_HASH_MASK;
2010 head->hash.fdir.id =
2011 rte_le_to_cpu_16(desc->wb.lower.hi_dword.csum_ip.ip_id);
2016 * ixgbe_recv_pkts_lro - receive handler for and LRO case.
2018 * @rx_queue Rx queue handle
2019 * @rx_pkts table of received packets
2020 * @nb_pkts size of rx_pkts table
2021 * @bulk_alloc if TRUE bulk allocation is used for a HW ring refilling
2023 * Handles the Rx HW ring completions when RSC feature is configured. Uses an
2024 * additional ring of ixgbe_rsc_entry's that will hold the relevant RSC info.
2026 * We use the same logic as in Linux and in FreeBSD ixgbe drivers:
2027 * 1) When non-EOP RSC completion arrives:
2028 * a) Update the HEAD of the current RSC aggregation cluster with the new
2029 * segment's data length.
2030 * b) Set the "next" pointer of the current segment to point to the segment
2031 * at the NEXTP index.
2032 * c) Pass the HEAD of RSC aggregation cluster on to the next NEXTP entry
2033 * in the sw_rsc_ring.
2034 * 2) When EOP arrives we just update the cluster's total length and offload
2035 * flags and deliver the cluster up to the upper layers. In our case - put it
2036 * in the rx_pkts table.
2038 * Returns the number of received packets/clusters (according to the "bulk
2039 * receive" interface).
2041 static inline uint16_t
2042 ixgbe_recv_pkts_lro(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts,
2045 struct ixgbe_rx_queue *rxq = rx_queue;
2046 volatile union ixgbe_adv_rx_desc *rx_ring = rxq->rx_ring;
2047 struct ixgbe_rx_entry *sw_ring = rxq->sw_ring;
2048 struct ixgbe_scattered_rx_entry *sw_sc_ring = rxq->sw_sc_ring;
2049 uint16_t rx_id = rxq->rx_tail;
2051 uint16_t nb_hold = rxq->nb_rx_hold;
2052 uint16_t prev_id = rxq->rx_tail;
2054 while (nb_rx < nb_pkts) {
2056 struct ixgbe_rx_entry *rxe;
2057 struct ixgbe_scattered_rx_entry *sc_entry;
2058 struct ixgbe_scattered_rx_entry *next_sc_entry = NULL;
2059 struct ixgbe_rx_entry *next_rxe = NULL;
2060 struct rte_mbuf *first_seg;
2061 struct rte_mbuf *rxm;
2062 struct rte_mbuf *nmb = NULL;
2063 union ixgbe_adv_rx_desc rxd;
2066 volatile union ixgbe_adv_rx_desc *rxdp;
2071 * The code in this whole file uses the volatile pointer to
2072 * ensure the read ordering of the status and the rest of the
2073 * descriptor fields (on the compiler level only!!!). This is so
2074 * UGLY - why not to just use the compiler barrier instead? DPDK
2075 * even has the rte_compiler_barrier() for that.
2077 * But most importantly this is just wrong because this doesn't
2078 * ensure memory ordering in a general case at all. For
2079 * instance, DPDK is supposed to work on Power CPUs where
2080 * compiler barrier may just not be enough!
2082 * I tried to write only this function properly to have a
2083 * starting point (as a part of an LRO/RSC series) but the
2084 * compiler cursed at me when I tried to cast away the
2085 * "volatile" from rx_ring (yes, it's volatile too!!!). So, I'm
2086 * keeping it the way it is for now.
2088 * The code in this file is broken in so many other places and
2089 * will just not work on a big endian CPU anyway therefore the
2090 * lines below will have to be revisited together with the rest
2094 * - Get rid of "volatile" and let the compiler do its job.
2095 * - Use the proper memory barrier (rte_rmb()) to ensure the
2096 * memory ordering below.
2098 rxdp = &rx_ring[rx_id];
2099 staterr = rte_le_to_cpu_32(rxdp->wb.upper.status_error);
2101 if (!(staterr & IXGBE_RXDADV_STAT_DD))
2106 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u "
2107 "staterr=0x%x data_len=%u",
2108 rxq->port_id, rxq->queue_id, rx_id, staterr,
2109 rte_le_to_cpu_16(rxd.wb.upper.length));
2112 nmb = rte_mbuf_raw_alloc(rxq->mb_pool);
2114 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed "
2115 "port_id=%u queue_id=%u",
2116 rxq->port_id, rxq->queue_id);
2118 rte_eth_devices[rxq->port_id].data->
2119 rx_mbuf_alloc_failed++;
2122 } else if (nb_hold > rxq->rx_free_thresh) {
2123 uint16_t next_rdt = rxq->rx_free_trigger;
2125 if (!ixgbe_rx_alloc_bufs(rxq, false)) {
2127 IXGBE_PCI_REG_WC_WRITE_RELAXED(
2130 nb_hold -= rxq->rx_free_thresh;
2132 PMD_RX_LOG(DEBUG, "RX bulk alloc failed "
2133 "port_id=%u queue_id=%u",
2134 rxq->port_id, rxq->queue_id);
2136 rte_eth_devices[rxq->port_id].data->
2137 rx_mbuf_alloc_failed++;
2143 rxe = &sw_ring[rx_id];
2144 eop = staterr & IXGBE_RXDADV_STAT_EOP;
2146 next_id = rx_id + 1;
2147 if (next_id == rxq->nb_rx_desc)
2150 /* Prefetch next mbuf while processing current one. */
2151 rte_ixgbe_prefetch(sw_ring[next_id].mbuf);
2154 * When next RX descriptor is on a cache-line boundary,
2155 * prefetch the next 4 RX descriptors and the next 4 pointers
2158 if ((next_id & 0x3) == 0) {
2159 rte_ixgbe_prefetch(&rx_ring[next_id]);
2160 rte_ixgbe_prefetch(&sw_ring[next_id]);
2167 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
2169 * Update RX descriptor with the physical address of the
2170 * new data buffer of the new allocated mbuf.
2174 rxm->data_off = RTE_PKTMBUF_HEADROOM;
2175 rxdp->read.hdr_addr = 0;
2176 rxdp->read.pkt_addr = dma;
2181 * Set data length & data buffer address of mbuf.
2183 data_len = rte_le_to_cpu_16(rxd.wb.upper.length);
2184 rxm->data_len = data_len;
2189 * Get next descriptor index:
2190 * - For RSC it's in the NEXTP field.
2191 * - For a scattered packet - it's just a following
2194 if (ixgbe_rsc_count(&rxd))
2196 (staterr & IXGBE_RXDADV_NEXTP_MASK) >>
2197 IXGBE_RXDADV_NEXTP_SHIFT;
2201 next_sc_entry = &sw_sc_ring[nextp_id];
2202 next_rxe = &sw_ring[nextp_id];
2203 rte_ixgbe_prefetch(next_rxe);
2206 sc_entry = &sw_sc_ring[rx_id];
2207 first_seg = sc_entry->fbuf;
2208 sc_entry->fbuf = NULL;
2211 * If this is the first buffer of the received packet,
2212 * set the pointer to the first mbuf of the packet and
2213 * initialize its context.
2214 * Otherwise, update the total length and the number of segments
2215 * of the current scattered packet, and update the pointer to
2216 * the last mbuf of the current packet.
2218 if (first_seg == NULL) {
2220 first_seg->pkt_len = data_len;
2221 first_seg->nb_segs = 1;
2223 first_seg->pkt_len += data_len;
2224 first_seg->nb_segs++;
2231 * If this is not the last buffer of the received packet, update
2232 * the pointer to the first mbuf at the NEXTP entry in the
2233 * sw_sc_ring and continue to parse the RX ring.
2235 if (!eop && next_rxe) {
2236 rxm->next = next_rxe->mbuf;
2237 next_sc_entry->fbuf = first_seg;
2241 /* Initialize the first mbuf of the returned packet */
2242 ixgbe_fill_cluster_head_buf(first_seg, &rxd, rxq, staterr);
2245 * Deal with the case, when HW CRC srip is disabled.
2246 * That can't happen when LRO is enabled, but still could
2247 * happen for scattered RX mode.
2249 first_seg->pkt_len -= rxq->crc_len;
2250 if (unlikely(rxm->data_len <= rxq->crc_len)) {
2251 struct rte_mbuf *lp;
2253 for (lp = first_seg; lp->next != rxm; lp = lp->next)
2256 first_seg->nb_segs--;
2257 lp->data_len -= rxq->crc_len - rxm->data_len;
2259 rte_pktmbuf_free_seg(rxm);
2261 rxm->data_len -= rxq->crc_len;
2263 /* Prefetch data of first segment, if configured to do so. */
2264 rte_packet_prefetch((char *)first_seg->buf_addr +
2265 first_seg->data_off);
2268 * Store the mbuf address into the next entry of the array
2269 * of returned packets.
2271 rx_pkts[nb_rx++] = first_seg;
2275 * Record index of the next RX descriptor to probe.
2277 rxq->rx_tail = rx_id;
2280 * If the number of free RX descriptors is greater than the RX free
2281 * threshold of the queue, advance the Receive Descriptor Tail (RDT)
2283 * Update the RDT with the value of the last processed RX descriptor
2284 * minus 1, to guarantee that the RDT register is never equal to the
2285 * RDH register, which creates a "full" ring situtation from the
2286 * hardware point of view...
2288 if (!bulk_alloc && nb_hold > rxq->rx_free_thresh) {
2289 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u "
2290 "nb_hold=%u nb_rx=%u",
2291 rxq->port_id, rxq->queue_id, rx_id, nb_hold, nb_rx);
2294 IXGBE_PCI_REG_WC_WRITE_RELAXED(rxq->rdt_reg_addr, prev_id);
2298 rxq->nb_rx_hold = nb_hold;
2303 ixgbe_recv_pkts_lro_single_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
2306 return ixgbe_recv_pkts_lro(rx_queue, rx_pkts, nb_pkts, false);
2310 ixgbe_recv_pkts_lro_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
2313 return ixgbe_recv_pkts_lro(rx_queue, rx_pkts, nb_pkts, true);
2316 /*********************************************************************
2318 * Queue management functions
2320 **********************************************************************/
2322 static void __rte_cold
2323 ixgbe_tx_queue_release_mbufs(struct ixgbe_tx_queue *txq)
2327 if (txq->sw_ring != NULL) {
2328 for (i = 0; i < txq->nb_tx_desc; i++) {
2329 if (txq->sw_ring[i].mbuf != NULL) {
2330 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
2331 txq->sw_ring[i].mbuf = NULL;
2338 ixgbe_tx_done_cleanup_full(struct ixgbe_tx_queue *txq, uint32_t free_cnt)
2340 struct ixgbe_tx_entry *swr_ring = txq->sw_ring;
2341 uint16_t i, tx_last, tx_id;
2342 uint16_t nb_tx_free_last;
2343 uint16_t nb_tx_to_clean;
2346 /* Start free mbuf from the next of tx_tail */
2347 tx_last = txq->tx_tail;
2348 tx_id = swr_ring[tx_last].next_id;
2350 if (txq->nb_tx_free == 0 && ixgbe_xmit_cleanup(txq))
2353 nb_tx_to_clean = txq->nb_tx_free;
2354 nb_tx_free_last = txq->nb_tx_free;
2356 free_cnt = txq->nb_tx_desc;
2358 /* Loop through swr_ring to count the amount of
2359 * freeable mubfs and packets.
2361 for (pkt_cnt = 0; pkt_cnt < free_cnt; ) {
2362 for (i = 0; i < nb_tx_to_clean &&
2363 pkt_cnt < free_cnt &&
2364 tx_id != tx_last; i++) {
2365 if (swr_ring[tx_id].mbuf != NULL) {
2366 rte_pktmbuf_free_seg(swr_ring[tx_id].mbuf);
2367 swr_ring[tx_id].mbuf = NULL;
2370 * last segment in the packet,
2371 * increment packet count
2373 pkt_cnt += (swr_ring[tx_id].last_id == tx_id);
2376 tx_id = swr_ring[tx_id].next_id;
2379 if (txq->tx_rs_thresh > txq->nb_tx_desc -
2380 txq->nb_tx_free || tx_id == tx_last)
2383 if (pkt_cnt < free_cnt) {
2384 if (ixgbe_xmit_cleanup(txq))
2387 nb_tx_to_clean = txq->nb_tx_free - nb_tx_free_last;
2388 nb_tx_free_last = txq->nb_tx_free;
2392 return (int)pkt_cnt;
2396 ixgbe_tx_done_cleanup_simple(struct ixgbe_tx_queue *txq,
2401 if (free_cnt == 0 || free_cnt > txq->nb_tx_desc)
2402 free_cnt = txq->nb_tx_desc;
2404 cnt = free_cnt - free_cnt % txq->tx_rs_thresh;
2406 for (i = 0; i < cnt; i += n) {
2407 if (txq->nb_tx_desc - txq->nb_tx_free < txq->tx_rs_thresh)
2410 n = ixgbe_tx_free_bufs(txq);
2420 ixgbe_tx_done_cleanup_vec(struct ixgbe_tx_queue *txq __rte_unused,
2421 uint32_t free_cnt __rte_unused)
2427 ixgbe_dev_tx_done_cleanup(void *tx_queue, uint32_t free_cnt)
2429 struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue;
2430 if (txq->offloads == 0 &&
2431 #ifdef RTE_LIB_SECURITY
2432 !(txq->using_ipsec) &&
2434 txq->tx_rs_thresh >= RTE_PMD_IXGBE_TX_MAX_BURST) {
2435 if (txq->tx_rs_thresh <= RTE_IXGBE_TX_MAX_FREE_BUF_SZ &&
2436 rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128 &&
2437 (rte_eal_process_type() != RTE_PROC_PRIMARY ||
2438 txq->sw_ring_v != NULL)) {
2439 return ixgbe_tx_done_cleanup_vec(txq, free_cnt);
2441 return ixgbe_tx_done_cleanup_simple(txq, free_cnt);
2445 return ixgbe_tx_done_cleanup_full(txq, free_cnt);
2448 static void __rte_cold
2449 ixgbe_tx_free_swring(struct ixgbe_tx_queue *txq)
2452 txq->sw_ring != NULL)
2453 rte_free(txq->sw_ring);
2456 static void __rte_cold
2457 ixgbe_tx_queue_release(struct ixgbe_tx_queue *txq)
2459 if (txq != NULL && txq->ops != NULL) {
2460 txq->ops->release_mbufs(txq);
2461 txq->ops->free_swring(txq);
2467 ixgbe_dev_tx_queue_release(void *txq)
2469 ixgbe_tx_queue_release(txq);
2472 /* (Re)set dynamic ixgbe_tx_queue fields to defaults */
2473 static void __rte_cold
2474 ixgbe_reset_tx_queue(struct ixgbe_tx_queue *txq)
2476 static const union ixgbe_adv_tx_desc zeroed_desc = {{0}};
2477 struct ixgbe_tx_entry *txe = txq->sw_ring;
2480 /* Zero out HW ring memory */
2481 for (i = 0; i < txq->nb_tx_desc; i++) {
2482 txq->tx_ring[i] = zeroed_desc;
2485 /* Initialize SW ring entries */
2486 prev = (uint16_t) (txq->nb_tx_desc - 1);
2487 for (i = 0; i < txq->nb_tx_desc; i++) {
2488 volatile union ixgbe_adv_tx_desc *txd = &txq->tx_ring[i];
2490 txd->wb.status = rte_cpu_to_le_32(IXGBE_TXD_STAT_DD);
2493 txe[prev].next_id = i;
2497 txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
2498 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
2501 txq->nb_tx_used = 0;
2503 * Always allow 1 descriptor to be un-allocated to avoid
2504 * a H/W race condition
2506 txq->last_desc_cleaned = (uint16_t)(txq->nb_tx_desc - 1);
2507 txq->nb_tx_free = (uint16_t)(txq->nb_tx_desc - 1);
2509 memset((void *)&txq->ctx_cache, 0,
2510 IXGBE_CTX_NUM * sizeof(struct ixgbe_advctx_info));
2513 static const struct ixgbe_txq_ops def_txq_ops = {
2514 .release_mbufs = ixgbe_tx_queue_release_mbufs,
2515 .free_swring = ixgbe_tx_free_swring,
2516 .reset = ixgbe_reset_tx_queue,
2519 /* Takes an ethdev and a queue and sets up the tx function to be used based on
2520 * the queue parameters. Used in tx_queue_setup by primary process and then
2521 * in dev_init by secondary process when attaching to an existing ethdev.
2524 ixgbe_set_tx_function(struct rte_eth_dev *dev, struct ixgbe_tx_queue *txq)
2526 /* Use a simple Tx queue (no offloads, no multi segs) if possible */
2527 if ((txq->offloads == 0) &&
2528 #ifdef RTE_LIB_SECURITY
2529 !(txq->using_ipsec) &&
2531 (txq->tx_rs_thresh >= RTE_PMD_IXGBE_TX_MAX_BURST)) {
2532 PMD_INIT_LOG(DEBUG, "Using simple tx code path");
2533 dev->tx_pkt_prepare = NULL;
2534 if (txq->tx_rs_thresh <= RTE_IXGBE_TX_MAX_FREE_BUF_SZ &&
2535 rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128 &&
2536 (rte_eal_process_type() != RTE_PROC_PRIMARY ||
2537 ixgbe_txq_vec_setup(txq) == 0)) {
2538 PMD_INIT_LOG(DEBUG, "Vector tx enabled.");
2539 dev->tx_pkt_burst = ixgbe_xmit_pkts_vec;
2541 dev->tx_pkt_burst = ixgbe_xmit_pkts_simple;
2543 PMD_INIT_LOG(DEBUG, "Using full-featured tx code path");
2545 " - offloads = 0x%" PRIx64,
2548 " - tx_rs_thresh = %lu " "[RTE_PMD_IXGBE_TX_MAX_BURST=%lu]",
2549 (unsigned long)txq->tx_rs_thresh,
2550 (unsigned long)RTE_PMD_IXGBE_TX_MAX_BURST);
2551 dev->tx_pkt_burst = ixgbe_xmit_pkts;
2552 dev->tx_pkt_prepare = ixgbe_prep_pkts;
2557 ixgbe_get_tx_queue_offloads(struct rte_eth_dev *dev)
2565 ixgbe_get_tx_port_offloads(struct rte_eth_dev *dev)
2567 uint64_t tx_offload_capa;
2568 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2571 DEV_TX_OFFLOAD_VLAN_INSERT |
2572 DEV_TX_OFFLOAD_IPV4_CKSUM |
2573 DEV_TX_OFFLOAD_UDP_CKSUM |
2574 DEV_TX_OFFLOAD_TCP_CKSUM |
2575 DEV_TX_OFFLOAD_SCTP_CKSUM |
2576 DEV_TX_OFFLOAD_TCP_TSO |
2577 DEV_TX_OFFLOAD_MULTI_SEGS;
2579 if (hw->mac.type == ixgbe_mac_82599EB ||
2580 hw->mac.type == ixgbe_mac_X540)
2581 tx_offload_capa |= DEV_TX_OFFLOAD_MACSEC_INSERT;
2583 if (hw->mac.type == ixgbe_mac_X550 ||
2584 hw->mac.type == ixgbe_mac_X550EM_x ||
2585 hw->mac.type == ixgbe_mac_X550EM_a)
2586 tx_offload_capa |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM;
2588 #ifdef RTE_LIB_SECURITY
2589 if (dev->security_ctx)
2590 tx_offload_capa |= DEV_TX_OFFLOAD_SECURITY;
2592 return tx_offload_capa;
2596 ixgbe_dev_tx_queue_setup(struct rte_eth_dev *dev,
2599 unsigned int socket_id,
2600 const struct rte_eth_txconf *tx_conf)
2602 const struct rte_memzone *tz;
2603 struct ixgbe_tx_queue *txq;
2604 struct ixgbe_hw *hw;
2605 uint16_t tx_rs_thresh, tx_free_thresh;
2608 PMD_INIT_FUNC_TRACE();
2609 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2611 offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads;
2614 * Validate number of transmit descriptors.
2615 * It must not exceed hardware maximum, and must be multiple
2618 if (nb_desc % IXGBE_TXD_ALIGN != 0 ||
2619 (nb_desc > IXGBE_MAX_RING_DESC) ||
2620 (nb_desc < IXGBE_MIN_RING_DESC)) {
2625 * The following two parameters control the setting of the RS bit on
2626 * transmit descriptors.
2627 * TX descriptors will have their RS bit set after txq->tx_rs_thresh
2628 * descriptors have been used.
2629 * The TX descriptor ring will be cleaned after txq->tx_free_thresh
2630 * descriptors are used or if the number of descriptors required
2631 * to transmit a packet is greater than the number of free TX
2633 * The following constraints must be satisfied:
2634 * tx_rs_thresh must be greater than 0.
2635 * tx_rs_thresh must be less than the size of the ring minus 2.
2636 * tx_rs_thresh must be less than or equal to tx_free_thresh.
2637 * tx_rs_thresh must be a divisor of the ring size.
2638 * tx_free_thresh must be greater than 0.
2639 * tx_free_thresh must be less than the size of the ring minus 3.
2640 * tx_free_thresh + tx_rs_thresh must not exceed nb_desc.
2641 * One descriptor in the TX ring is used as a sentinel to avoid a
2642 * H/W race condition, hence the maximum threshold constraints.
2643 * When set to zero use default values.
2645 tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ?
2646 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH);
2647 /* force tx_rs_thresh to adapt an aggresive tx_free_thresh */
2648 tx_rs_thresh = (DEFAULT_TX_RS_THRESH + tx_free_thresh > nb_desc) ?
2649 nb_desc - tx_free_thresh : DEFAULT_TX_RS_THRESH;
2650 if (tx_conf->tx_rs_thresh > 0)
2651 tx_rs_thresh = tx_conf->tx_rs_thresh;
2652 if (tx_rs_thresh + tx_free_thresh > nb_desc) {
2653 PMD_INIT_LOG(ERR, "tx_rs_thresh + tx_free_thresh must not "
2654 "exceed nb_desc. (tx_rs_thresh=%u "
2655 "tx_free_thresh=%u nb_desc=%u port = %d queue=%d)",
2656 (unsigned int)tx_rs_thresh,
2657 (unsigned int)tx_free_thresh,
2658 (unsigned int)nb_desc,
2659 (int)dev->data->port_id,
2663 if (tx_rs_thresh >= (nb_desc - 2)) {
2664 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than the number "
2665 "of TX descriptors minus 2. (tx_rs_thresh=%u "
2666 "port=%d queue=%d)", (unsigned int)tx_rs_thresh,
2667 (int)dev->data->port_id, (int)queue_idx);
2670 if (tx_rs_thresh > DEFAULT_TX_RS_THRESH) {
2671 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less or equal than %u. "
2672 "(tx_rs_thresh=%u port=%d queue=%d)",
2673 DEFAULT_TX_RS_THRESH, (unsigned int)tx_rs_thresh,
2674 (int)dev->data->port_id, (int)queue_idx);
2677 if (tx_free_thresh >= (nb_desc - 3)) {
2678 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than the "
2679 "tx_free_thresh must be less than the number of "
2680 "TX descriptors minus 3. (tx_free_thresh=%u "
2681 "port=%d queue=%d)",
2682 (unsigned int)tx_free_thresh,
2683 (int)dev->data->port_id, (int)queue_idx);
2686 if (tx_rs_thresh > tx_free_thresh) {
2687 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than or equal to "
2688 "tx_free_thresh. (tx_free_thresh=%u "
2689 "tx_rs_thresh=%u port=%d queue=%d)",
2690 (unsigned int)tx_free_thresh,
2691 (unsigned int)tx_rs_thresh,
2692 (int)dev->data->port_id,
2696 if ((nb_desc % tx_rs_thresh) != 0) {
2697 PMD_INIT_LOG(ERR, "tx_rs_thresh must be a divisor of the "
2698 "number of TX descriptors. (tx_rs_thresh=%u "
2699 "port=%d queue=%d)", (unsigned int)tx_rs_thresh,
2700 (int)dev->data->port_id, (int)queue_idx);
2705 * If rs_bit_thresh is greater than 1, then TX WTHRESH should be
2706 * set to 0. If WTHRESH is greater than zero, the RS bit is ignored
2707 * by the NIC and all descriptors are written back after the NIC
2708 * accumulates WTHRESH descriptors.
2710 if ((tx_rs_thresh > 1) && (tx_conf->tx_thresh.wthresh != 0)) {
2711 PMD_INIT_LOG(ERR, "TX WTHRESH must be set to 0 if "
2712 "tx_rs_thresh is greater than 1. (tx_rs_thresh=%u "
2713 "port=%d queue=%d)", (unsigned int)tx_rs_thresh,
2714 (int)dev->data->port_id, (int)queue_idx);
2718 /* Free memory prior to re-allocation if needed... */
2719 if (dev->data->tx_queues[queue_idx] != NULL) {
2720 ixgbe_tx_queue_release(dev->data->tx_queues[queue_idx]);
2721 dev->data->tx_queues[queue_idx] = NULL;
2724 /* First allocate the tx queue data structure */
2725 txq = rte_zmalloc_socket("ethdev TX queue", sizeof(struct ixgbe_tx_queue),
2726 RTE_CACHE_LINE_SIZE, socket_id);
2731 * Allocate TX ring hardware descriptors. A memzone large enough to
2732 * handle the maximum ring size is allocated in order to allow for
2733 * resizing in later calls to the queue setup function.
2735 tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx,
2736 sizeof(union ixgbe_adv_tx_desc) * IXGBE_MAX_RING_DESC,
2737 IXGBE_ALIGN, socket_id);
2739 ixgbe_tx_queue_release(txq);
2743 txq->nb_tx_desc = nb_desc;
2744 txq->tx_rs_thresh = tx_rs_thresh;
2745 txq->tx_free_thresh = tx_free_thresh;
2746 txq->pthresh = tx_conf->tx_thresh.pthresh;
2747 txq->hthresh = tx_conf->tx_thresh.hthresh;
2748 txq->wthresh = tx_conf->tx_thresh.wthresh;
2749 txq->queue_id = queue_idx;
2750 txq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ?
2751 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx);
2752 txq->port_id = dev->data->port_id;
2753 txq->offloads = offloads;
2754 txq->ops = &def_txq_ops;
2755 txq->tx_deferred_start = tx_conf->tx_deferred_start;
2756 #ifdef RTE_LIB_SECURITY
2757 txq->using_ipsec = !!(dev->data->dev_conf.txmode.offloads &
2758 DEV_TX_OFFLOAD_SECURITY);
2762 * Modification to set VFTDT for virtual function if vf is detected
2764 if (hw->mac.type == ixgbe_mac_82599_vf ||
2765 hw->mac.type == ixgbe_mac_X540_vf ||
2766 hw->mac.type == ixgbe_mac_X550_vf ||
2767 hw->mac.type == ixgbe_mac_X550EM_x_vf ||
2768 hw->mac.type == ixgbe_mac_X550EM_a_vf)
2769 txq->tdt_reg_addr = IXGBE_PCI_REG_ADDR(hw, IXGBE_VFTDT(queue_idx));
2771 txq->tdt_reg_addr = IXGBE_PCI_REG_ADDR(hw, IXGBE_TDT(txq->reg_idx));
2773 txq->tx_ring_phys_addr = tz->iova;
2774 txq->tx_ring = (union ixgbe_adv_tx_desc *) tz->addr;
2776 /* Allocate software ring */
2777 txq->sw_ring = rte_zmalloc_socket("txq->sw_ring",
2778 sizeof(struct ixgbe_tx_entry) * nb_desc,
2779 RTE_CACHE_LINE_SIZE, socket_id);
2780 if (txq->sw_ring == NULL) {
2781 ixgbe_tx_queue_release(txq);
2784 PMD_INIT_LOG(DEBUG, "sw_ring=%p hw_ring=%p dma_addr=0x%"PRIx64,
2785 txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr);
2787 /* set up vector or scalar TX function as appropriate */
2788 ixgbe_set_tx_function(dev, txq);
2790 txq->ops->reset(txq);
2792 dev->data->tx_queues[queue_idx] = txq;
2799 * ixgbe_free_sc_cluster - free the not-yet-completed scattered cluster
2801 * The "next" pointer of the last segment of (not-yet-completed) RSC clusters
2802 * in the sw_rsc_ring is not set to NULL but rather points to the next
2803 * mbuf of this RSC aggregation (that has not been completed yet and still
2804 * resides on the HW ring). So, instead of calling for rte_pktmbuf_free() we
2805 * will just free first "nb_segs" segments of the cluster explicitly by calling
2806 * an rte_pktmbuf_free_seg().
2808 * @m scattered cluster head
2810 static void __rte_cold
2811 ixgbe_free_sc_cluster(struct rte_mbuf *m)
2813 uint16_t i, nb_segs = m->nb_segs;
2814 struct rte_mbuf *next_seg;
2816 for (i = 0; i < nb_segs; i++) {
2818 rte_pktmbuf_free_seg(m);
2823 static void __rte_cold
2824 ixgbe_rx_queue_release_mbufs(struct ixgbe_rx_queue *rxq)
2828 /* SSE Vector driver has a different way of releasing mbufs. */
2829 if (rxq->rx_using_sse) {
2830 ixgbe_rx_queue_release_mbufs_vec(rxq);
2834 if (rxq->sw_ring != NULL) {
2835 for (i = 0; i < rxq->nb_rx_desc; i++) {
2836 if (rxq->sw_ring[i].mbuf != NULL) {
2837 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
2838 rxq->sw_ring[i].mbuf = NULL;
2841 if (rxq->rx_nb_avail) {
2842 for (i = 0; i < rxq->rx_nb_avail; ++i) {
2843 struct rte_mbuf *mb;
2845 mb = rxq->rx_stage[rxq->rx_next_avail + i];
2846 rte_pktmbuf_free_seg(mb);
2848 rxq->rx_nb_avail = 0;
2852 if (rxq->sw_sc_ring)
2853 for (i = 0; i < rxq->nb_rx_desc; i++)
2854 if (rxq->sw_sc_ring[i].fbuf) {
2855 ixgbe_free_sc_cluster(rxq->sw_sc_ring[i].fbuf);
2856 rxq->sw_sc_ring[i].fbuf = NULL;
2860 static void __rte_cold
2861 ixgbe_rx_queue_release(struct ixgbe_rx_queue *rxq)
2864 ixgbe_rx_queue_release_mbufs(rxq);
2865 rte_free(rxq->sw_ring);
2866 rte_free(rxq->sw_sc_ring);
2872 ixgbe_dev_rx_queue_release(void *rxq)
2874 ixgbe_rx_queue_release(rxq);
2878 * Check if Rx Burst Bulk Alloc function can be used.
2880 * 0: the preconditions are satisfied and the bulk allocation function
2882 * -EINVAL: the preconditions are NOT satisfied and the default Rx burst
2883 * function must be used.
2885 static inline int __rte_cold
2886 check_rx_burst_bulk_alloc_preconditions(struct ixgbe_rx_queue *rxq)
2891 * Make sure the following pre-conditions are satisfied:
2892 * rxq->rx_free_thresh >= RTE_PMD_IXGBE_RX_MAX_BURST
2893 * rxq->rx_free_thresh < rxq->nb_rx_desc
2894 * (rxq->nb_rx_desc % rxq->rx_free_thresh) == 0
2895 * Scattered packets are not supported. This should be checked
2896 * outside of this function.
2898 if (!(rxq->rx_free_thresh >= RTE_PMD_IXGBE_RX_MAX_BURST)) {
2899 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
2900 "rxq->rx_free_thresh=%d, "
2901 "RTE_PMD_IXGBE_RX_MAX_BURST=%d",
2902 rxq->rx_free_thresh, RTE_PMD_IXGBE_RX_MAX_BURST);
2904 } else if (!(rxq->rx_free_thresh < rxq->nb_rx_desc)) {
2905 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
2906 "rxq->rx_free_thresh=%d, "
2907 "rxq->nb_rx_desc=%d",
2908 rxq->rx_free_thresh, rxq->nb_rx_desc);
2910 } else if (!((rxq->nb_rx_desc % rxq->rx_free_thresh) == 0)) {
2911 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
2912 "rxq->nb_rx_desc=%d, "
2913 "rxq->rx_free_thresh=%d",
2914 rxq->nb_rx_desc, rxq->rx_free_thresh);
2921 /* Reset dynamic ixgbe_rx_queue fields back to defaults */
2922 static void __rte_cold
2923 ixgbe_reset_rx_queue(struct ixgbe_adapter *adapter, struct ixgbe_rx_queue *rxq)
2925 static const union ixgbe_adv_rx_desc zeroed_desc = {{0}};
2927 uint16_t len = rxq->nb_rx_desc;
2930 * By default, the Rx queue setup function allocates enough memory for
2931 * IXGBE_MAX_RING_DESC. The Rx Burst bulk allocation function requires
2932 * extra memory at the end of the descriptor ring to be zero'd out.
2934 if (adapter->rx_bulk_alloc_allowed)
2935 /* zero out extra memory */
2936 len += RTE_PMD_IXGBE_RX_MAX_BURST;
2939 * Zero out HW ring memory. Zero out extra memory at the end of
2940 * the H/W ring so look-ahead logic in Rx Burst bulk alloc function
2941 * reads extra memory as zeros.
2943 for (i = 0; i < len; i++) {
2944 rxq->rx_ring[i] = zeroed_desc;
2948 * initialize extra software ring entries. Space for these extra
2949 * entries is always allocated
2951 memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf));
2952 for (i = rxq->nb_rx_desc; i < len; ++i) {
2953 rxq->sw_ring[i].mbuf = &rxq->fake_mbuf;
2956 rxq->rx_nb_avail = 0;
2957 rxq->rx_next_avail = 0;
2958 rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1);
2960 rxq->nb_rx_hold = 0;
2961 rxq->pkt_first_seg = NULL;
2962 rxq->pkt_last_seg = NULL;
2964 #if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64)
2965 rxq->rxrearm_start = 0;
2966 rxq->rxrearm_nb = 0;
2971 ixgbe_is_vf(struct rte_eth_dev *dev)
2973 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2975 switch (hw->mac.type) {
2976 case ixgbe_mac_82599_vf:
2977 case ixgbe_mac_X540_vf:
2978 case ixgbe_mac_X550_vf:
2979 case ixgbe_mac_X550EM_x_vf:
2980 case ixgbe_mac_X550EM_a_vf:
2988 ixgbe_get_rx_queue_offloads(struct rte_eth_dev *dev)
2990 uint64_t offloads = 0;
2991 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2993 if (hw->mac.type != ixgbe_mac_82598EB)
2994 offloads |= DEV_RX_OFFLOAD_VLAN_STRIP;
3000 ixgbe_get_rx_port_offloads(struct rte_eth_dev *dev)
3003 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3005 offloads = DEV_RX_OFFLOAD_IPV4_CKSUM |
3006 DEV_RX_OFFLOAD_UDP_CKSUM |
3007 DEV_RX_OFFLOAD_TCP_CKSUM |
3008 DEV_RX_OFFLOAD_KEEP_CRC |
3009 DEV_RX_OFFLOAD_JUMBO_FRAME |
3010 DEV_RX_OFFLOAD_VLAN_FILTER |
3011 DEV_RX_OFFLOAD_SCATTER |
3012 DEV_RX_OFFLOAD_RSS_HASH;
3014 if (hw->mac.type == ixgbe_mac_82598EB)
3015 offloads |= DEV_RX_OFFLOAD_VLAN_STRIP;
3017 if (ixgbe_is_vf(dev) == 0)
3018 offloads |= DEV_RX_OFFLOAD_VLAN_EXTEND;
3021 * RSC is only supported by 82599 and x540 PF devices in a non-SR-IOV
3024 if ((hw->mac.type == ixgbe_mac_82599EB ||
3025 hw->mac.type == ixgbe_mac_X540 ||
3026 hw->mac.type == ixgbe_mac_X550) &&
3027 !RTE_ETH_DEV_SRIOV(dev).active)
3028 offloads |= DEV_RX_OFFLOAD_TCP_LRO;
3030 if (hw->mac.type == ixgbe_mac_82599EB ||
3031 hw->mac.type == ixgbe_mac_X540)
3032 offloads |= DEV_RX_OFFLOAD_MACSEC_STRIP;
3034 if (hw->mac.type == ixgbe_mac_X550 ||
3035 hw->mac.type == ixgbe_mac_X550EM_x ||
3036 hw->mac.type == ixgbe_mac_X550EM_a)
3037 offloads |= DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM;
3039 #ifdef RTE_LIB_SECURITY
3040 if (dev->security_ctx)
3041 offloads |= DEV_RX_OFFLOAD_SECURITY;
3048 ixgbe_dev_rx_queue_setup(struct rte_eth_dev *dev,
3051 unsigned int socket_id,
3052 const struct rte_eth_rxconf *rx_conf,
3053 struct rte_mempool *mp)
3055 const struct rte_memzone *rz;
3056 struct ixgbe_rx_queue *rxq;
3057 struct ixgbe_hw *hw;
3059 struct ixgbe_adapter *adapter = dev->data->dev_private;
3062 PMD_INIT_FUNC_TRACE();
3063 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3065 offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads;
3068 * Validate number of receive descriptors.
3069 * It must not exceed hardware maximum, and must be multiple
3072 if (nb_desc % IXGBE_RXD_ALIGN != 0 ||
3073 (nb_desc > IXGBE_MAX_RING_DESC) ||
3074 (nb_desc < IXGBE_MIN_RING_DESC)) {
3078 /* Free memory prior to re-allocation if needed... */
3079 if (dev->data->rx_queues[queue_idx] != NULL) {
3080 ixgbe_rx_queue_release(dev->data->rx_queues[queue_idx]);
3081 dev->data->rx_queues[queue_idx] = NULL;
3084 /* First allocate the rx queue data structure */
3085 rxq = rte_zmalloc_socket("ethdev RX queue", sizeof(struct ixgbe_rx_queue),
3086 RTE_CACHE_LINE_SIZE, socket_id);
3090 rxq->nb_rx_desc = nb_desc;
3091 rxq->rx_free_thresh = rx_conf->rx_free_thresh;
3092 rxq->queue_id = queue_idx;
3093 rxq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ?
3094 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx);
3095 rxq->port_id = dev->data->port_id;
3096 if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC)
3097 rxq->crc_len = RTE_ETHER_CRC_LEN;
3100 rxq->drop_en = rx_conf->rx_drop_en;
3101 rxq->rx_deferred_start = rx_conf->rx_deferred_start;
3102 rxq->offloads = offloads;
3105 * The packet type in RX descriptor is different for different NICs.
3106 * Some bits are used for x550 but reserved for other NICS.
3107 * So set different masks for different NICs.
3109 if (hw->mac.type == ixgbe_mac_X550 ||
3110 hw->mac.type == ixgbe_mac_X550EM_x ||
3111 hw->mac.type == ixgbe_mac_X550EM_a ||
3112 hw->mac.type == ixgbe_mac_X550_vf ||
3113 hw->mac.type == ixgbe_mac_X550EM_x_vf ||
3114 hw->mac.type == ixgbe_mac_X550EM_a_vf)
3115 rxq->pkt_type_mask = IXGBE_PACKET_TYPE_MASK_X550;
3117 rxq->pkt_type_mask = IXGBE_PACKET_TYPE_MASK_82599;
3120 * Allocate RX ring hardware descriptors. A memzone large enough to
3121 * handle the maximum ring size is allocated in order to allow for
3122 * resizing in later calls to the queue setup function.
3124 rz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx,
3125 RX_RING_SZ, IXGBE_ALIGN, socket_id);
3127 ixgbe_rx_queue_release(rxq);
3132 * Zero init all the descriptors in the ring.
3134 memset(rz->addr, 0, RX_RING_SZ);
3137 * Modified to setup VFRDT for Virtual Function
3139 if (hw->mac.type == ixgbe_mac_82599_vf ||
3140 hw->mac.type == ixgbe_mac_X540_vf ||
3141 hw->mac.type == ixgbe_mac_X550_vf ||
3142 hw->mac.type == ixgbe_mac_X550EM_x_vf ||
3143 hw->mac.type == ixgbe_mac_X550EM_a_vf) {
3145 IXGBE_PCI_REG_ADDR(hw, IXGBE_VFRDT(queue_idx));
3147 IXGBE_PCI_REG_ADDR(hw, IXGBE_VFRDH(queue_idx));
3150 IXGBE_PCI_REG_ADDR(hw, IXGBE_RDT(rxq->reg_idx));
3152 IXGBE_PCI_REG_ADDR(hw, IXGBE_RDH(rxq->reg_idx));
3155 rxq->rx_ring_phys_addr = rz->iova;
3156 rxq->rx_ring = (union ixgbe_adv_rx_desc *) rz->addr;
3159 * Certain constraints must be met in order to use the bulk buffer
3160 * allocation Rx burst function. If any of Rx queues doesn't meet them
3161 * the feature should be disabled for the whole port.
3163 if (check_rx_burst_bulk_alloc_preconditions(rxq)) {
3164 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Rx Bulk Alloc "
3165 "preconditions - canceling the feature for "
3166 "the whole port[%d]",
3167 rxq->queue_id, rxq->port_id);
3168 adapter->rx_bulk_alloc_allowed = false;
3172 * Allocate software ring. Allow for space at the end of the
3173 * S/W ring to make sure look-ahead logic in bulk alloc Rx burst
3174 * function does not access an invalid memory region.
3177 if (adapter->rx_bulk_alloc_allowed)
3178 len += RTE_PMD_IXGBE_RX_MAX_BURST;
3180 rxq->sw_ring = rte_zmalloc_socket("rxq->sw_ring",
3181 sizeof(struct ixgbe_rx_entry) * len,
3182 RTE_CACHE_LINE_SIZE, socket_id);
3183 if (!rxq->sw_ring) {
3184 ixgbe_rx_queue_release(rxq);
3189 * Always allocate even if it's not going to be needed in order to
3190 * simplify the code.
3192 * This ring is used in LRO and Scattered Rx cases and Scattered Rx may
3193 * be requested in ixgbe_dev_rx_init(), which is called later from
3197 rte_zmalloc_socket("rxq->sw_sc_ring",
3198 sizeof(struct ixgbe_scattered_rx_entry) * len,
3199 RTE_CACHE_LINE_SIZE, socket_id);
3200 if (!rxq->sw_sc_ring) {
3201 ixgbe_rx_queue_release(rxq);
3205 PMD_INIT_LOG(DEBUG, "sw_ring=%p sw_sc_ring=%p hw_ring=%p "
3206 "dma_addr=0x%"PRIx64,
3207 rxq->sw_ring, rxq->sw_sc_ring, rxq->rx_ring,
3208 rxq->rx_ring_phys_addr);
3210 if (!rte_is_power_of_2(nb_desc)) {
3211 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
3212 "preconditions - canceling the feature for "
3213 "the whole port[%d]",
3214 rxq->queue_id, rxq->port_id);
3215 adapter->rx_vec_allowed = false;
3217 ixgbe_rxq_vec_setup(rxq);
3219 dev->data->rx_queues[queue_idx] = rxq;
3221 ixgbe_reset_rx_queue(adapter, rxq);
3227 ixgbe_dev_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id)
3229 #define IXGBE_RXQ_SCAN_INTERVAL 4
3230 volatile union ixgbe_adv_rx_desc *rxdp;
3231 struct ixgbe_rx_queue *rxq;
3234 rxq = dev->data->rx_queues[rx_queue_id];
3235 rxdp = &(rxq->rx_ring[rxq->rx_tail]);
3237 while ((desc < rxq->nb_rx_desc) &&
3238 (rxdp->wb.upper.status_error &
3239 rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD))) {
3240 desc += IXGBE_RXQ_SCAN_INTERVAL;
3241 rxdp += IXGBE_RXQ_SCAN_INTERVAL;
3242 if (rxq->rx_tail + desc >= rxq->nb_rx_desc)
3243 rxdp = &(rxq->rx_ring[rxq->rx_tail +
3244 desc - rxq->nb_rx_desc]);
3251 ixgbe_dev_rx_descriptor_done(void *rx_queue, uint16_t offset)
3253 volatile union ixgbe_adv_rx_desc *rxdp;
3254 struct ixgbe_rx_queue *rxq = rx_queue;
3257 if (unlikely(offset >= rxq->nb_rx_desc))
3259 desc = rxq->rx_tail + offset;
3260 if (desc >= rxq->nb_rx_desc)
3261 desc -= rxq->nb_rx_desc;
3263 rxdp = &rxq->rx_ring[desc];
3264 return !!(rxdp->wb.upper.status_error &
3265 rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD));
3269 ixgbe_dev_rx_descriptor_status(void *rx_queue, uint16_t offset)
3271 struct ixgbe_rx_queue *rxq = rx_queue;
3272 volatile uint32_t *status;
3273 uint32_t nb_hold, desc;
3275 if (unlikely(offset >= rxq->nb_rx_desc))
3278 #if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64)
3279 if (rxq->rx_using_sse)
3280 nb_hold = rxq->rxrearm_nb;
3283 nb_hold = rxq->nb_rx_hold;
3284 if (offset >= rxq->nb_rx_desc - nb_hold)
3285 return RTE_ETH_RX_DESC_UNAVAIL;
3287 desc = rxq->rx_tail + offset;
3288 if (desc >= rxq->nb_rx_desc)
3289 desc -= rxq->nb_rx_desc;
3291 status = &rxq->rx_ring[desc].wb.upper.status_error;
3292 if (*status & rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD))
3293 return RTE_ETH_RX_DESC_DONE;
3295 return RTE_ETH_RX_DESC_AVAIL;
3299 ixgbe_dev_tx_descriptor_status(void *tx_queue, uint16_t offset)
3301 struct ixgbe_tx_queue *txq = tx_queue;
3302 volatile uint32_t *status;
3305 if (unlikely(offset >= txq->nb_tx_desc))
3308 desc = txq->tx_tail + offset;
3309 /* go to next desc that has the RS bit */
3310 desc = ((desc + txq->tx_rs_thresh - 1) / txq->tx_rs_thresh) *
3312 if (desc >= txq->nb_tx_desc) {
3313 desc -= txq->nb_tx_desc;
3314 if (desc >= txq->nb_tx_desc)
3315 desc -= txq->nb_tx_desc;
3318 status = &txq->tx_ring[desc].wb.status;
3319 if (*status & rte_cpu_to_le_32(IXGBE_ADVTXD_STAT_DD))
3320 return RTE_ETH_TX_DESC_DONE;
3322 return RTE_ETH_TX_DESC_FULL;
3326 * Set up link loopback for X540/X550 mode Tx->Rx.
3328 static inline void __rte_cold
3329 ixgbe_setup_loopback_link_x540_x550(struct ixgbe_hw *hw, bool enable)
3332 PMD_INIT_FUNC_TRACE();
3334 u16 autoneg_reg = IXGBE_MII_AUTONEG_REG;
3336 hw->phy.ops.read_reg(hw, IXGBE_MDIO_AUTO_NEG_CONTROL,
3337 IXGBE_MDIO_AUTO_NEG_DEV_TYPE, &autoneg_reg);
3338 macc = IXGBE_READ_REG(hw, IXGBE_MACC);
3341 /* datasheet 15.2.1: disable AUTONEG (PHY Bit 7.0.C) */
3342 autoneg_reg |= IXGBE_MII_AUTONEG_ENABLE;
3343 /* datasheet 15.2.1: MACC.FLU = 1 (force link up) */
3344 macc |= IXGBE_MACC_FLU;
3346 autoneg_reg &= ~IXGBE_MII_AUTONEG_ENABLE;
3347 macc &= ~IXGBE_MACC_FLU;
3350 hw->phy.ops.write_reg(hw, IXGBE_MDIO_AUTO_NEG_CONTROL,
3351 IXGBE_MDIO_AUTO_NEG_DEV_TYPE, autoneg_reg);
3353 IXGBE_WRITE_REG(hw, IXGBE_MACC, macc);
3357 ixgbe_dev_clear_queues(struct rte_eth_dev *dev)
3360 struct ixgbe_adapter *adapter = dev->data->dev_private;
3361 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3363 PMD_INIT_FUNC_TRACE();
3365 for (i = 0; i < dev->data->nb_tx_queues; i++) {
3366 struct ixgbe_tx_queue *txq = dev->data->tx_queues[i];
3369 txq->ops->release_mbufs(txq);
3370 txq->ops->reset(txq);
3374 for (i = 0; i < dev->data->nb_rx_queues; i++) {
3375 struct ixgbe_rx_queue *rxq = dev->data->rx_queues[i];
3378 ixgbe_rx_queue_release_mbufs(rxq);
3379 ixgbe_reset_rx_queue(adapter, rxq);
3382 /* If loopback mode was enabled, reconfigure the link accordingly */
3383 if (dev->data->dev_conf.lpbk_mode != 0) {
3384 if (hw->mac.type == ixgbe_mac_X540 ||
3385 hw->mac.type == ixgbe_mac_X550 ||
3386 hw->mac.type == ixgbe_mac_X550EM_x ||
3387 hw->mac.type == ixgbe_mac_X550EM_a)
3388 ixgbe_setup_loopback_link_x540_x550(hw, false);
3393 ixgbe_dev_free_queues(struct rte_eth_dev *dev)
3397 PMD_INIT_FUNC_TRACE();
3399 for (i = 0; i < dev->data->nb_rx_queues; i++) {
3400 ixgbe_dev_rx_queue_release(dev->data->rx_queues[i]);
3401 dev->data->rx_queues[i] = NULL;
3402 rte_eth_dma_zone_free(dev, "rx_ring", i);
3404 dev->data->nb_rx_queues = 0;
3406 for (i = 0; i < dev->data->nb_tx_queues; i++) {
3407 ixgbe_dev_tx_queue_release(dev->data->tx_queues[i]);
3408 dev->data->tx_queues[i] = NULL;
3409 rte_eth_dma_zone_free(dev, "tx_ring", i);
3411 dev->data->nb_tx_queues = 0;
3414 /*********************************************************************
3416 * Device RX/TX init functions
3418 **********************************************************************/
3421 * Receive Side Scaling (RSS)
3422 * See section 7.1.2.8 in the following document:
3423 * "Intel 82599 10 GbE Controller Datasheet" - Revision 2.1 October 2009
3426 * The source and destination IP addresses of the IP header and the source
3427 * and destination ports of TCP/UDP headers, if any, of received packets are
3428 * hashed against a configurable random key to compute a 32-bit RSS hash result.
3429 * The seven (7) LSBs of the 32-bit hash result are used as an index into a
3430 * 128-entry redirection table (RETA). Each entry of the RETA provides a 3-bit
3431 * RSS output index which is used as the RX queue index where to store the
3433 * The following output is supplied in the RX write-back descriptor:
3434 * - 32-bit result of the Microsoft RSS hash function,
3435 * - 4-bit RSS type field.
3439 * RSS random key supplied in section 7.1.2.8.3 of the Intel 82599 datasheet.
3440 * Used as the default key.
3442 static uint8_t rss_intel_key[40] = {
3443 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
3444 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
3445 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
3446 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
3447 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
3451 ixgbe_rss_disable(struct rte_eth_dev *dev)
3453 struct ixgbe_hw *hw;
3457 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3458 mrqc_reg = ixgbe_mrqc_reg_get(hw->mac.type);
3459 mrqc = IXGBE_READ_REG(hw, mrqc_reg);
3460 mrqc &= ~IXGBE_MRQC_RSSEN;
3461 IXGBE_WRITE_REG(hw, mrqc_reg, mrqc);
3465 ixgbe_hw_rss_hash_set(struct ixgbe_hw *hw, struct rte_eth_rss_conf *rss_conf)
3475 mrqc_reg = ixgbe_mrqc_reg_get(hw->mac.type);
3476 rssrk_reg = ixgbe_rssrk_reg_get(hw->mac.type, 0);
3478 hash_key = rss_conf->rss_key;
3479 if (hash_key != NULL) {
3480 /* Fill in RSS hash key */
3481 for (i = 0; i < 10; i++) {
3482 rss_key = hash_key[(i * 4)];
3483 rss_key |= hash_key[(i * 4) + 1] << 8;
3484 rss_key |= hash_key[(i * 4) + 2] << 16;
3485 rss_key |= hash_key[(i * 4) + 3] << 24;
3486 IXGBE_WRITE_REG_ARRAY(hw, rssrk_reg, i, rss_key);
3490 /* Set configured hashing protocols in MRQC register */
3491 rss_hf = rss_conf->rss_hf;
3492 mrqc = IXGBE_MRQC_RSSEN; /* Enable RSS */
3493 if (rss_hf & ETH_RSS_IPV4)
3494 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4;
3495 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
3496 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP;
3497 if (rss_hf & ETH_RSS_IPV6)
3498 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6;
3499 if (rss_hf & ETH_RSS_IPV6_EX)
3500 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX;
3501 if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP)
3502 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP;
3503 if (rss_hf & ETH_RSS_IPV6_TCP_EX)
3504 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP;
3505 if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP)
3506 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP;
3507 if (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP)
3508 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP;
3509 if (rss_hf & ETH_RSS_IPV6_UDP_EX)
3510 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
3511 IXGBE_WRITE_REG(hw, mrqc_reg, mrqc);
3515 ixgbe_dev_rss_hash_update(struct rte_eth_dev *dev,
3516 struct rte_eth_rss_conf *rss_conf)
3518 struct ixgbe_hw *hw;
3523 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3525 if (!ixgbe_rss_update_sp(hw->mac.type)) {
3526 PMD_DRV_LOG(ERR, "RSS hash update is not supported on this "
3530 mrqc_reg = ixgbe_mrqc_reg_get(hw->mac.type);
3533 * Excerpt from section 7.1.2.8 Receive-Side Scaling (RSS):
3534 * "RSS enabling cannot be done dynamically while it must be
3535 * preceded by a software reset"
3536 * Before changing anything, first check that the update RSS operation
3537 * does not attempt to disable RSS, if RSS was enabled at
3538 * initialization time, or does not attempt to enable RSS, if RSS was
3539 * disabled at initialization time.
3541 rss_hf = rss_conf->rss_hf & IXGBE_RSS_OFFLOAD_ALL;
3542 mrqc = IXGBE_READ_REG(hw, mrqc_reg);
3543 if (!(mrqc & IXGBE_MRQC_RSSEN)) { /* RSS disabled */
3544 if (rss_hf != 0) /* Enable RSS */
3546 return 0; /* Nothing to do */
3549 if (rss_hf == 0) /* Disable RSS */
3551 ixgbe_hw_rss_hash_set(hw, rss_conf);
3556 ixgbe_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
3557 struct rte_eth_rss_conf *rss_conf)
3559 struct ixgbe_hw *hw;
3568 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3569 mrqc_reg = ixgbe_mrqc_reg_get(hw->mac.type);
3570 rssrk_reg = ixgbe_rssrk_reg_get(hw->mac.type, 0);
3571 hash_key = rss_conf->rss_key;
3572 if (hash_key != NULL) {
3573 /* Return RSS hash key */
3574 for (i = 0; i < 10; i++) {
3575 rss_key = IXGBE_READ_REG_ARRAY(hw, rssrk_reg, i);
3576 hash_key[(i * 4)] = rss_key & 0x000000FF;
3577 hash_key[(i * 4) + 1] = (rss_key >> 8) & 0x000000FF;
3578 hash_key[(i * 4) + 2] = (rss_key >> 16) & 0x000000FF;
3579 hash_key[(i * 4) + 3] = (rss_key >> 24) & 0x000000FF;
3583 /* Get RSS functions configured in MRQC register */
3584 mrqc = IXGBE_READ_REG(hw, mrqc_reg);
3585 if ((mrqc & IXGBE_MRQC_RSSEN) == 0) { /* RSS is disabled */
3586 rss_conf->rss_hf = 0;
3590 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV4)
3591 rss_hf |= ETH_RSS_IPV4;
3592 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV4_TCP)
3593 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP;
3594 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6)
3595 rss_hf |= ETH_RSS_IPV6;
3596 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_EX)
3597 rss_hf |= ETH_RSS_IPV6_EX;
3598 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_TCP)
3599 rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP;
3600 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP)
3601 rss_hf |= ETH_RSS_IPV6_TCP_EX;
3602 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV4_UDP)
3603 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP;
3604 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_UDP)
3605 rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP;
3606 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP)
3607 rss_hf |= ETH_RSS_IPV6_UDP_EX;
3608 rss_conf->rss_hf = rss_hf;
3613 ixgbe_rss_configure(struct rte_eth_dev *dev)
3615 struct rte_eth_rss_conf rss_conf;
3616 struct ixgbe_adapter *adapter;
3617 struct ixgbe_hw *hw;
3621 uint16_t sp_reta_size;
3624 PMD_INIT_FUNC_TRACE();
3625 adapter = dev->data->dev_private;
3626 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3628 sp_reta_size = ixgbe_reta_size_get(hw->mac.type);
3631 * Fill in redirection table
3632 * The byte-swap is needed because NIC registers are in
3633 * little-endian order.
3635 if (adapter->rss_reta_updated == 0) {
3637 for (i = 0, j = 0; i < sp_reta_size; i++, j++) {
3638 reta_reg = ixgbe_reta_reg_get(hw->mac.type, i);
3640 if (j == dev->data->nb_rx_queues)
3642 reta = (reta << 8) | j;
3644 IXGBE_WRITE_REG(hw, reta_reg,
3650 * Configure the RSS key and the RSS protocols used to compute
3651 * the RSS hash of input packets.
3653 rss_conf = dev->data->dev_conf.rx_adv_conf.rss_conf;
3654 if ((rss_conf.rss_hf & IXGBE_RSS_OFFLOAD_ALL) == 0) {
3655 ixgbe_rss_disable(dev);
3658 if (rss_conf.rss_key == NULL)
3659 rss_conf.rss_key = rss_intel_key; /* Default hash key */
3660 ixgbe_hw_rss_hash_set(hw, &rss_conf);
3663 #define NUM_VFTA_REGISTERS 128
3664 #define NIC_RX_BUFFER_SIZE 0x200
3665 #define X550_RX_BUFFER_SIZE 0x180
3668 ixgbe_vmdq_dcb_configure(struct rte_eth_dev *dev)
3670 struct rte_eth_vmdq_dcb_conf *cfg;
3671 struct ixgbe_hw *hw;
3672 enum rte_eth_nb_pools num_pools;
3673 uint32_t mrqc, vt_ctl, queue_mapping, vlanctrl;
3675 uint8_t nb_tcs; /* number of traffic classes */
3678 PMD_INIT_FUNC_TRACE();
3679 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3680 cfg = &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf;
3681 num_pools = cfg->nb_queue_pools;
3682 /* Check we have a valid number of pools */
3683 if (num_pools != ETH_16_POOLS && num_pools != ETH_32_POOLS) {
3684 ixgbe_rss_disable(dev);
3687 /* 16 pools -> 8 traffic classes, 32 pools -> 4 traffic classes */
3688 nb_tcs = (uint8_t)(ETH_VMDQ_DCB_NUM_QUEUES / (int)num_pools);
3692 * split rx buffer up into sections, each for 1 traffic class
3694 switch (hw->mac.type) {
3695 case ixgbe_mac_X550:
3696 case ixgbe_mac_X550EM_x:
3697 case ixgbe_mac_X550EM_a:
3698 pbsize = (uint16_t)(X550_RX_BUFFER_SIZE / nb_tcs);
3701 pbsize = (uint16_t)(NIC_RX_BUFFER_SIZE / nb_tcs);
3704 for (i = 0; i < nb_tcs; i++) {
3705 uint32_t rxpbsize = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(i));
3707 rxpbsize &= (~(0x3FF << IXGBE_RXPBSIZE_SHIFT));
3708 /* clear 10 bits. */
3709 rxpbsize |= (pbsize << IXGBE_RXPBSIZE_SHIFT); /* set value */
3710 IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), rxpbsize);
3712 /* zero alloc all unused TCs */
3713 for (i = nb_tcs; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3714 uint32_t rxpbsize = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(i));
3716 rxpbsize &= (~(0x3FF << IXGBE_RXPBSIZE_SHIFT));
3717 /* clear 10 bits. */
3718 IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), rxpbsize);
3721 /* MRQC: enable vmdq and dcb */
3722 mrqc = (num_pools == ETH_16_POOLS) ?
3723 IXGBE_MRQC_VMDQRT8TCEN : IXGBE_MRQC_VMDQRT4TCEN;
3724 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
3726 /* PFVTCTL: turn on virtualisation and set the default pool */
3727 vt_ctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
3728 if (cfg->enable_default_pool) {
3729 vt_ctl |= (cfg->default_pool << IXGBE_VT_CTL_POOL_SHIFT);
3731 vt_ctl |= IXGBE_VT_CTL_DIS_DEFPL;
3734 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vt_ctl);
3736 /* RTRUP2TC: mapping user priorities to traffic classes (TCs) */
3738 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++)
3740 * mapping is done with 3 bits per priority,
3741 * so shift by i*3 each time
3743 queue_mapping |= ((cfg->dcb_tc[i] & 0x07) << (i * 3));
3745 IXGBE_WRITE_REG(hw, IXGBE_RTRUP2TC, queue_mapping);
3747 /* RTRPCS: DCB related */
3748 IXGBE_WRITE_REG(hw, IXGBE_RTRPCS, IXGBE_RMCS_RRM);
3750 /* VLNCTRL: enable vlan filtering and allow all vlan tags through */
3751 vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
3752 vlanctrl |= IXGBE_VLNCTRL_VFE; /* enable vlan filters */
3753 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl);
3755 /* VFTA - enable all vlan filters */
3756 for (i = 0; i < NUM_VFTA_REGISTERS; i++) {
3757 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), 0xFFFFFFFF);
3760 /* VFRE: pool enabling for receive - 16 or 32 */
3761 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0),
3762 num_pools == ETH_16_POOLS ? 0xFFFF : 0xFFFFFFFF);
3765 * MPSAR - allow pools to read specific mac addresses
3766 * In this case, all pools should be able to read from mac addr 0
3768 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(0), 0xFFFFFFFF);
3769 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(0), 0xFFFFFFFF);
3771 /* PFVLVF, PFVLVFB: set up filters for vlan tags as configured */
3772 for (i = 0; i < cfg->nb_pool_maps; i++) {
3773 /* set vlan id in VF register and set the valid bit */
3774 IXGBE_WRITE_REG(hw, IXGBE_VLVF(i), (IXGBE_VLVF_VIEN |
3775 (cfg->pool_map[i].vlan_id & 0xFFF)));
3777 * Put the allowed pools in VFB reg. As we only have 16 or 32
3778 * pools, we only need to use the first half of the register
3781 IXGBE_WRITE_REG(hw, IXGBE_VLVFB(i*2), cfg->pool_map[i].pools);
3786 * ixgbe_dcb_config_tx_hw_config - Configure general DCB TX parameters
3787 * @dev: pointer to eth_dev structure
3788 * @dcb_config: pointer to ixgbe_dcb_config structure
3791 ixgbe_dcb_tx_hw_config(struct rte_eth_dev *dev,
3792 struct ixgbe_dcb_config *dcb_config)
3795 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3797 PMD_INIT_FUNC_TRACE();
3798 if (hw->mac.type != ixgbe_mac_82598EB) {
3799 /* Disable the Tx desc arbiter so that MTQC can be changed */
3800 reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3801 reg |= IXGBE_RTTDCS_ARBDIS;
3802 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg);
3804 /* Enable DCB for Tx with 8 TCs */
3805 if (dcb_config->num_tcs.pg_tcs == 8) {
3806 reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;
3808 reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
3810 if (dcb_config->vt_mode)
3811 reg |= IXGBE_MTQC_VT_ENA;
3812 IXGBE_WRITE_REG(hw, IXGBE_MTQC, reg);
3814 /* Enable the Tx desc arbiter */
3815 reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3816 reg &= ~IXGBE_RTTDCS_ARBDIS;
3817 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg);
3819 /* Enable Security TX Buffer IFG for DCB */
3820 reg = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG);
3821 reg |= IXGBE_SECTX_DCB;
3822 IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, reg);
3827 * ixgbe_vmdq_dcb_hw_tx_config - Configure general VMDQ+DCB TX parameters
3828 * @dev: pointer to rte_eth_dev structure
3829 * @dcb_config: pointer to ixgbe_dcb_config structure
3832 ixgbe_vmdq_dcb_hw_tx_config(struct rte_eth_dev *dev,
3833 struct ixgbe_dcb_config *dcb_config)
3835 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3836 &dev->data->dev_conf.tx_adv_conf.vmdq_dcb_tx_conf;
3837 struct ixgbe_hw *hw =
3838 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3840 PMD_INIT_FUNC_TRACE();
3841 if (hw->mac.type != ixgbe_mac_82598EB)
3842 /*PF VF Transmit Enable*/
3843 IXGBE_WRITE_REG(hw, IXGBE_VFTE(0),
3844 vmdq_tx_conf->nb_queue_pools == ETH_16_POOLS ? 0xFFFF : 0xFFFFFFFF);
3846 /*Configure general DCB TX parameters*/
3847 ixgbe_dcb_tx_hw_config(dev, dcb_config);
3851 ixgbe_vmdq_dcb_rx_config(struct rte_eth_dev *dev,
3852 struct ixgbe_dcb_config *dcb_config)
3854 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3855 &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf;
3856 struct ixgbe_dcb_tc_config *tc;
3859 /* convert rte_eth_conf.rx_adv_conf to struct ixgbe_dcb_config */
3860 if (vmdq_rx_conf->nb_queue_pools == ETH_16_POOLS) {
3861 dcb_config->num_tcs.pg_tcs = ETH_8_TCS;
3862 dcb_config->num_tcs.pfc_tcs = ETH_8_TCS;
3864 dcb_config->num_tcs.pg_tcs = ETH_4_TCS;
3865 dcb_config->num_tcs.pfc_tcs = ETH_4_TCS;
3868 /* Initialize User Priority to Traffic Class mapping */
3869 for (j = 0; j < IXGBE_DCB_MAX_TRAFFIC_CLASS; j++) {
3870 tc = &dcb_config->tc_config[j];
3871 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0;
3874 /* User Priority to Traffic Class mapping */
3875 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3876 j = vmdq_rx_conf->dcb_tc[i];
3877 tc = &dcb_config->tc_config[j];
3878 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap |=
3884 ixgbe_dcb_vt_tx_config(struct rte_eth_dev *dev,
3885 struct ixgbe_dcb_config *dcb_config)
3887 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3888 &dev->data->dev_conf.tx_adv_conf.vmdq_dcb_tx_conf;
3889 struct ixgbe_dcb_tc_config *tc;
3892 /* convert rte_eth_conf.rx_adv_conf to struct ixgbe_dcb_config */
3893 if (vmdq_tx_conf->nb_queue_pools == ETH_16_POOLS) {
3894 dcb_config->num_tcs.pg_tcs = ETH_8_TCS;
3895 dcb_config->num_tcs.pfc_tcs = ETH_8_TCS;
3897 dcb_config->num_tcs.pg_tcs = ETH_4_TCS;
3898 dcb_config->num_tcs.pfc_tcs = ETH_4_TCS;
3901 /* Initialize User Priority to Traffic Class mapping */
3902 for (j = 0; j < IXGBE_DCB_MAX_TRAFFIC_CLASS; j++) {
3903 tc = &dcb_config->tc_config[j];
3904 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0;
3907 /* User Priority to Traffic Class mapping */
3908 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3909 j = vmdq_tx_conf->dcb_tc[i];
3910 tc = &dcb_config->tc_config[j];
3911 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap |=
3917 ixgbe_dcb_rx_config(struct rte_eth_dev *dev,
3918 struct ixgbe_dcb_config *dcb_config)
3920 struct rte_eth_dcb_rx_conf *rx_conf =
3921 &dev->data->dev_conf.rx_adv_conf.dcb_rx_conf;
3922 struct ixgbe_dcb_tc_config *tc;
3925 dcb_config->num_tcs.pg_tcs = (uint8_t)rx_conf->nb_tcs;
3926 dcb_config->num_tcs.pfc_tcs = (uint8_t)rx_conf->nb_tcs;
3928 /* Initialize User Priority to Traffic Class mapping */
3929 for (j = 0; j < IXGBE_DCB_MAX_TRAFFIC_CLASS; j++) {
3930 tc = &dcb_config->tc_config[j];
3931 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0;
3934 /* User Priority to Traffic Class mapping */
3935 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3936 j = rx_conf->dcb_tc[i];
3937 tc = &dcb_config->tc_config[j];
3938 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap |=
3944 ixgbe_dcb_tx_config(struct rte_eth_dev *dev,
3945 struct ixgbe_dcb_config *dcb_config)
3947 struct rte_eth_dcb_tx_conf *tx_conf =
3948 &dev->data->dev_conf.tx_adv_conf.dcb_tx_conf;
3949 struct ixgbe_dcb_tc_config *tc;
3952 dcb_config->num_tcs.pg_tcs = (uint8_t)tx_conf->nb_tcs;
3953 dcb_config->num_tcs.pfc_tcs = (uint8_t)tx_conf->nb_tcs;
3955 /* Initialize User Priority to Traffic Class mapping */
3956 for (j = 0; j < IXGBE_DCB_MAX_TRAFFIC_CLASS; j++) {
3957 tc = &dcb_config->tc_config[j];
3958 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0;
3961 /* User Priority to Traffic Class mapping */
3962 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3963 j = tx_conf->dcb_tc[i];
3964 tc = &dcb_config->tc_config[j];
3965 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap |=
3971 * ixgbe_dcb_rx_hw_config - Configure general DCB RX HW parameters
3972 * @dev: pointer to eth_dev structure
3973 * @dcb_config: pointer to ixgbe_dcb_config structure
3976 ixgbe_dcb_rx_hw_config(struct rte_eth_dev *dev,
3977 struct ixgbe_dcb_config *dcb_config)
3983 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3985 PMD_INIT_FUNC_TRACE();
3987 * Disable the arbiter before changing parameters
3988 * (always enable recycle mode; WSP)
3990 reg = IXGBE_RTRPCS_RRM | IXGBE_RTRPCS_RAC | IXGBE_RTRPCS_ARBDIS;
3991 IXGBE_WRITE_REG(hw, IXGBE_RTRPCS, reg);
3993 if (hw->mac.type != ixgbe_mac_82598EB) {
3994 reg = IXGBE_READ_REG(hw, IXGBE_MRQC);
3995 if (dcb_config->num_tcs.pg_tcs == 4) {
3996 if (dcb_config->vt_mode)
3997 reg = (reg & ~IXGBE_MRQC_MRQE_MASK) |
3998 IXGBE_MRQC_VMDQRT4TCEN;
4000 /* no matter the mode is DCB or DCB_RSS, just
4001 * set the MRQE to RSSXTCEN. RSS is controlled
4004 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, 0);
4005 reg = (reg & ~IXGBE_MRQC_MRQE_MASK) |
4006 IXGBE_MRQC_RTRSS4TCEN;
4009 if (dcb_config->num_tcs.pg_tcs == 8) {
4010 if (dcb_config->vt_mode)
4011 reg = (reg & ~IXGBE_MRQC_MRQE_MASK) |
4012 IXGBE_MRQC_VMDQRT8TCEN;
4014 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, 0);
4015 reg = (reg & ~IXGBE_MRQC_MRQE_MASK) |
4016 IXGBE_MRQC_RTRSS8TCEN;
4020 IXGBE_WRITE_REG(hw, IXGBE_MRQC, reg);
4022 if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
4023 /* Disable drop for all queues in VMDQ mode*/
4024 for (q = 0; q < IXGBE_MAX_RX_QUEUE_NUM; q++)
4025 IXGBE_WRITE_REG(hw, IXGBE_QDE,
4027 (q << IXGBE_QDE_IDX_SHIFT)));
4029 /* Enable drop for all queues in SRIOV mode */
4030 for (q = 0; q < IXGBE_MAX_RX_QUEUE_NUM; q++)
4031 IXGBE_WRITE_REG(hw, IXGBE_QDE,
4033 (q << IXGBE_QDE_IDX_SHIFT) |
4038 /* VLNCTRL: enable vlan filtering and allow all vlan tags through */
4039 vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4040 vlanctrl |= IXGBE_VLNCTRL_VFE; /* enable vlan filters */
4041 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl);
4043 /* VFTA - enable all vlan filters */
4044 for (i = 0; i < NUM_VFTA_REGISTERS; i++) {
4045 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), 0xFFFFFFFF);
4049 * Configure Rx packet plane (recycle mode; WSP) and
4052 reg = IXGBE_RTRPCS_RRM | IXGBE_RTRPCS_RAC;
4053 IXGBE_WRITE_REG(hw, IXGBE_RTRPCS, reg);
4057 ixgbe_dcb_hw_arbite_rx_config(struct ixgbe_hw *hw, uint16_t *refill,
4058 uint16_t *max, uint8_t *bwg_id, uint8_t *tsa, uint8_t *map)
4060 switch (hw->mac.type) {
4061 case ixgbe_mac_82598EB:
4062 ixgbe_dcb_config_rx_arbiter_82598(hw, refill, max, tsa);
4064 case ixgbe_mac_82599EB:
4065 case ixgbe_mac_X540:
4066 case ixgbe_mac_X550:
4067 case ixgbe_mac_X550EM_x:
4068 case ixgbe_mac_X550EM_a:
4069 ixgbe_dcb_config_rx_arbiter_82599(hw, refill, max, bwg_id,
4078 ixgbe_dcb_hw_arbite_tx_config(struct ixgbe_hw *hw, uint16_t *refill, uint16_t *max,
4079 uint8_t *bwg_id, uint8_t *tsa, uint8_t *map)
4081 switch (hw->mac.type) {
4082 case ixgbe_mac_82598EB:
4083 ixgbe_dcb_config_tx_desc_arbiter_82598(hw, refill, max, bwg_id, tsa);
4084 ixgbe_dcb_config_tx_data_arbiter_82598(hw, refill, max, bwg_id, tsa);
4086 case ixgbe_mac_82599EB:
4087 case ixgbe_mac_X540:
4088 case ixgbe_mac_X550:
4089 case ixgbe_mac_X550EM_x:
4090 case ixgbe_mac_X550EM_a:
4091 ixgbe_dcb_config_tx_desc_arbiter_82599(hw, refill, max, bwg_id, tsa);
4092 ixgbe_dcb_config_tx_data_arbiter_82599(hw, refill, max, bwg_id, tsa, map);
4099 #define DCB_RX_CONFIG 1
4100 #define DCB_TX_CONFIG 1
4101 #define DCB_TX_PB 1024
4103 * ixgbe_dcb_hw_configure - Enable DCB and configure
4104 * general DCB in VT mode and non-VT mode parameters
4105 * @dev: pointer to rte_eth_dev structure
4106 * @dcb_config: pointer to ixgbe_dcb_config structure
4109 ixgbe_dcb_hw_configure(struct rte_eth_dev *dev,
4110 struct ixgbe_dcb_config *dcb_config)
4113 uint8_t i, pfc_en, nb_tcs;
4114 uint16_t pbsize, rx_buffer_size;
4115 uint8_t config_dcb_rx = 0;
4116 uint8_t config_dcb_tx = 0;
4117 uint8_t tsa[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0};
4118 uint8_t bwgid[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0};
4119 uint16_t refill[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0};
4120 uint16_t max[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0};
4121 uint8_t map[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0};
4122 struct ixgbe_dcb_tc_config *tc;
4123 uint32_t max_frame = dev->data->mtu + RTE_ETHER_HDR_LEN +
4125 struct ixgbe_hw *hw =
4126 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4127 struct ixgbe_bw_conf *bw_conf =
4128 IXGBE_DEV_PRIVATE_TO_BW_CONF(dev->data->dev_private);
4130 switch (dev->data->dev_conf.rxmode.mq_mode) {
4131 case ETH_MQ_RX_VMDQ_DCB:
4132 dcb_config->vt_mode = true;
4133 if (hw->mac.type != ixgbe_mac_82598EB) {
4134 config_dcb_rx = DCB_RX_CONFIG;
4136 *get dcb and VT rx configuration parameters
4139 ixgbe_vmdq_dcb_rx_config(dev, dcb_config);
4140 /*Configure general VMDQ and DCB RX parameters*/
4141 ixgbe_vmdq_dcb_configure(dev);
4145 case ETH_MQ_RX_DCB_RSS:
4146 dcb_config->vt_mode = false;
4147 config_dcb_rx = DCB_RX_CONFIG;
4148 /* Get dcb TX configuration parameters from rte_eth_conf */
4149 ixgbe_dcb_rx_config(dev, dcb_config);
4150 /*Configure general DCB RX parameters*/
4151 ixgbe_dcb_rx_hw_config(dev, dcb_config);
4154 PMD_INIT_LOG(ERR, "Incorrect DCB RX mode configuration");
4157 switch (dev->data->dev_conf.txmode.mq_mode) {
4158 case ETH_MQ_TX_VMDQ_DCB:
4159 dcb_config->vt_mode = true;
4160 config_dcb_tx = DCB_TX_CONFIG;
4161 /* get DCB and VT TX configuration parameters
4164 ixgbe_dcb_vt_tx_config(dev, dcb_config);
4165 /*Configure general VMDQ and DCB TX parameters*/
4166 ixgbe_vmdq_dcb_hw_tx_config(dev, dcb_config);
4170 dcb_config->vt_mode = false;
4171 config_dcb_tx = DCB_TX_CONFIG;
4172 /*get DCB TX configuration parameters from rte_eth_conf*/
4173 ixgbe_dcb_tx_config(dev, dcb_config);
4174 /*Configure general DCB TX parameters*/
4175 ixgbe_dcb_tx_hw_config(dev, dcb_config);
4178 PMD_INIT_LOG(ERR, "Incorrect DCB TX mode configuration");
4182 nb_tcs = dcb_config->num_tcs.pfc_tcs;
4184 ixgbe_dcb_unpack_map_cee(dcb_config, IXGBE_DCB_RX_CONFIG, map);
4185 if (nb_tcs == ETH_4_TCS) {
4186 /* Avoid un-configured priority mapping to TC0 */
4188 uint8_t mask = 0xFF;
4190 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES - 4; i++)
4191 mask = (uint8_t)(mask & (~(1 << map[i])));
4192 for (i = 0; mask && (i < IXGBE_DCB_MAX_TRAFFIC_CLASS); i++) {
4193 if ((mask & 0x1) && (j < ETH_DCB_NUM_USER_PRIORITIES))
4197 /* Re-configure 4 TCs BW */
4198 for (i = 0; i < nb_tcs; i++) {
4199 tc = &dcb_config->tc_config[i];
4200 if (bw_conf->tc_num != nb_tcs)
4201 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent =
4202 (uint8_t)(100 / nb_tcs);
4203 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent =
4204 (uint8_t)(100 / nb_tcs);
4206 for (; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
4207 tc = &dcb_config->tc_config[i];
4208 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent = 0;
4209 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent = 0;
4212 /* Re-configure 8 TCs BW */
4213 for (i = 0; i < nb_tcs; i++) {
4214 tc = &dcb_config->tc_config[i];
4215 if (bw_conf->tc_num != nb_tcs)
4216 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent =
4217 (uint8_t)(100 / nb_tcs + (i & 1));
4218 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent =
4219 (uint8_t)(100 / nb_tcs + (i & 1));
4223 switch (hw->mac.type) {
4224 case ixgbe_mac_X550:
4225 case ixgbe_mac_X550EM_x:
4226 case ixgbe_mac_X550EM_a:
4227 rx_buffer_size = X550_RX_BUFFER_SIZE;
4230 rx_buffer_size = NIC_RX_BUFFER_SIZE;
4234 if (config_dcb_rx) {
4235 /* Set RX buffer size */
4236 pbsize = (uint16_t)(rx_buffer_size / nb_tcs);
4237 uint32_t rxpbsize = pbsize << IXGBE_RXPBSIZE_SHIFT;
4239 for (i = 0; i < nb_tcs; i++) {
4240 IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), rxpbsize);
4242 /* zero alloc all unused TCs */
4243 for (; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
4244 IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), 0);
4247 if (config_dcb_tx) {
4248 /* Only support an equally distributed
4249 * Tx packet buffer strategy.
4251 uint32_t txpktsize = IXGBE_TXPBSIZE_MAX / nb_tcs;
4252 uint32_t txpbthresh = (txpktsize / DCB_TX_PB) - IXGBE_TXPKT_SIZE_MAX;
4254 for (i = 0; i < nb_tcs; i++) {
4255 IXGBE_WRITE_REG(hw, IXGBE_TXPBSIZE(i), txpktsize);
4256 IXGBE_WRITE_REG(hw, IXGBE_TXPBTHRESH(i), txpbthresh);
4258 /* Clear unused TCs, if any, to zero buffer size*/
4259 for (; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
4260 IXGBE_WRITE_REG(hw, IXGBE_TXPBSIZE(i), 0);
4261 IXGBE_WRITE_REG(hw, IXGBE_TXPBTHRESH(i), 0);
4265 /*Calculates traffic class credits*/
4266 ixgbe_dcb_calculate_tc_credits_cee(hw, dcb_config, max_frame,
4267 IXGBE_DCB_TX_CONFIG);
4268 ixgbe_dcb_calculate_tc_credits_cee(hw, dcb_config, max_frame,
4269 IXGBE_DCB_RX_CONFIG);
4271 if (config_dcb_rx) {
4272 /* Unpack CEE standard containers */
4273 ixgbe_dcb_unpack_refill_cee(dcb_config, IXGBE_DCB_RX_CONFIG, refill);
4274 ixgbe_dcb_unpack_max_cee(dcb_config, max);
4275 ixgbe_dcb_unpack_bwgid_cee(dcb_config, IXGBE_DCB_RX_CONFIG, bwgid);
4276 ixgbe_dcb_unpack_tsa_cee(dcb_config, IXGBE_DCB_RX_CONFIG, tsa);
4277 /* Configure PG(ETS) RX */
4278 ixgbe_dcb_hw_arbite_rx_config(hw, refill, max, bwgid, tsa, map);
4281 if (config_dcb_tx) {
4282 /* Unpack CEE standard containers */
4283 ixgbe_dcb_unpack_refill_cee(dcb_config, IXGBE_DCB_TX_CONFIG, refill);
4284 ixgbe_dcb_unpack_max_cee(dcb_config, max);
4285 ixgbe_dcb_unpack_bwgid_cee(dcb_config, IXGBE_DCB_TX_CONFIG, bwgid);
4286 ixgbe_dcb_unpack_tsa_cee(dcb_config, IXGBE_DCB_TX_CONFIG, tsa);
4287 /* Configure PG(ETS) TX */
4288 ixgbe_dcb_hw_arbite_tx_config(hw, refill, max, bwgid, tsa, map);
4291 /*Configure queue statistics registers*/
4292 ixgbe_dcb_config_tc_stats_82599(hw, dcb_config);
4294 /* Check if the PFC is supported */
4295 if (dev->data->dev_conf.dcb_capability_en & ETH_DCB_PFC_SUPPORT) {
4296 pbsize = (uint16_t)(rx_buffer_size / nb_tcs);
4297 for (i = 0; i < nb_tcs; i++) {
4299 * If the TC count is 8,and the default high_water is 48,
4300 * the low_water is 16 as default.
4302 hw->fc.high_water[i] = (pbsize * 3) / 4;
4303 hw->fc.low_water[i] = pbsize / 4;
4304 /* Enable pfc for this TC */
4305 tc = &dcb_config->tc_config[i];
4306 tc->pfc = ixgbe_dcb_pfc_enabled;
4308 ixgbe_dcb_unpack_pfc_cee(dcb_config, map, &pfc_en);
4309 if (dcb_config->num_tcs.pfc_tcs == ETH_4_TCS)
4311 ret = ixgbe_dcb_config_pfc(hw, pfc_en, map);
4318 * ixgbe_configure_dcb - Configure DCB Hardware
4319 * @dev: pointer to rte_eth_dev
4321 void ixgbe_configure_dcb(struct rte_eth_dev *dev)
4323 struct ixgbe_dcb_config *dcb_cfg =
4324 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private);
4325 struct rte_eth_conf *dev_conf = &(dev->data->dev_conf);
4327 PMD_INIT_FUNC_TRACE();
4329 /* check support mq_mode for DCB */
4330 if ((dev_conf->rxmode.mq_mode != ETH_MQ_RX_VMDQ_DCB) &&
4331 (dev_conf->rxmode.mq_mode != ETH_MQ_RX_DCB) &&
4332 (dev_conf->rxmode.mq_mode != ETH_MQ_RX_DCB_RSS))
4335 if (dev->data->nb_rx_queues > ETH_DCB_NUM_QUEUES)
4338 /** Configure DCB hardware **/
4339 ixgbe_dcb_hw_configure(dev, dcb_cfg);
4343 * VMDq only support for 10 GbE NIC.
4346 ixgbe_vmdq_rx_hw_configure(struct rte_eth_dev *dev)
4348 struct rte_eth_vmdq_rx_conf *cfg;
4349 struct ixgbe_hw *hw;
4350 enum rte_eth_nb_pools num_pools;
4351 uint32_t mrqc, vt_ctl, vlanctrl;
4355 PMD_INIT_FUNC_TRACE();
4356 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4357 cfg = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
4358 num_pools = cfg->nb_queue_pools;
4360 ixgbe_rss_disable(dev);
4362 /* MRQC: enable vmdq */
4363 mrqc = IXGBE_MRQC_VMDQEN;
4364 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4366 /* PFVTCTL: turn on virtualisation and set the default pool */
4367 vt_ctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
4368 if (cfg->enable_default_pool)
4369 vt_ctl |= (cfg->default_pool << IXGBE_VT_CTL_POOL_SHIFT);
4371 vt_ctl |= IXGBE_VT_CTL_DIS_DEFPL;
4373 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vt_ctl);
4375 for (i = 0; i < (int)num_pools; i++) {
4376 vmolr = ixgbe_convert_vm_rx_mask_to_val(cfg->rx_mode, vmolr);
4377 IXGBE_WRITE_REG(hw, IXGBE_VMOLR(i), vmolr);
4380 /* VLNCTRL: enable vlan filtering and allow all vlan tags through */
4381 vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4382 vlanctrl |= IXGBE_VLNCTRL_VFE; /* enable vlan filters */
4383 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl);
4385 /* VFTA - enable all vlan filters */
4386 for (i = 0; i < NUM_VFTA_REGISTERS; i++)
4387 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), UINT32_MAX);
4389 /* VFRE: pool enabling for receive - 64 */
4390 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), UINT32_MAX);
4391 if (num_pools == ETH_64_POOLS)
4392 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), UINT32_MAX);
4395 * MPSAR - allow pools to read specific mac addresses
4396 * In this case, all pools should be able to read from mac addr 0
4398 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(0), UINT32_MAX);
4399 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(0), UINT32_MAX);
4401 /* PFVLVF, PFVLVFB: set up filters for vlan tags as configured */
4402 for (i = 0; i < cfg->nb_pool_maps; i++) {
4403 /* set vlan id in VF register and set the valid bit */
4404 IXGBE_WRITE_REG(hw, IXGBE_VLVF(i), (IXGBE_VLVF_VIEN |
4405 (cfg->pool_map[i].vlan_id & IXGBE_RXD_VLAN_ID_MASK)));
4407 * Put the allowed pools in VFB reg. As we only have 16 or 64
4408 * pools, we only need to use the first half of the register
4411 if (((cfg->pool_map[i].pools >> 32) & UINT32_MAX) == 0)
4412 IXGBE_WRITE_REG(hw, IXGBE_VLVFB(i * 2),
4413 (cfg->pool_map[i].pools & UINT32_MAX));
4415 IXGBE_WRITE_REG(hw, IXGBE_VLVFB((i * 2 + 1)),
4416 ((cfg->pool_map[i].pools >> 32) & UINT32_MAX));
4420 /* PFDMA Tx General Switch Control Enables VMDQ loopback */
4421 if (cfg->enable_loop_back) {
4422 IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN);
4423 for (i = 0; i < RTE_IXGBE_VMTXSW_REGISTER_COUNT; i++)
4424 IXGBE_WRITE_REG(hw, IXGBE_VMTXSW(i), UINT32_MAX);
4427 IXGBE_WRITE_FLUSH(hw);
4431 * ixgbe_dcb_config_tx_hw_config - Configure general VMDq TX parameters
4432 * @hw: pointer to hardware structure
4435 ixgbe_vmdq_tx_hw_configure(struct ixgbe_hw *hw)
4440 PMD_INIT_FUNC_TRACE();
4441 /*PF VF Transmit Enable*/
4442 IXGBE_WRITE_REG(hw, IXGBE_VFTE(0), UINT32_MAX);
4443 IXGBE_WRITE_REG(hw, IXGBE_VFTE(1), UINT32_MAX);
4445 /* Disable the Tx desc arbiter so that MTQC can be changed */
4446 reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
4447 reg |= IXGBE_RTTDCS_ARBDIS;
4448 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg);
4450 reg = IXGBE_MTQC_VT_ENA | IXGBE_MTQC_64VF;
4451 IXGBE_WRITE_REG(hw, IXGBE_MTQC, reg);
4453 /* Disable drop for all queues */
4454 for (q = 0; q < IXGBE_MAX_RX_QUEUE_NUM; q++)
4455 IXGBE_WRITE_REG(hw, IXGBE_QDE,
4456 (IXGBE_QDE_WRITE | (q << IXGBE_QDE_IDX_SHIFT)));
4458 /* Enable the Tx desc arbiter */
4459 reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
4460 reg &= ~IXGBE_RTTDCS_ARBDIS;
4461 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg);
4463 IXGBE_WRITE_FLUSH(hw);
4466 static int __rte_cold
4467 ixgbe_alloc_rx_queue_mbufs(struct ixgbe_rx_queue *rxq)
4469 struct ixgbe_rx_entry *rxe = rxq->sw_ring;
4473 /* Initialize software ring entries */
4474 for (i = 0; i < rxq->nb_rx_desc; i++) {
4475 volatile union ixgbe_adv_rx_desc *rxd;
4476 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mb_pool);
4479 PMD_INIT_LOG(ERR, "RX mbuf alloc failed queue_id=%u",
4480 (unsigned) rxq->queue_id);
4484 mbuf->data_off = RTE_PKTMBUF_HEADROOM;
4485 mbuf->port = rxq->port_id;
4488 rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf));
4489 rxd = &rxq->rx_ring[i];
4490 rxd->read.hdr_addr = 0;
4491 rxd->read.pkt_addr = dma_addr;
4499 ixgbe_config_vf_rss(struct rte_eth_dev *dev)
4501 struct ixgbe_hw *hw;
4504 ixgbe_rss_configure(dev);
4506 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4508 /* MRQC: enable VF RSS */
4509 mrqc = IXGBE_READ_REG(hw, IXGBE_MRQC);
4510 mrqc &= ~IXGBE_MRQC_MRQE_MASK;
4511 switch (RTE_ETH_DEV_SRIOV(dev).active) {
4513 mrqc |= IXGBE_MRQC_VMDQRSS64EN;
4517 mrqc |= IXGBE_MRQC_VMDQRSS32EN;
4521 PMD_INIT_LOG(ERR, "Invalid pool number in IOV mode with VMDQ RSS");
4525 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4531 ixgbe_config_vf_default(struct rte_eth_dev *dev)
4533 struct ixgbe_hw *hw =
4534 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4536 switch (RTE_ETH_DEV_SRIOV(dev).active) {
4538 IXGBE_WRITE_REG(hw, IXGBE_MRQC,
4543 IXGBE_WRITE_REG(hw, IXGBE_MRQC,
4544 IXGBE_MRQC_VMDQRT4TCEN);
4548 IXGBE_WRITE_REG(hw, IXGBE_MRQC,
4549 IXGBE_MRQC_VMDQRT8TCEN);
4553 "invalid pool number in IOV mode");
4560 ixgbe_dev_mq_rx_configure(struct rte_eth_dev *dev)
4562 struct ixgbe_hw *hw =
4563 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4565 if (hw->mac.type == ixgbe_mac_82598EB)
4568 if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
4570 * SRIOV inactive scheme
4571 * any DCB/RSS w/o VMDq multi-queue setting
4573 switch (dev->data->dev_conf.rxmode.mq_mode) {
4575 case ETH_MQ_RX_DCB_RSS:
4576 case ETH_MQ_RX_VMDQ_RSS:
4577 ixgbe_rss_configure(dev);
4580 case ETH_MQ_RX_VMDQ_DCB:
4581 ixgbe_vmdq_dcb_configure(dev);
4584 case ETH_MQ_RX_VMDQ_ONLY:
4585 ixgbe_vmdq_rx_hw_configure(dev);
4588 case ETH_MQ_RX_NONE:
4590 /* if mq_mode is none, disable rss mode.*/
4591 ixgbe_rss_disable(dev);
4595 /* SRIOV active scheme
4596 * Support RSS together with SRIOV.
4598 switch (dev->data->dev_conf.rxmode.mq_mode) {
4600 case ETH_MQ_RX_VMDQ_RSS:
4601 ixgbe_config_vf_rss(dev);
4603 case ETH_MQ_RX_VMDQ_DCB:
4605 /* In SRIOV, the configuration is the same as VMDq case */
4606 ixgbe_vmdq_dcb_configure(dev);
4608 /* DCB/RSS together with SRIOV is not supported */
4609 case ETH_MQ_RX_VMDQ_DCB_RSS:
4610 case ETH_MQ_RX_DCB_RSS:
4612 "Could not support DCB/RSS with VMDq & SRIOV");
4615 ixgbe_config_vf_default(dev);
4624 ixgbe_dev_mq_tx_configure(struct rte_eth_dev *dev)
4626 struct ixgbe_hw *hw =
4627 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4631 if (hw->mac.type == ixgbe_mac_82598EB)
4634 /* disable arbiter before setting MTQC */
4635 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
4636 rttdcs |= IXGBE_RTTDCS_ARBDIS;
4637 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
4639 if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
4641 * SRIOV inactive scheme
4642 * any DCB w/o VMDq multi-queue setting
4644 if (dev->data->dev_conf.txmode.mq_mode == ETH_MQ_TX_VMDQ_ONLY)
4645 ixgbe_vmdq_tx_hw_configure(hw);
4647 mtqc = IXGBE_MTQC_64Q_1PB;
4648 IXGBE_WRITE_REG(hw, IXGBE_MTQC, mtqc);
4651 switch (RTE_ETH_DEV_SRIOV(dev).active) {
4654 * SRIOV active scheme
4655 * FIXME if support DCB together with VMDq & SRIOV
4658 mtqc = IXGBE_MTQC_VT_ENA | IXGBE_MTQC_64VF;
4661 mtqc = IXGBE_MTQC_VT_ENA | IXGBE_MTQC_32VF;
4664 mtqc = IXGBE_MTQC_VT_ENA | IXGBE_MTQC_RT_ENA |
4668 mtqc = IXGBE_MTQC_64Q_1PB;
4669 PMD_INIT_LOG(ERR, "invalid pool number in IOV mode");
4671 IXGBE_WRITE_REG(hw, IXGBE_MTQC, mtqc);
4674 /* re-enable arbiter */
4675 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
4676 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
4682 * ixgbe_get_rscctl_maxdesc - Calculate the RSCCTL[n].MAXDESC for PF
4684 * Return the RSCCTL[n].MAXDESC for 82599 and x540 PF devices according to the
4685 * spec rev. 3.0 chapter 8.2.3.8.13.
4687 * @pool Memory pool of the Rx queue
4689 static inline uint32_t
4690 ixgbe_get_rscctl_maxdesc(struct rte_mempool *pool)
4692 struct rte_pktmbuf_pool_private *mp_priv = rte_mempool_get_priv(pool);
4694 /* MAXDESC * SRRCTL.BSIZEPKT must not exceed 64 KB minus one */
4696 RTE_IPV4_MAX_PKT_LEN /
4697 (mp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM);
4700 return IXGBE_RSCCTL_MAXDESC_16;
4701 else if (maxdesc >= 8)
4702 return IXGBE_RSCCTL_MAXDESC_8;
4703 else if (maxdesc >= 4)
4704 return IXGBE_RSCCTL_MAXDESC_4;
4706 return IXGBE_RSCCTL_MAXDESC_1;
4710 * ixgbe_set_ivar - Setup the correct IVAR register for a particular MSIX
4713 * (Taken from FreeBSD tree)
4714 * (yes this is all very magic and confusing :)
4717 * @entry the register array entry
4718 * @vector the MSIX vector for this queue
4722 ixgbe_set_ivar(struct rte_eth_dev *dev, u8 entry, u8 vector, s8 type)
4724 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4727 vector |= IXGBE_IVAR_ALLOC_VAL;
4729 switch (hw->mac.type) {
4731 case ixgbe_mac_82598EB:
4733 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
4735 entry += (type * 64);
4736 index = (entry >> 2) & 0x1F;
4737 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4738 ivar &= ~(0xFF << (8 * (entry & 0x3)));
4739 ivar |= (vector << (8 * (entry & 0x3)));
4740 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
4743 case ixgbe_mac_82599EB:
4744 case ixgbe_mac_X540:
4745 if (type == -1) { /* MISC IVAR */
4746 index = (entry & 1) * 8;
4747 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4748 ivar &= ~(0xFF << index);
4749 ivar |= (vector << index);
4750 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4751 } else { /* RX/TX IVARS */
4752 index = (16 * (entry & 1)) + (8 * type);
4753 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
4754 ivar &= ~(0xFF << index);
4755 ivar |= (vector << index);
4756 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
4767 ixgbe_set_rx_function(struct rte_eth_dev *dev)
4769 uint16_t i, rx_using_sse;
4770 struct ixgbe_adapter *adapter = dev->data->dev_private;
4773 * In order to allow Vector Rx there are a few configuration
4774 * conditions to be met and Rx Bulk Allocation should be allowed.
4776 if (ixgbe_rx_vec_dev_conf_condition_check(dev) ||
4777 !adapter->rx_bulk_alloc_allowed ||
4778 rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128) {
4779 PMD_INIT_LOG(DEBUG, "Port[%d] doesn't meet Vector Rx "
4781 dev->data->port_id);
4783 adapter->rx_vec_allowed = false;
4787 * Initialize the appropriate LRO callback.
4789 * If all queues satisfy the bulk allocation preconditions
4790 * (hw->rx_bulk_alloc_allowed is TRUE) then we may use bulk allocation.
4791 * Otherwise use a single allocation version.
4793 if (dev->data->lro) {
4794 if (adapter->rx_bulk_alloc_allowed) {
4795 PMD_INIT_LOG(DEBUG, "LRO is requested. Using a bulk "
4796 "allocation version");
4797 dev->rx_pkt_burst = ixgbe_recv_pkts_lro_bulk_alloc;
4799 PMD_INIT_LOG(DEBUG, "LRO is requested. Using a single "
4800 "allocation version");
4801 dev->rx_pkt_burst = ixgbe_recv_pkts_lro_single_alloc;
4803 } else if (dev->data->scattered_rx) {
4805 * Set the non-LRO scattered callback: there are Vector and
4806 * single allocation versions.
4808 if (adapter->rx_vec_allowed) {
4809 PMD_INIT_LOG(DEBUG, "Using Vector Scattered Rx "
4810 "callback (port=%d).",
4811 dev->data->port_id);
4813 dev->rx_pkt_burst = ixgbe_recv_scattered_pkts_vec;
4814 } else if (adapter->rx_bulk_alloc_allowed) {
4815 PMD_INIT_LOG(DEBUG, "Using a Scattered with bulk "
4816 "allocation callback (port=%d).",
4817 dev->data->port_id);
4818 dev->rx_pkt_burst = ixgbe_recv_pkts_lro_bulk_alloc;
4820 PMD_INIT_LOG(DEBUG, "Using Regualr (non-vector, "
4821 "single allocation) "
4822 "Scattered Rx callback "
4824 dev->data->port_id);
4826 dev->rx_pkt_burst = ixgbe_recv_pkts_lro_single_alloc;
4829 * Below we set "simple" callbacks according to port/queues parameters.
4830 * If parameters allow we are going to choose between the following
4834 * - Single buffer allocation (the simplest one)
4836 } else if (adapter->rx_vec_allowed) {
4837 PMD_INIT_LOG(DEBUG, "Vector rx enabled, please make sure RX "
4838 "burst size no less than %d (port=%d).",
4839 RTE_IXGBE_DESCS_PER_LOOP,
4840 dev->data->port_id);
4842 dev->rx_pkt_burst = ixgbe_recv_pkts_vec;
4843 } else if (adapter->rx_bulk_alloc_allowed) {
4844 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are "
4845 "satisfied. Rx Burst Bulk Alloc function "
4846 "will be used on port=%d.",
4847 dev->data->port_id);
4849 dev->rx_pkt_burst = ixgbe_recv_pkts_bulk_alloc;
4851 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are not "
4852 "satisfied, or Scattered Rx is requested "
4854 dev->data->port_id);
4856 dev->rx_pkt_burst = ixgbe_recv_pkts;
4859 /* Propagate information about RX function choice through all queues. */
4862 (dev->rx_pkt_burst == ixgbe_recv_scattered_pkts_vec ||
4863 dev->rx_pkt_burst == ixgbe_recv_pkts_vec);
4865 for (i = 0; i < dev->data->nb_rx_queues; i++) {
4866 struct ixgbe_rx_queue *rxq = dev->data->rx_queues[i];
4868 rxq->rx_using_sse = rx_using_sse;
4869 #ifdef RTE_LIB_SECURITY
4870 rxq->using_ipsec = !!(dev->data->dev_conf.rxmode.offloads &
4871 DEV_RX_OFFLOAD_SECURITY);
4877 * ixgbe_set_rsc - configure RSC related port HW registers
4879 * Configures the port's RSC related registers according to the 4.6.7.2 chapter
4880 * of 82599 Spec (x540 configuration is virtually the same).
4884 * Returns 0 in case of success or a non-zero error code
4887 ixgbe_set_rsc(struct rte_eth_dev *dev)
4889 struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode;
4890 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4891 struct rte_eth_dev_info dev_info = { 0 };
4892 bool rsc_capable = false;
4898 dev->dev_ops->dev_infos_get(dev, &dev_info);
4899 if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TCP_LRO)
4902 if (!rsc_capable && (rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO)) {
4903 PMD_INIT_LOG(CRIT, "LRO is requested on HW that doesn't "
4908 /* RSC global configuration (chapter 4.6.7.2.1 of 82599 Spec) */
4910 if ((rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC) &&
4911 (rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO)) {
4913 * According to chapter of 4.6.7.2.1 of the Spec Rev.
4914 * 3.0 RSC configuration requires HW CRC stripping being
4915 * enabled. If user requested both HW CRC stripping off
4916 * and RSC on - return an error.
4918 PMD_INIT_LOG(CRIT, "LRO can't be enabled when HW CRC "
4923 /* RFCTL configuration */
4924 rfctl = IXGBE_READ_REG(hw, IXGBE_RFCTL);
4925 if ((rsc_capable) && (rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO))
4926 rfctl &= ~IXGBE_RFCTL_RSC_DIS;
4928 rfctl |= IXGBE_RFCTL_RSC_DIS;
4929 /* disable NFS filtering */
4930 rfctl |= IXGBE_RFCTL_NFSW_DIS | IXGBE_RFCTL_NFSR_DIS;
4931 IXGBE_WRITE_REG(hw, IXGBE_RFCTL, rfctl);
4933 /* If LRO hasn't been requested - we are done here. */
4934 if (!(rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO))
4937 /* Set RDRXCTL.RSCACKC bit */
4938 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
4939 rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
4940 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
4942 /* Per-queue RSC configuration (chapter 4.6.7.2.2 of 82599 Spec) */
4943 for (i = 0; i < dev->data->nb_rx_queues; i++) {
4944 struct ixgbe_rx_queue *rxq = dev->data->rx_queues[i];
4946 IXGBE_READ_REG(hw, IXGBE_SRRCTL(rxq->reg_idx));
4948 IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxq->reg_idx));
4950 IXGBE_READ_REG(hw, IXGBE_PSRTYPE(rxq->reg_idx));
4952 IXGBE_READ_REG(hw, IXGBE_EITR(rxq->reg_idx));
4955 * ixgbe PMD doesn't support header-split at the moment.
4957 * Following the 4.6.7.2.1 chapter of the 82599/x540
4958 * Spec if RSC is enabled the SRRCTL[n].BSIZEHEADER
4959 * should be configured even if header split is not
4960 * enabled. We will configure it 128 bytes following the
4961 * recommendation in the spec.
4963 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
4964 srrctl |= (128 << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) &
4965 IXGBE_SRRCTL_BSIZEHDR_MASK;
4968 * TODO: Consider setting the Receive Descriptor Minimum
4969 * Threshold Size for an RSC case. This is not an obviously
4970 * beneficiary option but the one worth considering...
4973 rscctl |= IXGBE_RSCCTL_RSCEN;
4974 rscctl |= ixgbe_get_rscctl_maxdesc(rxq->mb_pool);
4975 psrtype |= IXGBE_PSRTYPE_TCPHDR;
4978 * RSC: Set ITR interval corresponding to 2K ints/s.
4980 * Full-sized RSC aggregations for a 10Gb/s link will
4981 * arrive at about 20K aggregation/s rate.
4983 * 2K inst/s rate will make only 10% of the
4984 * aggregations to be closed due to the interrupt timer
4985 * expiration for a streaming at wire-speed case.
4987 * For a sparse streaming case this setting will yield
4988 * at most 500us latency for a single RSC aggregation.
4990 eitr &= ~IXGBE_EITR_ITR_INT_MASK;
4991 eitr |= IXGBE_EITR_INTERVAL_US(IXGBE_QUEUE_ITR_INTERVAL_DEFAULT);
4992 eitr |= IXGBE_EITR_CNT_WDIS;
4994 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxq->reg_idx), srrctl);
4995 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxq->reg_idx), rscctl);
4996 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(rxq->reg_idx), psrtype);
4997 IXGBE_WRITE_REG(hw, IXGBE_EITR(rxq->reg_idx), eitr);
5000 * RSC requires the mapping of the queue to the
5003 ixgbe_set_ivar(dev, rxq->reg_idx, i, 0);
5008 PMD_INIT_LOG(DEBUG, "enabling LRO mode");
5014 * Initializes Receive Unit.
5017 ixgbe_dev_rx_init(struct rte_eth_dev *dev)
5019 struct ixgbe_hw *hw;
5020 struct ixgbe_rx_queue *rxq;
5031 struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode;
5034 PMD_INIT_FUNC_TRACE();
5035 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5038 * Make sure receives are disabled while setting
5039 * up the RX context (registers, descriptor rings, etc.).
5041 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
5042 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl & ~IXGBE_RXCTRL_RXEN);
5044 /* Enable receipt of broadcasted frames */
5045 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
5046 fctrl |= IXGBE_FCTRL_BAM;
5047 fctrl |= IXGBE_FCTRL_DPF;
5048 fctrl |= IXGBE_FCTRL_PMCF;
5049 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
5052 * Configure CRC stripping, if any.
5054 hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
5055 if (rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC)
5056 hlreg0 &= ~IXGBE_HLREG0_RXCRCSTRP;
5058 hlreg0 |= IXGBE_HLREG0_RXCRCSTRP;
5061 * Configure jumbo frame support, if any.
5063 if (rx_conf->offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) {
5064 hlreg0 |= IXGBE_HLREG0_JUMBOEN;
5065 maxfrs = IXGBE_READ_REG(hw, IXGBE_MAXFRS);
5066 maxfrs &= 0x0000FFFF;
5067 maxfrs |= (rx_conf->max_rx_pkt_len << 16);
5068 IXGBE_WRITE_REG(hw, IXGBE_MAXFRS, maxfrs);
5070 hlreg0 &= ~IXGBE_HLREG0_JUMBOEN;
5073 * If loopback mode is configured, set LPBK bit.
5075 if (dev->data->dev_conf.lpbk_mode != 0) {
5076 rc = ixgbe_check_supported_loopback_mode(dev);
5078 PMD_INIT_LOG(ERR, "Unsupported loopback mode");
5081 hlreg0 |= IXGBE_HLREG0_LPBK;
5083 hlreg0 &= ~IXGBE_HLREG0_LPBK;
5086 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);
5089 * Assume no header split and no VLAN strip support
5090 * on any Rx queue first .
5092 rx_conf->offloads &= ~DEV_RX_OFFLOAD_VLAN_STRIP;
5093 /* Setup RX queues */
5094 for (i = 0; i < dev->data->nb_rx_queues; i++) {
5095 rxq = dev->data->rx_queues[i];
5098 * Reset crc_len in case it was changed after queue setup by a
5099 * call to configure.
5101 if (rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC)
5102 rxq->crc_len = RTE_ETHER_CRC_LEN;
5106 /* Setup the Base and Length of the Rx Descriptor Rings */
5107 bus_addr = rxq->rx_ring_phys_addr;
5108 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(rxq->reg_idx),
5109 (uint32_t)(bus_addr & 0x00000000ffffffffULL));
5110 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(rxq->reg_idx),
5111 (uint32_t)(bus_addr >> 32));
5112 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(rxq->reg_idx),
5113 rxq->nb_rx_desc * sizeof(union ixgbe_adv_rx_desc));
5114 IXGBE_WRITE_REG(hw, IXGBE_RDH(rxq->reg_idx), 0);
5115 IXGBE_WRITE_REG(hw, IXGBE_RDT(rxq->reg_idx), 0);
5117 /* Configure the SRRCTL register */
5118 srrctl = IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
5120 /* Set if packets are dropped when no descriptors available */
5122 srrctl |= IXGBE_SRRCTL_DROP_EN;
5125 * Configure the RX buffer size in the BSIZEPACKET field of
5126 * the SRRCTL register of the queue.
5127 * The value is in 1 KB resolution. Valid values can be from
5130 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) -
5131 RTE_PKTMBUF_HEADROOM);
5132 srrctl |= ((buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) &
5133 IXGBE_SRRCTL_BSIZEPKT_MASK);
5135 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxq->reg_idx), srrctl);
5137 buf_size = (uint16_t) ((srrctl & IXGBE_SRRCTL_BSIZEPKT_MASK) <<
5138 IXGBE_SRRCTL_BSIZEPKT_SHIFT);
5140 /* It adds dual VLAN length for supporting dual VLAN */
5141 if (dev->data->dev_conf.rxmode.max_rx_pkt_len +
5142 2 * IXGBE_VLAN_TAG_SIZE > buf_size)
5143 dev->data->scattered_rx = 1;
5144 if (rxq->offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
5145 rx_conf->offloads |= DEV_RX_OFFLOAD_VLAN_STRIP;
5148 if (rx_conf->offloads & DEV_RX_OFFLOAD_SCATTER)
5149 dev->data->scattered_rx = 1;
5152 * Device configured with multiple RX queues.
5154 ixgbe_dev_mq_rx_configure(dev);
5157 * Setup the Checksum Register.
5158 * Disable Full-Packet Checksum which is mutually exclusive with RSS.
5159 * Enable IP/L4 checkum computation by hardware if requested to do so.
5161 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
5162 rxcsum |= IXGBE_RXCSUM_PCSD;
5163 if (rx_conf->offloads & DEV_RX_OFFLOAD_CHECKSUM)
5164 rxcsum |= IXGBE_RXCSUM_IPPCSE;
5166 rxcsum &= ~IXGBE_RXCSUM_IPPCSE;
5168 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
5170 if (hw->mac.type == ixgbe_mac_82599EB ||
5171 hw->mac.type == ixgbe_mac_X540) {
5172 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
5173 if (rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC)
5174 rdrxctl &= ~IXGBE_RDRXCTL_CRCSTRIP;
5176 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
5177 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
5178 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
5181 rc = ixgbe_set_rsc(dev);
5185 ixgbe_set_rx_function(dev);
5191 * Initializes Transmit Unit.
5194 ixgbe_dev_tx_init(struct rte_eth_dev *dev)
5196 struct ixgbe_hw *hw;
5197 struct ixgbe_tx_queue *txq;
5203 PMD_INIT_FUNC_TRACE();
5204 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5206 /* Enable TX CRC (checksum offload requirement) and hw padding
5209 hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
5210 hlreg0 |= (IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN);
5211 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);
5213 /* Setup the Base and Length of the Tx Descriptor Rings */
5214 for (i = 0; i < dev->data->nb_tx_queues; i++) {
5215 txq = dev->data->tx_queues[i];
5217 bus_addr = txq->tx_ring_phys_addr;
5218 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(txq->reg_idx),
5219 (uint32_t)(bus_addr & 0x00000000ffffffffULL));
5220 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(txq->reg_idx),
5221 (uint32_t)(bus_addr >> 32));
5222 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(txq->reg_idx),
5223 txq->nb_tx_desc * sizeof(union ixgbe_adv_tx_desc));
5224 /* Setup the HW Tx Head and TX Tail descriptor pointers */
5225 IXGBE_WRITE_REG(hw, IXGBE_TDH(txq->reg_idx), 0);
5226 IXGBE_WRITE_REG(hw, IXGBE_TDT(txq->reg_idx), 0);
5229 * Disable Tx Head Writeback RO bit, since this hoses
5230 * bookkeeping if things aren't delivered in order.
5232 switch (hw->mac.type) {
5233 case ixgbe_mac_82598EB:
5234 txctrl = IXGBE_READ_REG(hw,
5235 IXGBE_DCA_TXCTRL(txq->reg_idx));
5236 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
5237 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(txq->reg_idx),
5241 case ixgbe_mac_82599EB:
5242 case ixgbe_mac_X540:
5243 case ixgbe_mac_X550:
5244 case ixgbe_mac_X550EM_x:
5245 case ixgbe_mac_X550EM_a:
5247 txctrl = IXGBE_READ_REG(hw,
5248 IXGBE_DCA_TXCTRL_82599(txq->reg_idx));
5249 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
5250 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(txq->reg_idx),
5256 /* Device configured with multiple TX queues. */
5257 ixgbe_dev_mq_tx_configure(dev);
5261 * Check if requested loopback mode is supported
5264 ixgbe_check_supported_loopback_mode(struct rte_eth_dev *dev)
5266 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5268 if (dev->data->dev_conf.lpbk_mode == IXGBE_LPBK_TX_RX)
5269 if (hw->mac.type == ixgbe_mac_82599EB ||
5270 hw->mac.type == ixgbe_mac_X540 ||
5271 hw->mac.type == ixgbe_mac_X550 ||
5272 hw->mac.type == ixgbe_mac_X550EM_x ||
5273 hw->mac.type == ixgbe_mac_X550EM_a)
5280 * Set up link for 82599 loopback mode Tx->Rx.
5282 static inline void __rte_cold
5283 ixgbe_setup_loopback_link_82599(struct ixgbe_hw *hw)
5285 PMD_INIT_FUNC_TRACE();
5287 if (ixgbe_verify_lesm_fw_enabled_82599(hw)) {
5288 if (hw->mac.ops.acquire_swfw_sync(hw, IXGBE_GSSR_MAC_CSR_SM) !=
5290 PMD_INIT_LOG(ERR, "Could not enable loopback mode");
5299 IXGBE_AUTOC_LMS_10G_LINK_NO_AN | IXGBE_AUTOC_FLU);
5300 ixgbe_reset_pipeline_82599(hw);
5302 hw->mac.ops.release_swfw_sync(hw, IXGBE_GSSR_MAC_CSR_SM);
5308 * Start Transmit and Receive Units.
5311 ixgbe_dev_rxtx_start(struct rte_eth_dev *dev)
5313 struct ixgbe_hw *hw;
5314 struct ixgbe_tx_queue *txq;
5315 struct ixgbe_rx_queue *rxq;
5322 PMD_INIT_FUNC_TRACE();
5323 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5325 for (i = 0; i < dev->data->nb_tx_queues; i++) {
5326 txq = dev->data->tx_queues[i];
5327 /* Setup Transmit Threshold Registers */
5328 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txq->reg_idx));
5329 txdctl |= txq->pthresh & 0x7F;
5330 txdctl |= ((txq->hthresh & 0x7F) << 8);
5331 txdctl |= ((txq->wthresh & 0x7F) << 16);
5332 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txq->reg_idx), txdctl);
5335 if (hw->mac.type != ixgbe_mac_82598EB) {
5336 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
5337 dmatxctl |= IXGBE_DMATXCTL_TE;
5338 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
5341 for (i = 0; i < dev->data->nb_tx_queues; i++) {
5342 txq = dev->data->tx_queues[i];
5343 if (!txq->tx_deferred_start) {
5344 ret = ixgbe_dev_tx_queue_start(dev, i);
5350 for (i = 0; i < dev->data->nb_rx_queues; i++) {
5351 rxq = dev->data->rx_queues[i];
5352 if (!rxq->rx_deferred_start) {
5353 ret = ixgbe_dev_rx_queue_start(dev, i);
5359 /* Enable Receive engine */
5360 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
5361 if (hw->mac.type == ixgbe_mac_82598EB)
5362 rxctrl |= IXGBE_RXCTRL_DMBYPS;
5363 rxctrl |= IXGBE_RXCTRL_RXEN;
5364 hw->mac.ops.enable_rx_dma(hw, rxctrl);
5366 /* If loopback mode is enabled, set up the link accordingly */
5367 if (dev->data->dev_conf.lpbk_mode != 0) {
5368 if (hw->mac.type == ixgbe_mac_82599EB)
5369 ixgbe_setup_loopback_link_82599(hw);
5370 else if (hw->mac.type == ixgbe_mac_X540 ||
5371 hw->mac.type == ixgbe_mac_X550 ||
5372 hw->mac.type == ixgbe_mac_X550EM_x ||
5373 hw->mac.type == ixgbe_mac_X550EM_a)
5374 ixgbe_setup_loopback_link_x540_x550(hw, true);
5377 #ifdef RTE_LIB_SECURITY
5378 if ((dev->data->dev_conf.rxmode.offloads &
5379 DEV_RX_OFFLOAD_SECURITY) ||
5380 (dev->data->dev_conf.txmode.offloads &
5381 DEV_TX_OFFLOAD_SECURITY)) {
5382 ret = ixgbe_crypto_enable_ipsec(dev);
5385 "ixgbe_crypto_enable_ipsec fails with %d.",
5396 * Start Receive Units for specified queue.
5399 ixgbe_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
5401 struct ixgbe_hw *hw;
5402 struct ixgbe_rx_queue *rxq;
5406 PMD_INIT_FUNC_TRACE();
5407 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5409 rxq = dev->data->rx_queues[rx_queue_id];
5411 /* Allocate buffers for descriptor rings */
5412 if (ixgbe_alloc_rx_queue_mbufs(rxq) != 0) {
5413 PMD_INIT_LOG(ERR, "Could not alloc mbuf for queue:%d",
5417 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
5418 rxdctl |= IXGBE_RXDCTL_ENABLE;
5419 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxq->reg_idx), rxdctl);
5421 /* Wait until RX Enable ready */
5422 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
5425 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
5426 } while (--poll_ms && !(rxdctl & IXGBE_RXDCTL_ENABLE));
5428 PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", rx_queue_id);
5430 IXGBE_WRITE_REG(hw, IXGBE_RDH(rxq->reg_idx), 0);
5431 IXGBE_WRITE_REG(hw, IXGBE_RDT(rxq->reg_idx), rxq->nb_rx_desc - 1);
5432 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
5438 * Stop Receive Units for specified queue.
5441 ixgbe_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
5443 struct ixgbe_hw *hw;
5444 struct ixgbe_adapter *adapter = dev->data->dev_private;
5445 struct ixgbe_rx_queue *rxq;
5449 PMD_INIT_FUNC_TRACE();
5450 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5452 rxq = dev->data->rx_queues[rx_queue_id];
5454 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
5455 rxdctl &= ~IXGBE_RXDCTL_ENABLE;
5456 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxq->reg_idx), rxdctl);
5458 /* Wait until RX Enable bit clear */
5459 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
5462 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
5463 } while (--poll_ms && (rxdctl & IXGBE_RXDCTL_ENABLE));
5465 PMD_INIT_LOG(ERR, "Could not disable Rx Queue %d", rx_queue_id);
5467 rte_delay_us(RTE_IXGBE_WAIT_100_US);
5469 ixgbe_rx_queue_release_mbufs(rxq);
5470 ixgbe_reset_rx_queue(adapter, rxq);
5471 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
5478 * Start Transmit Units for specified queue.
5481 ixgbe_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
5483 struct ixgbe_hw *hw;
5484 struct ixgbe_tx_queue *txq;
5488 PMD_INIT_FUNC_TRACE();
5489 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5491 txq = dev->data->tx_queues[tx_queue_id];
5492 IXGBE_WRITE_REG(hw, IXGBE_TDH(txq->reg_idx), 0);
5493 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txq->reg_idx));
5494 txdctl |= IXGBE_TXDCTL_ENABLE;
5495 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txq->reg_idx), txdctl);
5497 /* Wait until TX Enable ready */
5498 if (hw->mac.type == ixgbe_mac_82599EB) {
5499 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
5502 txdctl = IXGBE_READ_REG(hw,
5503 IXGBE_TXDCTL(txq->reg_idx));
5504 } while (--poll_ms && !(txdctl & IXGBE_TXDCTL_ENABLE));
5506 PMD_INIT_LOG(ERR, "Could not enable Tx Queue %d",
5510 IXGBE_WRITE_REG(hw, IXGBE_TDT(txq->reg_idx), 0);
5511 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
5517 * Stop Transmit Units for specified queue.
5520 ixgbe_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
5522 struct ixgbe_hw *hw;
5523 struct ixgbe_tx_queue *txq;
5525 uint32_t txtdh, txtdt;
5528 PMD_INIT_FUNC_TRACE();
5529 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5531 txq = dev->data->tx_queues[tx_queue_id];
5533 /* Wait until TX queue is empty */
5534 if (hw->mac.type == ixgbe_mac_82599EB) {
5535 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
5537 rte_delay_us(RTE_IXGBE_WAIT_100_US);
5538 txtdh = IXGBE_READ_REG(hw,
5539 IXGBE_TDH(txq->reg_idx));
5540 txtdt = IXGBE_READ_REG(hw,
5541 IXGBE_TDT(txq->reg_idx));
5542 } while (--poll_ms && (txtdh != txtdt));
5545 "Tx Queue %d is not empty when stopping.",
5549 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txq->reg_idx));
5550 txdctl &= ~IXGBE_TXDCTL_ENABLE;
5551 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txq->reg_idx), txdctl);
5553 /* Wait until TX Enable bit clear */
5554 if (hw->mac.type == ixgbe_mac_82599EB) {
5555 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
5558 txdctl = IXGBE_READ_REG(hw,
5559 IXGBE_TXDCTL(txq->reg_idx));
5560 } while (--poll_ms && (txdctl & IXGBE_TXDCTL_ENABLE));
5562 PMD_INIT_LOG(ERR, "Could not disable Tx Queue %d",
5566 if (txq->ops != NULL) {
5567 txq->ops->release_mbufs(txq);
5568 txq->ops->reset(txq);
5570 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
5576 ixgbe_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
5577 struct rte_eth_rxq_info *qinfo)
5579 struct ixgbe_rx_queue *rxq;
5581 rxq = dev->data->rx_queues[queue_id];
5583 qinfo->mp = rxq->mb_pool;
5584 qinfo->scattered_rx = dev->data->scattered_rx;
5585 qinfo->nb_desc = rxq->nb_rx_desc;
5587 qinfo->conf.rx_free_thresh = rxq->rx_free_thresh;
5588 qinfo->conf.rx_drop_en = rxq->drop_en;
5589 qinfo->conf.rx_deferred_start = rxq->rx_deferred_start;
5590 qinfo->conf.offloads = rxq->offloads;
5594 ixgbe_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
5595 struct rte_eth_txq_info *qinfo)
5597 struct ixgbe_tx_queue *txq;
5599 txq = dev->data->tx_queues[queue_id];
5601 qinfo->nb_desc = txq->nb_tx_desc;
5603 qinfo->conf.tx_thresh.pthresh = txq->pthresh;
5604 qinfo->conf.tx_thresh.hthresh = txq->hthresh;
5605 qinfo->conf.tx_thresh.wthresh = txq->wthresh;
5607 qinfo->conf.tx_free_thresh = txq->tx_free_thresh;
5608 qinfo->conf.tx_rs_thresh = txq->tx_rs_thresh;
5609 qinfo->conf.offloads = txq->offloads;
5610 qinfo->conf.tx_deferred_start = txq->tx_deferred_start;
5614 * [VF] Initializes Receive Unit.
5617 ixgbevf_dev_rx_init(struct rte_eth_dev *dev)
5619 struct ixgbe_hw *hw;
5620 struct ixgbe_rx_queue *rxq;
5621 struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
5623 uint32_t srrctl, psrtype = 0;
5628 PMD_INIT_FUNC_TRACE();
5629 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5631 if (rte_is_power_of_2(dev->data->nb_rx_queues) == 0) {
5632 PMD_INIT_LOG(ERR, "The number of Rx queue invalid, "
5633 "it should be power of 2");
5637 if (dev->data->nb_rx_queues > hw->mac.max_rx_queues) {
5638 PMD_INIT_LOG(ERR, "The number of Rx queue invalid, "
5639 "it should be equal to or less than %d",
5640 hw->mac.max_rx_queues);
5645 * When the VF driver issues a IXGBE_VF_RESET request, the PF driver
5646 * disables the VF receipt of packets if the PF MTU is > 1500.
5647 * This is done to deal with 82599 limitations that imposes
5648 * the PF and all VFs to share the same MTU.
5649 * Then, the PF driver enables again the VF receipt of packet when
5650 * the VF driver issues a IXGBE_VF_SET_LPE request.
5651 * In the meantime, the VF device cannot be used, even if the VF driver
5652 * and the Guest VM network stack are ready to accept packets with a
5653 * size up to the PF MTU.
5654 * As a work-around to this PF behaviour, force the call to
5655 * ixgbevf_rlpml_set_vf even if jumbo frames are not used. This way,
5656 * VF packets received can work in all cases.
5658 if (ixgbevf_rlpml_set_vf(hw,
5659 (uint16_t)dev->data->dev_conf.rxmode.max_rx_pkt_len)) {
5660 PMD_INIT_LOG(ERR, "Set max packet length to %d failed.",
5661 dev->data->dev_conf.rxmode.max_rx_pkt_len);
5666 * Assume no header split and no VLAN strip support
5667 * on any Rx queue first .
5669 rxmode->offloads &= ~DEV_RX_OFFLOAD_VLAN_STRIP;
5670 /* Setup RX queues */
5671 for (i = 0; i < dev->data->nb_rx_queues; i++) {
5672 rxq = dev->data->rx_queues[i];
5674 /* Allocate buffers for descriptor rings */
5675 ret = ixgbe_alloc_rx_queue_mbufs(rxq);
5679 /* Setup the Base and Length of the Rx Descriptor Rings */
5680 bus_addr = rxq->rx_ring_phys_addr;
5682 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(i),
5683 (uint32_t)(bus_addr & 0x00000000ffffffffULL));
5684 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(i),
5685 (uint32_t)(bus_addr >> 32));
5686 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(i),
5687 rxq->nb_rx_desc * sizeof(union ixgbe_adv_rx_desc));
5688 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(i), 0);
5689 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(i), 0);
5692 /* Configure the SRRCTL register */
5693 srrctl = IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
5695 /* Set if packets are dropped when no descriptors available */
5697 srrctl |= IXGBE_SRRCTL_DROP_EN;
5700 * Configure the RX buffer size in the BSIZEPACKET field of
5701 * the SRRCTL register of the queue.
5702 * The value is in 1 KB resolution. Valid values can be from
5705 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) -
5706 RTE_PKTMBUF_HEADROOM);
5707 srrctl |= ((buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) &
5708 IXGBE_SRRCTL_BSIZEPKT_MASK);
5711 * VF modification to write virtual function SRRCTL register
5713 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(i), srrctl);
5715 buf_size = (uint16_t) ((srrctl & IXGBE_SRRCTL_BSIZEPKT_MASK) <<
5716 IXGBE_SRRCTL_BSIZEPKT_SHIFT);
5718 if (rxmode->offloads & DEV_RX_OFFLOAD_SCATTER ||
5719 /* It adds dual VLAN length for supporting dual VLAN */
5720 (rxmode->max_rx_pkt_len +
5721 2 * IXGBE_VLAN_TAG_SIZE) > buf_size) {
5722 if (!dev->data->scattered_rx)
5723 PMD_INIT_LOG(DEBUG, "forcing scatter mode");
5724 dev->data->scattered_rx = 1;
5727 if (rxq->offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
5728 rxmode->offloads |= DEV_RX_OFFLOAD_VLAN_STRIP;
5731 /* Set RQPL for VF RSS according to max Rx queue */
5732 psrtype |= (dev->data->nb_rx_queues >> 1) <<
5733 IXGBE_PSRTYPE_RQPL_SHIFT;
5734 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype);
5736 ixgbe_set_rx_function(dev);
5742 * [VF] Initializes Transmit Unit.
5745 ixgbevf_dev_tx_init(struct rte_eth_dev *dev)
5747 struct ixgbe_hw *hw;
5748 struct ixgbe_tx_queue *txq;
5753 PMD_INIT_FUNC_TRACE();
5754 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5756 /* Setup the Base and Length of the Tx Descriptor Rings */
5757 for (i = 0; i < dev->data->nb_tx_queues; i++) {
5758 txq = dev->data->tx_queues[i];
5759 bus_addr = txq->tx_ring_phys_addr;
5760 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(i),
5761 (uint32_t)(bus_addr & 0x00000000ffffffffULL));
5762 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(i),
5763 (uint32_t)(bus_addr >> 32));
5764 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(i),
5765 txq->nb_tx_desc * sizeof(union ixgbe_adv_tx_desc));
5766 /* Setup the HW Tx Head and TX Tail descriptor pointers */
5767 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(i), 0);
5768 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(i), 0);
5771 * Disable Tx Head Writeback RO bit, since this hoses
5772 * bookkeeping if things aren't delivered in order.
5774 txctrl = IXGBE_READ_REG(hw,
5775 IXGBE_VFDCA_TXCTRL(i));
5776 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
5777 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(i),
5783 * [VF] Start Transmit and Receive Units.
5786 ixgbevf_dev_rxtx_start(struct rte_eth_dev *dev)
5788 struct ixgbe_hw *hw;
5789 struct ixgbe_tx_queue *txq;
5790 struct ixgbe_rx_queue *rxq;
5796 PMD_INIT_FUNC_TRACE();
5797 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5799 for (i = 0; i < dev->data->nb_tx_queues; i++) {
5800 txq = dev->data->tx_queues[i];
5801 /* Setup Transmit Threshold Registers */
5802 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i));
5803 txdctl |= txq->pthresh & 0x7F;
5804 txdctl |= ((txq->hthresh & 0x7F) << 8);
5805 txdctl |= ((txq->wthresh & 0x7F) << 16);
5806 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl);
5809 for (i = 0; i < dev->data->nb_tx_queues; i++) {
5811 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i));
5812 txdctl |= IXGBE_TXDCTL_ENABLE;
5813 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl);
5816 /* Wait until TX Enable ready */
5819 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i));
5820 } while (--poll_ms && !(txdctl & IXGBE_TXDCTL_ENABLE));
5822 PMD_INIT_LOG(ERR, "Could not enable Tx Queue %d", i);
5824 for (i = 0; i < dev->data->nb_rx_queues; i++) {
5826 rxq = dev->data->rx_queues[i];
5828 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i));
5829 rxdctl |= IXGBE_RXDCTL_ENABLE;
5830 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), rxdctl);
5832 /* Wait until RX Enable ready */
5836 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i));
5837 } while (--poll_ms && !(rxdctl & IXGBE_RXDCTL_ENABLE));
5839 PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", i);
5841 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(i), rxq->nb_rx_desc - 1);
5847 ixgbe_rss_conf_init(struct ixgbe_rte_flow_rss_conf *out,
5848 const struct rte_flow_action_rss *in)
5850 if (in->key_len > RTE_DIM(out->key) ||
5851 in->queue_num > RTE_DIM(out->queue))
5853 out->conf = (struct rte_flow_action_rss){
5857 .key_len = in->key_len,
5858 .queue_num = in->queue_num,
5859 .key = memcpy(out->key, in->key, in->key_len),
5860 .queue = memcpy(out->queue, in->queue,
5861 sizeof(*in->queue) * in->queue_num),
5867 ixgbe_action_rss_same(const struct rte_flow_action_rss *comp,
5868 const struct rte_flow_action_rss *with)
5870 return (comp->func == with->func &&
5871 comp->level == with->level &&
5872 comp->types == with->types &&
5873 comp->key_len == with->key_len &&
5874 comp->queue_num == with->queue_num &&
5875 !memcmp(comp->key, with->key, with->key_len) &&
5876 !memcmp(comp->queue, with->queue,
5877 sizeof(*with->queue) * with->queue_num));
5881 ixgbe_config_rss_filter(struct rte_eth_dev *dev,
5882 struct ixgbe_rte_flow_rss_conf *conf, bool add)
5884 struct ixgbe_hw *hw;
5888 uint16_t sp_reta_size;
5890 struct rte_eth_rss_conf rss_conf = {
5891 .rss_key = conf->conf.key_len ?
5892 (void *)(uintptr_t)conf->conf.key : NULL,
5893 .rss_key_len = conf->conf.key_len,
5894 .rss_hf = conf->conf.types,
5896 struct ixgbe_filter_info *filter_info =
5897 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
5899 PMD_INIT_FUNC_TRACE();
5900 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5902 sp_reta_size = ixgbe_reta_size_get(hw->mac.type);
5905 if (ixgbe_action_rss_same(&filter_info->rss_info.conf,
5907 ixgbe_rss_disable(dev);
5908 memset(&filter_info->rss_info, 0,
5909 sizeof(struct ixgbe_rte_flow_rss_conf));
5915 if (filter_info->rss_info.conf.queue_num)
5917 /* Fill in redirection table
5918 * The byte-swap is needed because NIC registers are in
5919 * little-endian order.
5922 for (i = 0, j = 0; i < sp_reta_size; i++, j++) {
5923 reta_reg = ixgbe_reta_reg_get(hw->mac.type, i);
5925 if (j == conf->conf.queue_num)
5927 reta = (reta << 8) | conf->conf.queue[j];
5929 IXGBE_WRITE_REG(hw, reta_reg,
5933 /* Configure the RSS key and the RSS protocols used to compute
5934 * the RSS hash of input packets.
5936 if ((rss_conf.rss_hf & IXGBE_RSS_OFFLOAD_ALL) == 0) {
5937 ixgbe_rss_disable(dev);
5940 if (rss_conf.rss_key == NULL)
5941 rss_conf.rss_key = rss_intel_key; /* Default hash key */
5942 ixgbe_hw_rss_hash_set(hw, &rss_conf);
5944 if (ixgbe_rss_conf_init(&filter_info->rss_info, &conf->conf))
5950 /* Stubs needed for linkage when RTE_ARCH_PPC_64 is set */
5951 #if defined(RTE_ARCH_PPC_64)
5953 ixgbe_rx_vec_dev_conf_condition_check(struct rte_eth_dev __rte_unused *dev)
5959 ixgbe_recv_pkts_vec(
5960 void __rte_unused *rx_queue,
5961 struct rte_mbuf __rte_unused **rx_pkts,
5962 uint16_t __rte_unused nb_pkts)
5968 ixgbe_recv_scattered_pkts_vec(
5969 void __rte_unused *rx_queue,
5970 struct rte_mbuf __rte_unused **rx_pkts,
5971 uint16_t __rte_unused nb_pkts)
5977 ixgbe_rxq_vec_setup(struct ixgbe_rx_queue __rte_unused *rxq)
5983 ixgbe_xmit_fixed_burst_vec(void __rte_unused *tx_queue,
5984 struct rte_mbuf __rte_unused **tx_pkts,
5985 uint16_t __rte_unused nb_pkts)
5991 ixgbe_txq_vec_setup(struct ixgbe_tx_queue __rte_unused *txq)
5997 ixgbe_rx_queue_release_mbufs_vec(struct ixgbe_rx_queue __rte_unused *rxq)