4 * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
5 * Copyright 2014 6WIND S.A.
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
12 * * Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * * Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in
16 * the documentation and/or other materials provided with the
18 * * Neither the name of Intel Corporation nor the names of its
19 * contributors may be used to endorse or promote products derived
20 * from this software without specific prior written permission.
22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
25 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
26 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
28 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 #include <sys/queue.h>
46 #include <rte_byteorder.h>
47 #include <rte_common.h>
48 #include <rte_cycles.h>
50 #include <rte_debug.h>
51 #include <rte_interrupts.h>
53 #include <rte_memory.h>
54 #include <rte_memzone.h>
55 #include <rte_launch.h>
57 #include <rte_per_lcore.h>
58 #include <rte_lcore.h>
59 #include <rte_atomic.h>
60 #include <rte_branch_prediction.h>
62 #include <rte_mempool.h>
63 #include <rte_malloc.h>
65 #include <rte_ether.h>
66 #include <rte_ethdev.h>
67 #include <rte_prefetch.h>
71 #include <rte_string_fns.h>
72 #include <rte_errno.h>
74 #include "ixgbe_logs.h"
75 #include "ixgbe/ixgbe_api.h"
76 #include "ixgbe/ixgbe_vf.h"
77 #include "ixgbe_ethdev.h"
78 #include "ixgbe/ixgbe_dcb.h"
79 #include "ixgbe/ixgbe_common.h"
80 #include "ixgbe_rxtx.h"
82 /* Bit Mask to indicate what bits required for building TX context */
83 #define IXGBE_TX_OFFLOAD_MASK ( \
89 static inline struct rte_mbuf *
90 rte_rxmbuf_alloc(struct rte_mempool *mp)
94 m = __rte_mbuf_raw_alloc(mp);
95 __rte_mbuf_sanity_check_raw(m, 0);
101 #define RTE_PMD_USE_PREFETCH
104 #ifdef RTE_PMD_USE_PREFETCH
106 * Prefetch a cache line into all cache levels.
108 #define rte_ixgbe_prefetch(p) rte_prefetch0(p)
110 #define rte_ixgbe_prefetch(p) do {} while(0)
113 /*********************************************************************
117 **********************************************************************/
120 * Check for descriptors with their DD bit set and free mbufs.
121 * Return the total number of buffers freed.
123 static inline int __attribute__((always_inline))
124 ixgbe_tx_free_bufs(struct ixgbe_tx_queue *txq)
126 struct ixgbe_tx_entry *txep;
130 /* check DD bit on threshold descriptor */
131 status = txq->tx_ring[txq->tx_next_dd].wb.status;
132 if (! (status & IXGBE_ADVTXD_STAT_DD))
136 * first buffer to free from S/W ring is at index
137 * tx_next_dd - (tx_rs_thresh-1)
139 txep = &(txq->sw_ring[txq->tx_next_dd - (txq->tx_rs_thresh - 1)]);
141 /* free buffers one at a time */
142 if ((txq->txq_flags & (uint32_t)ETH_TXQ_FLAGS_NOREFCOUNT) != 0) {
143 for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) {
144 txep->mbuf->next = NULL;
145 rte_mempool_put(txep->mbuf->pool, txep->mbuf);
149 for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) {
150 rte_pktmbuf_free_seg(txep->mbuf);
155 /* buffers were freed, update counters */
156 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh);
157 txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh);
158 if (txq->tx_next_dd >= txq->nb_tx_desc)
159 txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
161 return txq->tx_rs_thresh;
164 /* Populate 4 descriptors with data from 4 mbufs */
166 tx4(volatile union ixgbe_adv_tx_desc *txdp, struct rte_mbuf **pkts)
168 uint64_t buf_dma_addr;
172 for (i = 0; i < 4; ++i, ++txdp, ++pkts) {
173 buf_dma_addr = RTE_MBUF_DATA_DMA_ADDR(*pkts);
174 pkt_len = (*pkts)->data_len;
176 /* write data to descriptor */
177 txdp->read.buffer_addr = buf_dma_addr;
178 txdp->read.cmd_type_len =
179 ((uint32_t)DCMD_DTYP_FLAGS | pkt_len);
180 txdp->read.olinfo_status =
181 (pkt_len << IXGBE_ADVTXD_PAYLEN_SHIFT);
182 rte_prefetch0(&(*pkts)->pool);
186 /* Populate 1 descriptor with data from 1 mbuf */
188 tx1(volatile union ixgbe_adv_tx_desc *txdp, struct rte_mbuf **pkts)
190 uint64_t buf_dma_addr;
193 buf_dma_addr = RTE_MBUF_DATA_DMA_ADDR(*pkts);
194 pkt_len = (*pkts)->data_len;
196 /* write data to descriptor */
197 txdp->read.buffer_addr = buf_dma_addr;
198 txdp->read.cmd_type_len =
199 ((uint32_t)DCMD_DTYP_FLAGS | pkt_len);
200 txdp->read.olinfo_status =
201 (pkt_len << IXGBE_ADVTXD_PAYLEN_SHIFT);
202 rte_prefetch0(&(*pkts)->pool);
206 * Fill H/W descriptor ring with mbuf data.
207 * Copy mbuf pointers to the S/W ring.
210 ixgbe_tx_fill_hw_ring(struct ixgbe_tx_queue *txq, struct rte_mbuf **pkts,
213 volatile union ixgbe_adv_tx_desc *txdp = &(txq->tx_ring[txq->tx_tail]);
214 struct ixgbe_tx_entry *txep = &(txq->sw_ring[txq->tx_tail]);
215 const int N_PER_LOOP = 4;
216 const int N_PER_LOOP_MASK = N_PER_LOOP-1;
217 int mainpart, leftover;
221 * Process most of the packets in chunks of N pkts. Any
222 * leftover packets will get processed one at a time.
224 mainpart = (nb_pkts & ((uint32_t) ~N_PER_LOOP_MASK));
225 leftover = (nb_pkts & ((uint32_t) N_PER_LOOP_MASK));
226 for (i = 0; i < mainpart; i += N_PER_LOOP) {
227 /* Copy N mbuf pointers to the S/W ring */
228 for (j = 0; j < N_PER_LOOP; ++j) {
229 (txep + i + j)->mbuf = *(pkts + i + j);
231 tx4(txdp + i, pkts + i);
234 if (unlikely(leftover > 0)) {
235 for (i = 0; i < leftover; ++i) {
236 (txep + mainpart + i)->mbuf = *(pkts + mainpart + i);
237 tx1(txdp + mainpart + i, pkts + mainpart + i);
242 static inline uint16_t
243 tx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
246 struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue;
247 volatile union ixgbe_adv_tx_desc *tx_r = txq->tx_ring;
251 * Begin scanning the H/W ring for done descriptors when the
252 * number of available descriptors drops below tx_free_thresh. For
253 * each done descriptor, free the associated buffer.
255 if (txq->nb_tx_free < txq->tx_free_thresh)
256 ixgbe_tx_free_bufs(txq);
258 /* Only use descriptors that are available */
259 nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts);
260 if (unlikely(nb_pkts == 0))
263 /* Use exactly nb_pkts descriptors */
264 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts);
267 * At this point, we know there are enough descriptors in the
268 * ring to transmit all the packets. This assumes that each
269 * mbuf contains a single segment, and that no new offloads
270 * are expected, which would require a new context descriptor.
274 * See if we're going to wrap-around. If so, handle the top
275 * of the descriptor ring first, then do the bottom. If not,
276 * the processing looks just like the "bottom" part anyway...
278 if ((txq->tx_tail + nb_pkts) > txq->nb_tx_desc) {
279 n = (uint16_t)(txq->nb_tx_desc - txq->tx_tail);
280 ixgbe_tx_fill_hw_ring(txq, tx_pkts, n);
283 * We know that the last descriptor in the ring will need to
284 * have its RS bit set because tx_rs_thresh has to be
285 * a divisor of the ring size
287 tx_r[txq->tx_next_rs].read.cmd_type_len |=
288 rte_cpu_to_le_32(IXGBE_ADVTXD_DCMD_RS);
289 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
294 /* Fill H/W descriptor ring with mbuf data */
295 ixgbe_tx_fill_hw_ring(txq, tx_pkts + n, (uint16_t)(nb_pkts - n));
296 txq->tx_tail = (uint16_t)(txq->tx_tail + (nb_pkts - n));
299 * Determine if RS bit should be set
300 * This is what we actually want:
301 * if ((txq->tx_tail - 1) >= txq->tx_next_rs)
302 * but instead of subtracting 1 and doing >=, we can just do
303 * greater than without subtracting.
305 if (txq->tx_tail > txq->tx_next_rs) {
306 tx_r[txq->tx_next_rs].read.cmd_type_len |=
307 rte_cpu_to_le_32(IXGBE_ADVTXD_DCMD_RS);
308 txq->tx_next_rs = (uint16_t)(txq->tx_next_rs +
310 if (txq->tx_next_rs >= txq->nb_tx_desc)
311 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
315 * Check for wrap-around. This would only happen if we used
316 * up to the last descriptor in the ring, no more, no less.
318 if (txq->tx_tail >= txq->nb_tx_desc)
321 /* update tail pointer */
323 IXGBE_PCI_REG_WRITE(txq->tdt_reg_addr, txq->tx_tail);
329 ixgbe_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts,
334 /* Try to transmit at least chunks of TX_MAX_BURST pkts */
335 if (likely(nb_pkts <= RTE_PMD_IXGBE_TX_MAX_BURST))
336 return tx_xmit_pkts(tx_queue, tx_pkts, nb_pkts);
338 /* transmit more than the max burst, in chunks of TX_MAX_BURST */
342 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_IXGBE_TX_MAX_BURST);
343 ret = tx_xmit_pkts(tx_queue, &(tx_pkts[nb_tx]), n);
344 nb_tx = (uint16_t)(nb_tx + ret);
345 nb_pkts = (uint16_t)(nb_pkts - ret);
354 ixgbe_set_xmit_ctx(struct ixgbe_tx_queue *txq,
355 volatile struct ixgbe_adv_tx_context_desc *ctx_txd,
356 uint64_t ol_flags, union ixgbe_tx_offload tx_offload)
358 uint32_t type_tucmd_mlhl;
359 uint32_t mss_l4len_idx = 0;
361 uint32_t vlan_macip_lens;
362 union ixgbe_tx_offload tx_offload_mask;
364 ctx_idx = txq->ctx_curr;
365 tx_offload_mask.data = 0;
368 /* Specify which HW CTX to upload. */
369 mss_l4len_idx |= (ctx_idx << IXGBE_ADVTXD_IDX_SHIFT);
371 if (ol_flags & PKT_TX_VLAN_PKT) {
372 tx_offload_mask.vlan_tci |= ~0;
375 /* check if TCP segmentation required for this packet */
376 if (ol_flags & PKT_TX_TCP_SEG) {
377 /* implies IP cksum and TCP cksum */
378 type_tucmd_mlhl = IXGBE_ADVTXD_TUCMD_IPV4 |
379 IXGBE_ADVTXD_TUCMD_L4T_TCP |
380 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
382 tx_offload_mask.l2_len |= ~0;
383 tx_offload_mask.l3_len |= ~0;
384 tx_offload_mask.l4_len |= ~0;
385 tx_offload_mask.tso_segsz |= ~0;
386 mss_l4len_idx |= tx_offload.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT;
387 mss_l4len_idx |= tx_offload.l4_len << IXGBE_ADVTXD_L4LEN_SHIFT;
388 } else { /* no TSO, check if hardware checksum is needed */
389 if (ol_flags & PKT_TX_IP_CKSUM) {
390 type_tucmd_mlhl = IXGBE_ADVTXD_TUCMD_IPV4;
391 tx_offload_mask.l2_len |= ~0;
392 tx_offload_mask.l3_len |= ~0;
395 switch (ol_flags & PKT_TX_L4_MASK) {
396 case PKT_TX_UDP_CKSUM:
397 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP |
398 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
399 mss_l4len_idx |= sizeof(struct udp_hdr) << IXGBE_ADVTXD_L4LEN_SHIFT;
400 tx_offload_mask.l2_len |= ~0;
401 tx_offload_mask.l3_len |= ~0;
403 case PKT_TX_TCP_CKSUM:
404 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP |
405 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
406 mss_l4len_idx |= sizeof(struct tcp_hdr) << IXGBE_ADVTXD_L4LEN_SHIFT;
407 tx_offload_mask.l2_len |= ~0;
408 tx_offload_mask.l3_len |= ~0;
409 tx_offload_mask.l4_len |= ~0;
411 case PKT_TX_SCTP_CKSUM:
412 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP |
413 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
414 mss_l4len_idx |= sizeof(struct sctp_hdr) << IXGBE_ADVTXD_L4LEN_SHIFT;
415 tx_offload_mask.l2_len |= ~0;
416 tx_offload_mask.l3_len |= ~0;
419 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_RSV |
420 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
425 txq->ctx_cache[ctx_idx].flags = ol_flags;
426 txq->ctx_cache[ctx_idx].tx_offload.data =
427 tx_offload_mask.data & tx_offload.data;
428 txq->ctx_cache[ctx_idx].tx_offload_mask = tx_offload_mask;
430 ctx_txd->type_tucmd_mlhl = rte_cpu_to_le_32(type_tucmd_mlhl);
431 vlan_macip_lens = tx_offload.l3_len;
432 vlan_macip_lens |= (tx_offload.l2_len << IXGBE_ADVTXD_MACLEN_SHIFT);
433 vlan_macip_lens |= ((uint32_t)tx_offload.vlan_tci << IXGBE_ADVTXD_VLAN_SHIFT);
434 ctx_txd->vlan_macip_lens = rte_cpu_to_le_32(vlan_macip_lens);
435 ctx_txd->mss_l4len_idx = rte_cpu_to_le_32(mss_l4len_idx);
436 ctx_txd->seqnum_seed = 0;
440 * Check which hardware context can be used. Use the existing match
441 * or create a new context descriptor.
443 static inline uint32_t
444 what_advctx_update(struct ixgbe_tx_queue *txq, uint64_t flags,
445 union ixgbe_tx_offload tx_offload)
447 /* If match with the current used context */
448 if (likely((txq->ctx_cache[txq->ctx_curr].flags == flags) &&
449 (txq->ctx_cache[txq->ctx_curr].tx_offload.data ==
450 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data & tx_offload.data)))) {
451 return txq->ctx_curr;
454 /* What if match with the next context */
456 if (likely((txq->ctx_cache[txq->ctx_curr].flags == flags) &&
457 (txq->ctx_cache[txq->ctx_curr].tx_offload.data ==
458 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data & tx_offload.data)))) {
459 return txq->ctx_curr;
462 /* Mismatch, use the previous context */
463 return (IXGBE_CTX_NUM);
466 static inline uint32_t
467 tx_desc_cksum_flags_to_olinfo(uint64_t ol_flags)
470 if ((ol_flags & PKT_TX_L4_MASK) != PKT_TX_L4_NO_CKSUM)
471 tmp |= IXGBE_ADVTXD_POPTS_TXSM;
472 if (ol_flags & PKT_TX_IP_CKSUM)
473 tmp |= IXGBE_ADVTXD_POPTS_IXSM;
474 if (ol_flags & PKT_TX_TCP_SEG)
475 tmp |= IXGBE_ADVTXD_POPTS_TXSM;
479 static inline uint32_t
480 tx_desc_ol_flags_to_cmdtype(uint64_t ol_flags)
482 uint32_t cmdtype = 0;
483 if (ol_flags & PKT_TX_VLAN_PKT)
484 cmdtype |= IXGBE_ADVTXD_DCMD_VLE;
485 if (ol_flags & PKT_TX_TCP_SEG)
486 cmdtype |= IXGBE_ADVTXD_DCMD_TSE;
490 /* Default RS bit threshold values */
491 #ifndef DEFAULT_TX_RS_THRESH
492 #define DEFAULT_TX_RS_THRESH 32
494 #ifndef DEFAULT_TX_FREE_THRESH
495 #define DEFAULT_TX_FREE_THRESH 32
498 /* Reset transmit descriptors after they have been used */
500 ixgbe_xmit_cleanup(struct ixgbe_tx_queue *txq)
502 struct ixgbe_tx_entry *sw_ring = txq->sw_ring;
503 volatile union ixgbe_adv_tx_desc *txr = txq->tx_ring;
504 uint16_t last_desc_cleaned = txq->last_desc_cleaned;
505 uint16_t nb_tx_desc = txq->nb_tx_desc;
506 uint16_t desc_to_clean_to;
507 uint16_t nb_tx_to_clean;
509 /* Determine the last descriptor needing to be cleaned */
510 desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->tx_rs_thresh);
511 if (desc_to_clean_to >= nb_tx_desc)
512 desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc);
514 /* Check to make sure the last descriptor to clean is done */
515 desc_to_clean_to = sw_ring[desc_to_clean_to].last_id;
516 if (! (txr[desc_to_clean_to].wb.status & IXGBE_TXD_STAT_DD))
518 PMD_TX_FREE_LOG(DEBUG,
519 "TX descriptor %4u is not done"
520 "(port=%d queue=%d)",
522 txq->port_id, txq->queue_id);
523 /* Failed to clean any descriptors, better luck next time */
527 /* Figure out how many descriptors will be cleaned */
528 if (last_desc_cleaned > desc_to_clean_to)
529 nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) +
532 nb_tx_to_clean = (uint16_t)(desc_to_clean_to -
535 PMD_TX_FREE_LOG(DEBUG,
536 "Cleaning %4u TX descriptors: %4u to %4u "
537 "(port=%d queue=%d)",
538 nb_tx_to_clean, last_desc_cleaned, desc_to_clean_to,
539 txq->port_id, txq->queue_id);
542 * The last descriptor to clean is done, so that means all the
543 * descriptors from the last descriptor that was cleaned
544 * up to the last descriptor with the RS bit set
545 * are done. Only reset the threshold descriptor.
547 txr[desc_to_clean_to].wb.status = 0;
549 /* Update the txq to reflect the last descriptor that was cleaned */
550 txq->last_desc_cleaned = desc_to_clean_to;
551 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + nb_tx_to_clean);
558 ixgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
561 struct ixgbe_tx_queue *txq;
562 struct ixgbe_tx_entry *sw_ring;
563 struct ixgbe_tx_entry *txe, *txn;
564 volatile union ixgbe_adv_tx_desc *txr;
565 volatile union ixgbe_adv_tx_desc *txd;
566 struct rte_mbuf *tx_pkt;
567 struct rte_mbuf *m_seg;
568 uint64_t buf_dma_addr;
569 uint32_t olinfo_status;
570 uint32_t cmd_type_len;
581 union ixgbe_tx_offload tx_offload = { .data = 0 };
584 sw_ring = txq->sw_ring;
586 tx_id = txq->tx_tail;
587 txe = &sw_ring[tx_id];
589 /* Determine if the descriptor ring needs to be cleaned. */
590 if ((txq->nb_tx_desc - txq->nb_tx_free) > txq->tx_free_thresh) {
591 ixgbe_xmit_cleanup(txq);
594 rte_prefetch0(&txe->mbuf->pool);
597 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
600 pkt_len = tx_pkt->pkt_len;
603 * Determine how many (if any) context descriptors
604 * are needed for offload functionality.
606 ol_flags = tx_pkt->ol_flags;
608 /* If hardware offload required */
609 tx_ol_req = ol_flags & IXGBE_TX_OFFLOAD_MASK;
611 tx_offload.l2_len = tx_pkt->l2_len;
612 tx_offload.l3_len = tx_pkt->l3_len;
613 tx_offload.l4_len = tx_pkt->l4_len;
614 tx_offload.vlan_tci = tx_pkt->vlan_tci;
615 tx_offload.tso_segsz = tx_pkt->tso_segsz;
617 /* If new context need be built or reuse the exist ctx. */
618 ctx = what_advctx_update(txq, tx_ol_req,
620 /* Only allocate context descriptor if required*/
621 new_ctx = (ctx == IXGBE_CTX_NUM);
626 * Keep track of how many descriptors are used this loop
627 * This will always be the number of segments + the number of
628 * Context descriptors required to transmit the packet
630 nb_used = (uint16_t)(tx_pkt->nb_segs + new_ctx);
633 * The number of descriptors that must be allocated for a
634 * packet is the number of segments of that packet, plus 1
635 * Context Descriptor for the hardware offload, if any.
636 * Determine the last TX descriptor to allocate in the TX ring
637 * for the packet, starting from the current position (tx_id)
640 tx_last = (uint16_t) (tx_id + nb_used - 1);
643 if (tx_last >= txq->nb_tx_desc)
644 tx_last = (uint16_t) (tx_last - txq->nb_tx_desc);
646 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u pktlen=%u"
647 " tx_first=%u tx_last=%u",
648 (unsigned) txq->port_id,
649 (unsigned) txq->queue_id,
655 * Make sure there are enough TX descriptors available to
656 * transmit the entire packet.
657 * nb_used better be less than or equal to txq->tx_rs_thresh
659 if (nb_used > txq->nb_tx_free) {
660 PMD_TX_FREE_LOG(DEBUG,
661 "Not enough free TX descriptors "
662 "nb_used=%4u nb_free=%4u "
663 "(port=%d queue=%d)",
664 nb_used, txq->nb_tx_free,
665 txq->port_id, txq->queue_id);
667 if (ixgbe_xmit_cleanup(txq) != 0) {
668 /* Could not clean any descriptors */
674 /* nb_used better be <= txq->tx_rs_thresh */
675 if (unlikely(nb_used > txq->tx_rs_thresh)) {
676 PMD_TX_FREE_LOG(DEBUG,
677 "The number of descriptors needed to "
678 "transmit the packet exceeds the "
679 "RS bit threshold. This will impact "
681 "nb_used=%4u nb_free=%4u "
683 "(port=%d queue=%d)",
684 nb_used, txq->nb_tx_free,
686 txq->port_id, txq->queue_id);
688 * Loop here until there are enough TX
689 * descriptors or until the ring cannot be
692 while (nb_used > txq->nb_tx_free) {
693 if (ixgbe_xmit_cleanup(txq) != 0) {
695 * Could not clean any
707 * By now there are enough free TX descriptors to transmit
712 * Set common flags of all TX Data Descriptors.
714 * The following bits must be set in all Data Descriptors:
715 * - IXGBE_ADVTXD_DTYP_DATA
716 * - IXGBE_ADVTXD_DCMD_DEXT
718 * The following bits must be set in the first Data Descriptor
719 * and are ignored in the other ones:
720 * - IXGBE_ADVTXD_DCMD_IFCS
721 * - IXGBE_ADVTXD_MAC_1588
722 * - IXGBE_ADVTXD_DCMD_VLE
724 * The following bits must only be set in the last Data
726 * - IXGBE_TXD_CMD_EOP
728 * The following bits can be set in any Data Descriptor, but
729 * are only set in the last Data Descriptor:
732 cmd_type_len = IXGBE_ADVTXD_DTYP_DATA |
733 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT;
735 #ifdef RTE_LIBRTE_IEEE1588
736 if (ol_flags & PKT_TX_IEEE1588_TMST)
737 cmd_type_len |= IXGBE_ADVTXD_MAC_1588;
743 if (ol_flags & PKT_TX_TCP_SEG) {
744 /* when TSO is on, paylen in descriptor is the
745 * not the packet len but the tcp payload len */
746 pkt_len -= (tx_offload.l2_len +
747 tx_offload.l3_len + tx_offload.l4_len);
751 * Setup the TX Advanced Context Descriptor if required
754 volatile struct ixgbe_adv_tx_context_desc *
757 ctx_txd = (volatile struct
758 ixgbe_adv_tx_context_desc *)
761 txn = &sw_ring[txe->next_id];
762 rte_prefetch0(&txn->mbuf->pool);
764 if (txe->mbuf != NULL) {
765 rte_pktmbuf_free_seg(txe->mbuf);
769 ixgbe_set_xmit_ctx(txq, ctx_txd, tx_ol_req,
772 txe->last_id = tx_last;
773 tx_id = txe->next_id;
778 * Setup the TX Advanced Data Descriptor,
779 * This path will go through
780 * whatever new/reuse the context descriptor
782 cmd_type_len |= tx_desc_ol_flags_to_cmdtype(ol_flags);
783 olinfo_status |= tx_desc_cksum_flags_to_olinfo(ol_flags);
784 olinfo_status |= ctx << IXGBE_ADVTXD_IDX_SHIFT;
787 olinfo_status |= (pkt_len << IXGBE_ADVTXD_PAYLEN_SHIFT);
792 txn = &sw_ring[txe->next_id];
793 rte_prefetch0(&txn->mbuf->pool);
795 if (txe->mbuf != NULL)
796 rte_pktmbuf_free_seg(txe->mbuf);
800 * Set up Transmit Data Descriptor.
802 slen = m_seg->data_len;
803 buf_dma_addr = RTE_MBUF_DATA_DMA_ADDR(m_seg);
804 txd->read.buffer_addr =
805 rte_cpu_to_le_64(buf_dma_addr);
806 txd->read.cmd_type_len =
807 rte_cpu_to_le_32(cmd_type_len | slen);
808 txd->read.olinfo_status =
809 rte_cpu_to_le_32(olinfo_status);
810 txe->last_id = tx_last;
811 tx_id = txe->next_id;
814 } while (m_seg != NULL);
817 * The last packet data descriptor needs End Of Packet (EOP)
819 cmd_type_len |= IXGBE_TXD_CMD_EOP;
820 txq->nb_tx_used = (uint16_t)(txq->nb_tx_used + nb_used);
821 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_used);
823 /* Set RS bit only on threshold packets' last descriptor */
824 if (txq->nb_tx_used >= txq->tx_rs_thresh) {
825 PMD_TX_FREE_LOG(DEBUG,
826 "Setting RS bit on TXD id="
827 "%4u (port=%d queue=%d)",
828 tx_last, txq->port_id, txq->queue_id);
830 cmd_type_len |= IXGBE_TXD_CMD_RS;
832 /* Update txq RS bit counters */
835 txd->read.cmd_type_len |= rte_cpu_to_le_32(cmd_type_len);
841 * Set the Transmit Descriptor Tail (TDT)
843 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u",
844 (unsigned) txq->port_id, (unsigned) txq->queue_id,
845 (unsigned) tx_id, (unsigned) nb_tx);
846 IXGBE_PCI_REG_WRITE(txq->tdt_reg_addr, tx_id);
847 txq->tx_tail = tx_id;
852 /*********************************************************************
856 **********************************************************************/
857 static inline uint64_t
858 rx_desc_hlen_type_rss_to_pkt_flags(uint32_t hl_tp_rs)
862 static const uint64_t ip_pkt_types_map[16] = {
863 0, PKT_RX_IPV4_HDR, PKT_RX_IPV4_HDR_EXT, PKT_RX_IPV4_HDR_EXT,
864 PKT_RX_IPV6_HDR, 0, 0, 0,
865 PKT_RX_IPV6_HDR_EXT, 0, 0, 0,
866 PKT_RX_IPV6_HDR_EXT, 0, 0, 0,
869 static const uint64_t ip_rss_types_map[16] = {
870 0, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH,
871 0, PKT_RX_RSS_HASH, 0, PKT_RX_RSS_HASH,
872 PKT_RX_RSS_HASH, 0, 0, 0,
873 0, 0, 0, PKT_RX_FDIR,
876 #ifdef RTE_LIBRTE_IEEE1588
877 static uint64_t ip_pkt_etqf_map[8] = {
878 0, 0, 0, PKT_RX_IEEE1588_PTP,
882 pkt_flags = (hl_tp_rs & IXGBE_RXDADV_PKTTYPE_ETQF) ?
883 ip_pkt_etqf_map[(hl_tp_rs >> 4) & 0x07] :
884 ip_pkt_types_map[(hl_tp_rs >> 4) & 0x0F];
886 pkt_flags = (hl_tp_rs & IXGBE_RXDADV_PKTTYPE_ETQF) ? 0 :
887 ip_pkt_types_map[(hl_tp_rs >> 4) & 0x0F];
890 return pkt_flags | ip_rss_types_map[hl_tp_rs & 0xF];
893 static inline uint64_t
894 rx_desc_status_to_pkt_flags(uint32_t rx_status)
899 * Check if VLAN present only.
900 * Do not check whether L3/L4 rx checksum done by NIC or not,
901 * That can be found from rte_eth_rxmode.hw_ip_checksum flag
903 pkt_flags = (rx_status & IXGBE_RXD_STAT_VP) ? PKT_RX_VLAN_PKT : 0;
905 #ifdef RTE_LIBRTE_IEEE1588
906 if (rx_status & IXGBE_RXD_STAT_TMST)
907 pkt_flags = pkt_flags | PKT_RX_IEEE1588_TMST;
912 static inline uint64_t
913 rx_desc_error_to_pkt_flags(uint32_t rx_status)
916 * Bit 31: IPE, IPv4 checksum error
917 * Bit 30: L4I, L4I integrity error
919 static uint64_t error_to_pkt_flags_map[4] = {
920 0, PKT_RX_L4_CKSUM_BAD, PKT_RX_IP_CKSUM_BAD,
921 PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD
923 return error_to_pkt_flags_map[(rx_status >>
924 IXGBE_RXDADV_ERR_CKSUM_BIT) & IXGBE_RXDADV_ERR_CKSUM_MSK];
927 #ifdef RTE_LIBRTE_IXGBE_RX_ALLOW_BULK_ALLOC
929 * LOOK_AHEAD defines how many desc statuses to check beyond the
930 * current descriptor.
931 * It must be a pound define for optimal performance.
932 * Do not change the value of LOOK_AHEAD, as the ixgbe_rx_scan_hw_ring
933 * function only works with LOOK_AHEAD=8.
936 #if (LOOK_AHEAD != 8)
937 #error "PMD IXGBE: LOOK_AHEAD must be 8\n"
940 ixgbe_rx_scan_hw_ring(struct ixgbe_rx_queue *rxq)
942 volatile union ixgbe_adv_rx_desc *rxdp;
943 struct ixgbe_rx_entry *rxep;
947 int s[LOOK_AHEAD], nb_dd;
951 /* get references to current descriptor and S/W ring entry */
952 rxdp = &rxq->rx_ring[rxq->rx_tail];
953 rxep = &rxq->sw_ring[rxq->rx_tail];
955 /* check to make sure there is at least 1 packet to receive */
956 if (! (rxdp->wb.upper.status_error & IXGBE_RXDADV_STAT_DD))
960 * Scan LOOK_AHEAD descriptors at a time to determine which descriptors
961 * reference packets that are ready to be received.
963 for (i = 0; i < RTE_PMD_IXGBE_RX_MAX_BURST;
964 i += LOOK_AHEAD, rxdp += LOOK_AHEAD, rxep += LOOK_AHEAD)
966 /* Read desc statuses backwards to avoid race condition */
967 for (j = LOOK_AHEAD-1; j >= 0; --j)
968 s[j] = rxdp[j].wb.upper.status_error;
970 /* Compute how many status bits were set */
972 for (j = 0; j < LOOK_AHEAD; ++j)
973 nb_dd += s[j] & IXGBE_RXDADV_STAT_DD;
977 /* Translate descriptor info to mbuf format */
978 for (j = 0; j < nb_dd; ++j) {
980 pkt_len = (uint16_t)(rxdp[j].wb.upper.length - rxq->crc_len);
981 mb->data_len = pkt_len;
982 mb->pkt_len = pkt_len;
983 mb->vlan_tci = rxdp[j].wb.upper.vlan;
984 mb->vlan_tci = rte_le_to_cpu_16(rxdp[j].wb.upper.vlan);
986 /* convert descriptor fields to rte mbuf flags */
987 pkt_flags = rx_desc_hlen_type_rss_to_pkt_flags(
988 rxdp[j].wb.lower.lo_dword.data);
989 /* reuse status field from scan list */
990 pkt_flags |= rx_desc_status_to_pkt_flags(s[j]);
991 pkt_flags |= rx_desc_error_to_pkt_flags(s[j]);
992 mb->ol_flags = pkt_flags;
994 if (likely(pkt_flags & PKT_RX_RSS_HASH))
995 mb->hash.rss = rxdp[j].wb.lower.hi_dword.rss;
996 else if (pkt_flags & PKT_RX_FDIR) {
998 (uint16_t)((rxdp[j].wb.lower.hi_dword.csum_ip.csum)
999 & IXGBE_ATR_HASH_MASK);
1000 mb->hash.fdir.id = rxdp[j].wb.lower.hi_dword.csum_ip.ip_id;
1004 /* Move mbuf pointers from the S/W ring to the stage */
1005 for (j = 0; j < LOOK_AHEAD; ++j) {
1006 rxq->rx_stage[i + j] = rxep[j].mbuf;
1009 /* stop if all requested packets could not be received */
1010 if (nb_dd != LOOK_AHEAD)
1014 /* clear software ring entries so we can cleanup correctly */
1015 for (i = 0; i < nb_rx; ++i) {
1016 rxq->sw_ring[rxq->rx_tail + i].mbuf = NULL;
1024 ixgbe_rx_alloc_bufs(struct ixgbe_rx_queue *rxq)
1026 volatile union ixgbe_adv_rx_desc *rxdp;
1027 struct ixgbe_rx_entry *rxep;
1028 struct rte_mbuf *mb;
1033 /* allocate buffers in bulk directly into the S/W ring */
1034 alloc_idx = rxq->rx_free_trigger - (rxq->rx_free_thresh - 1);
1035 rxep = &rxq->sw_ring[alloc_idx];
1036 diag = rte_mempool_get_bulk(rxq->mb_pool, (void *)rxep,
1037 rxq->rx_free_thresh);
1038 if (unlikely(diag != 0))
1041 rxdp = &rxq->rx_ring[alloc_idx];
1042 for (i = 0; i < rxq->rx_free_thresh; ++i) {
1043 /* populate the static rte mbuf fields */
1045 rte_mbuf_refcnt_set(mb, 1);
1047 mb->data_off = RTE_PKTMBUF_HEADROOM;
1049 mb->port = rxq->port_id;
1051 /* populate the descriptors */
1052 dma_addr = rte_cpu_to_le_64(RTE_MBUF_DATA_DMA_ADDR_DEFAULT(mb));
1053 rxdp[i].read.hdr_addr = dma_addr;
1054 rxdp[i].read.pkt_addr = dma_addr;
1057 /* update tail pointer */
1059 IXGBE_PCI_REG_WRITE(rxq->rdt_reg_addr, rxq->rx_free_trigger);
1061 /* update state of internal queue structure */
1062 rxq->rx_free_trigger = rxq->rx_free_trigger + rxq->rx_free_thresh;
1063 if (rxq->rx_free_trigger >= rxq->nb_rx_desc)
1064 rxq->rx_free_trigger = rxq->rx_free_thresh - 1;
1070 static inline uint16_t
1071 ixgbe_rx_fill_from_stage(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
1074 struct rte_mbuf **stage = &rxq->rx_stage[rxq->rx_next_avail];
1077 /* how many packets are ready to return? */
1078 nb_pkts = (uint16_t)RTE_MIN(nb_pkts, rxq->rx_nb_avail);
1080 /* copy mbuf pointers to the application's packet list */
1081 for (i = 0; i < nb_pkts; ++i)
1082 rx_pkts[i] = stage[i];
1084 /* update internal queue state */
1085 rxq->rx_nb_avail = (uint16_t)(rxq->rx_nb_avail - nb_pkts);
1086 rxq->rx_next_avail = (uint16_t)(rxq->rx_next_avail + nb_pkts);
1091 static inline uint16_t
1092 rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
1095 struct ixgbe_rx_queue *rxq = (struct ixgbe_rx_queue *)rx_queue;
1098 /* Any previously recv'd pkts will be returned from the Rx stage */
1099 if (rxq->rx_nb_avail)
1100 return ixgbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
1102 /* Scan the H/W ring for packets to receive */
1103 nb_rx = (uint16_t)ixgbe_rx_scan_hw_ring(rxq);
1105 /* update internal queue state */
1106 rxq->rx_next_avail = 0;
1107 rxq->rx_nb_avail = nb_rx;
1108 rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_rx);
1110 /* if required, allocate new buffers to replenish descriptors */
1111 if (rxq->rx_tail > rxq->rx_free_trigger) {
1112 if (ixgbe_rx_alloc_bufs(rxq) != 0) {
1114 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u "
1115 "queue_id=%u", (unsigned) rxq->port_id,
1116 (unsigned) rxq->queue_id);
1118 rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed +=
1119 rxq->rx_free_thresh;
1122 * Need to rewind any previous receives if we cannot
1123 * allocate new buffers to replenish the old ones.
1125 rxq->rx_nb_avail = 0;
1126 rxq->rx_tail = (uint16_t)(rxq->rx_tail - nb_rx);
1127 for (i = 0, j = rxq->rx_tail; i < nb_rx; ++i, ++j)
1128 rxq->sw_ring[j].mbuf = rxq->rx_stage[i];
1134 if (rxq->rx_tail >= rxq->nb_rx_desc)
1137 /* received any packets this loop? */
1138 if (rxq->rx_nb_avail)
1139 return ixgbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
1144 /* split requests into chunks of size RTE_PMD_IXGBE_RX_MAX_BURST */
1146 ixgbe_recv_pkts_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
1151 if (unlikely(nb_pkts == 0))
1154 if (likely(nb_pkts <= RTE_PMD_IXGBE_RX_MAX_BURST))
1155 return rx_recv_pkts(rx_queue, rx_pkts, nb_pkts);
1157 /* request is relatively large, chunk it up */
1161 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_IXGBE_RX_MAX_BURST);
1162 ret = rx_recv_pkts(rx_queue, &rx_pkts[nb_rx], n);
1163 nb_rx = (uint16_t)(nb_rx + ret);
1164 nb_pkts = (uint16_t)(nb_pkts - ret);
1174 /* Stub to avoid extra ifdefs */
1176 ixgbe_recv_pkts_bulk_alloc(__rte_unused void *rx_queue,
1177 __rte_unused struct rte_mbuf **rx_pkts, __rte_unused uint16_t nb_pkts)
1182 #endif /* RTE_LIBRTE_IXGBE_RX_ALLOW_BULK_ALLOC */
1185 ixgbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
1188 struct ixgbe_rx_queue *rxq;
1189 volatile union ixgbe_adv_rx_desc *rx_ring;
1190 volatile union ixgbe_adv_rx_desc *rxdp;
1191 struct ixgbe_rx_entry *sw_ring;
1192 struct ixgbe_rx_entry *rxe;
1193 struct rte_mbuf *rxm;
1194 struct rte_mbuf *nmb;
1195 union ixgbe_adv_rx_desc rxd;
1198 uint32_t hlen_type_rss;
1208 rx_id = rxq->rx_tail;
1209 rx_ring = rxq->rx_ring;
1210 sw_ring = rxq->sw_ring;
1211 while (nb_rx < nb_pkts) {
1213 * The order of operations here is important as the DD status
1214 * bit must not be read after any other descriptor fields.
1215 * rx_ring and rxdp are pointing to volatile data so the order
1216 * of accesses cannot be reordered by the compiler. If they were
1217 * not volatile, they could be reordered which could lead to
1218 * using invalid descriptor fields when read from rxd.
1220 rxdp = &rx_ring[rx_id];
1221 staterr = rxdp->wb.upper.status_error;
1222 if (! (staterr & rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD)))
1229 * If the IXGBE_RXDADV_STAT_EOP flag is not set, the RX packet
1230 * is likely to be invalid and to be dropped by the various
1231 * validation checks performed by the network stack.
1233 * Allocate a new mbuf to replenish the RX ring descriptor.
1234 * If the allocation fails:
1235 * - arrange for that RX descriptor to be the first one
1236 * being parsed the next time the receive function is
1237 * invoked [on the same queue].
1239 * - Stop parsing the RX ring and return immediately.
1241 * This policy do not drop the packet received in the RX
1242 * descriptor for which the allocation of a new mbuf failed.
1243 * Thus, it allows that packet to be later retrieved if
1244 * mbuf have been freed in the mean time.
1245 * As a side effect, holding RX descriptors instead of
1246 * systematically giving them back to the NIC may lead to
1247 * RX ring exhaustion situations.
1248 * However, the NIC can gracefully prevent such situations
1249 * to happen by sending specific "back-pressure" flow control
1250 * frames to its peer(s).
1252 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u "
1253 "ext_err_stat=0x%08x pkt_len=%u",
1254 (unsigned) rxq->port_id, (unsigned) rxq->queue_id,
1255 (unsigned) rx_id, (unsigned) staterr,
1256 (unsigned) rte_le_to_cpu_16(rxd.wb.upper.length));
1258 nmb = rte_rxmbuf_alloc(rxq->mb_pool);
1260 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u "
1261 "queue_id=%u", (unsigned) rxq->port_id,
1262 (unsigned) rxq->queue_id);
1263 rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed++;
1268 rxe = &sw_ring[rx_id];
1270 if (rx_id == rxq->nb_rx_desc)
1273 /* Prefetch next mbuf while processing current one. */
1274 rte_ixgbe_prefetch(sw_ring[rx_id].mbuf);
1277 * When next RX descriptor is on a cache-line boundary,
1278 * prefetch the next 4 RX descriptors and the next 8 pointers
1281 if ((rx_id & 0x3) == 0) {
1282 rte_ixgbe_prefetch(&rx_ring[rx_id]);
1283 rte_ixgbe_prefetch(&sw_ring[rx_id]);
1289 rte_cpu_to_le_64(RTE_MBUF_DATA_DMA_ADDR_DEFAULT(nmb));
1290 rxdp->read.hdr_addr = dma_addr;
1291 rxdp->read.pkt_addr = dma_addr;
1294 * Initialize the returned mbuf.
1295 * 1) setup generic mbuf fields:
1296 * - number of segments,
1299 * - RX port identifier.
1300 * 2) integrate hardware offload data, if any:
1301 * - RSS flag & hash,
1302 * - IP checksum flag,
1303 * - VLAN TCI, if any,
1306 pkt_len = (uint16_t) (rte_le_to_cpu_16(rxd.wb.upper.length) -
1308 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1309 rte_packet_prefetch((char *)rxm->buf_addr + rxm->data_off);
1312 rxm->pkt_len = pkt_len;
1313 rxm->data_len = pkt_len;
1314 rxm->port = rxq->port_id;
1316 hlen_type_rss = rte_le_to_cpu_32(rxd.wb.lower.lo_dword.data);
1317 /* Only valid if PKT_RX_VLAN_PKT set in pkt_flags */
1318 rxm->vlan_tci = rte_le_to_cpu_16(rxd.wb.upper.vlan);
1320 pkt_flags = rx_desc_hlen_type_rss_to_pkt_flags(hlen_type_rss);
1321 pkt_flags = pkt_flags | rx_desc_status_to_pkt_flags(staterr);
1322 pkt_flags = pkt_flags | rx_desc_error_to_pkt_flags(staterr);
1323 rxm->ol_flags = pkt_flags;
1325 if (likely(pkt_flags & PKT_RX_RSS_HASH))
1326 rxm->hash.rss = rxd.wb.lower.hi_dword.rss;
1327 else if (pkt_flags & PKT_RX_FDIR) {
1328 rxm->hash.fdir.hash =
1329 (uint16_t)((rxd.wb.lower.hi_dword.csum_ip.csum)
1330 & IXGBE_ATR_HASH_MASK);
1331 rxm->hash.fdir.id = rxd.wb.lower.hi_dword.csum_ip.ip_id;
1334 * Store the mbuf address into the next entry of the array
1335 * of returned packets.
1337 rx_pkts[nb_rx++] = rxm;
1339 rxq->rx_tail = rx_id;
1342 * If the number of free RX descriptors is greater than the RX free
1343 * threshold of the queue, advance the Receive Descriptor Tail (RDT)
1345 * Update the RDT with the value of the last processed RX descriptor
1346 * minus 1, to guarantee that the RDT register is never equal to the
1347 * RDH register, which creates a "full" ring situtation from the
1348 * hardware point of view...
1350 nb_hold = (uint16_t) (nb_hold + rxq->nb_rx_hold);
1351 if (nb_hold > rxq->rx_free_thresh) {
1352 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u "
1353 "nb_hold=%u nb_rx=%u",
1354 (unsigned) rxq->port_id, (unsigned) rxq->queue_id,
1355 (unsigned) rx_id, (unsigned) nb_hold,
1357 rx_id = (uint16_t) ((rx_id == 0) ?
1358 (rxq->nb_rx_desc - 1) : (rx_id - 1));
1359 IXGBE_PCI_REG_WRITE(rxq->rdt_reg_addr, rx_id);
1362 rxq->nb_rx_hold = nb_hold;
1367 ixgbe_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
1370 struct ixgbe_rx_queue *rxq;
1371 volatile union ixgbe_adv_rx_desc *rx_ring;
1372 volatile union ixgbe_adv_rx_desc *rxdp;
1373 struct ixgbe_rx_entry *sw_ring;
1374 struct ixgbe_rx_entry *rxe;
1375 struct rte_mbuf *first_seg;
1376 struct rte_mbuf *last_seg;
1377 struct rte_mbuf *rxm;
1378 struct rte_mbuf *nmb;
1379 union ixgbe_adv_rx_desc rxd;
1380 uint64_t dma; /* Physical address of mbuf data buffer */
1382 uint32_t hlen_type_rss;
1392 rx_id = rxq->rx_tail;
1393 rx_ring = rxq->rx_ring;
1394 sw_ring = rxq->sw_ring;
1397 * Retrieve RX context of current packet, if any.
1399 first_seg = rxq->pkt_first_seg;
1400 last_seg = rxq->pkt_last_seg;
1402 while (nb_rx < nb_pkts) {
1405 * The order of operations here is important as the DD status
1406 * bit must not be read after any other descriptor fields.
1407 * rx_ring and rxdp are pointing to volatile data so the order
1408 * of accesses cannot be reordered by the compiler. If they were
1409 * not volatile, they could be reordered which could lead to
1410 * using invalid descriptor fields when read from rxd.
1412 rxdp = &rx_ring[rx_id];
1413 staterr = rxdp->wb.upper.status_error;
1414 if (! (staterr & rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD)))
1421 * Allocate a new mbuf to replenish the RX ring descriptor.
1422 * If the allocation fails:
1423 * - arrange for that RX descriptor to be the first one
1424 * being parsed the next time the receive function is
1425 * invoked [on the same queue].
1427 * - Stop parsing the RX ring and return immediately.
1429 * This policy does not drop the packet received in the RX
1430 * descriptor for which the allocation of a new mbuf failed.
1431 * Thus, it allows that packet to be later retrieved if
1432 * mbuf have been freed in the mean time.
1433 * As a side effect, holding RX descriptors instead of
1434 * systematically giving them back to the NIC may lead to
1435 * RX ring exhaustion situations.
1436 * However, the NIC can gracefully prevent such situations
1437 * to happen by sending specific "back-pressure" flow control
1438 * frames to its peer(s).
1440 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u "
1441 "staterr=0x%x data_len=%u",
1442 (unsigned) rxq->port_id, (unsigned) rxq->queue_id,
1443 (unsigned) rx_id, (unsigned) staterr,
1444 (unsigned) rte_le_to_cpu_16(rxd.wb.upper.length));
1446 nmb = rte_rxmbuf_alloc(rxq->mb_pool);
1448 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u "
1449 "queue_id=%u", (unsigned) rxq->port_id,
1450 (unsigned) rxq->queue_id);
1451 rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed++;
1456 rxe = &sw_ring[rx_id];
1458 if (rx_id == rxq->nb_rx_desc)
1461 /* Prefetch next mbuf while processing current one. */
1462 rte_ixgbe_prefetch(sw_ring[rx_id].mbuf);
1465 * When next RX descriptor is on a cache-line boundary,
1466 * prefetch the next 4 RX descriptors and the next 8 pointers
1469 if ((rx_id & 0x3) == 0) {
1470 rte_ixgbe_prefetch(&rx_ring[rx_id]);
1471 rte_ixgbe_prefetch(&sw_ring[rx_id]);
1475 * Update RX descriptor with the physical address of the new
1476 * data buffer of the new allocated mbuf.
1480 dma = rte_cpu_to_le_64(RTE_MBUF_DATA_DMA_ADDR_DEFAULT(nmb));
1481 rxdp->read.hdr_addr = dma;
1482 rxdp->read.pkt_addr = dma;
1485 * Set data length & data buffer address of mbuf.
1487 data_len = rte_le_to_cpu_16(rxd.wb.upper.length);
1488 rxm->data_len = data_len;
1489 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1492 * If this is the first buffer of the received packet,
1493 * set the pointer to the first mbuf of the packet and
1494 * initialize its context.
1495 * Otherwise, update the total length and the number of segments
1496 * of the current scattered packet, and update the pointer to
1497 * the last mbuf of the current packet.
1499 if (first_seg == NULL) {
1501 first_seg->pkt_len = data_len;
1502 first_seg->nb_segs = 1;
1504 first_seg->pkt_len = (uint16_t)(first_seg->pkt_len
1506 first_seg->nb_segs++;
1507 last_seg->next = rxm;
1511 * If this is not the last buffer of the received packet,
1512 * update the pointer to the last mbuf of the current scattered
1513 * packet and continue to parse the RX ring.
1515 if (! (staterr & IXGBE_RXDADV_STAT_EOP)) {
1521 * This is the last buffer of the received packet.
1522 * If the CRC is not stripped by the hardware:
1523 * - Subtract the CRC length from the total packet length.
1524 * - If the last buffer only contains the whole CRC or a part
1525 * of it, free the mbuf associated to the last buffer.
1526 * If part of the CRC is also contained in the previous
1527 * mbuf, subtract the length of that CRC part from the
1528 * data length of the previous mbuf.
1531 if (unlikely(rxq->crc_len > 0)) {
1532 first_seg->pkt_len -= ETHER_CRC_LEN;
1533 if (data_len <= ETHER_CRC_LEN) {
1534 rte_pktmbuf_free_seg(rxm);
1535 first_seg->nb_segs--;
1536 last_seg->data_len = (uint16_t)
1537 (last_seg->data_len -
1538 (ETHER_CRC_LEN - data_len));
1539 last_seg->next = NULL;
1542 (uint16_t) (data_len - ETHER_CRC_LEN);
1546 * Initialize the first mbuf of the returned packet:
1547 * - RX port identifier,
1548 * - hardware offload data, if any:
1549 * - RSS flag & hash,
1550 * - IP checksum flag,
1551 * - VLAN TCI, if any,
1554 first_seg->port = rxq->port_id;
1557 * The vlan_tci field is only valid when PKT_RX_VLAN_PKT is
1558 * set in the pkt_flags field.
1560 first_seg->vlan_tci = rte_le_to_cpu_16(rxd.wb.upper.vlan);
1561 hlen_type_rss = rte_le_to_cpu_32(rxd.wb.lower.lo_dword.data);
1562 pkt_flags = rx_desc_hlen_type_rss_to_pkt_flags(hlen_type_rss);
1563 pkt_flags = (pkt_flags |
1564 rx_desc_status_to_pkt_flags(staterr));
1565 pkt_flags = (pkt_flags |
1566 rx_desc_error_to_pkt_flags(staterr));
1567 first_seg->ol_flags = pkt_flags;
1569 if (likely(pkt_flags & PKT_RX_RSS_HASH))
1570 first_seg->hash.rss =
1571 rte_le_to_cpu_32(rxd.wb.lower.hi_dword.rss);
1572 else if (pkt_flags & PKT_RX_FDIR) {
1573 first_seg->hash.fdir.hash =
1574 rte_le_to_cpu_16(rxd.wb.lower.hi_dword.csum_ip.csum)
1575 & IXGBE_ATR_HASH_MASK;
1576 first_seg->hash.fdir.id =
1577 rte_le_to_cpu_16(rxd.wb.lower.hi_dword.csum_ip.ip_id);
1580 /* Prefetch data of first segment, if configured to do so. */
1581 rte_packet_prefetch((char *)first_seg->buf_addr +
1582 first_seg->data_off);
1585 * Store the mbuf address into the next entry of the array
1586 * of returned packets.
1588 rx_pkts[nb_rx++] = first_seg;
1591 * Setup receipt context for a new packet.
1597 * Record index of the next RX descriptor to probe.
1599 rxq->rx_tail = rx_id;
1602 * Save receive context.
1604 rxq->pkt_first_seg = first_seg;
1605 rxq->pkt_last_seg = last_seg;
1608 * If the number of free RX descriptors is greater than the RX free
1609 * threshold of the queue, advance the Receive Descriptor Tail (RDT)
1611 * Update the RDT with the value of the last processed RX descriptor
1612 * minus 1, to guarantee that the RDT register is never equal to the
1613 * RDH register, which creates a "full" ring situtation from the
1614 * hardware point of view...
1616 nb_hold = (uint16_t) (nb_hold + rxq->nb_rx_hold);
1617 if (nb_hold > rxq->rx_free_thresh) {
1618 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u "
1619 "nb_hold=%u nb_rx=%u",
1620 (unsigned) rxq->port_id, (unsigned) rxq->queue_id,
1621 (unsigned) rx_id, (unsigned) nb_hold,
1623 rx_id = (uint16_t) ((rx_id == 0) ?
1624 (rxq->nb_rx_desc - 1) : (rx_id - 1));
1625 IXGBE_PCI_REG_WRITE(rxq->rdt_reg_addr, rx_id);
1628 rxq->nb_rx_hold = nb_hold;
1632 /*********************************************************************
1634 * Queue management functions
1636 **********************************************************************/
1639 * Rings setup and release.
1641 * TDBA/RDBA should be aligned on 16 byte boundary. But TDLEN/RDLEN should be
1642 * multiple of 128 bytes. So we align TDBA/RDBA on 128 byte boundary. This will
1643 * also optimize cache line size effect. H/W supports up to cache line size 128.
1645 #define IXGBE_ALIGN 128
1648 * Maximum number of Ring Descriptors.
1650 * Since RDLEN/TDLEN should be multiple of 128 bytes, the number of ring
1651 * descriptors should meet the following condition:
1652 * (num_ring_desc * sizeof(rx/tx descriptor)) % 128 == 0
1654 #define IXGBE_MIN_RING_DESC 32
1655 #define IXGBE_MAX_RING_DESC 4096
1658 * Create memzone for HW rings. malloc can't be used as the physical address is
1659 * needed. If the memzone is already created, then this function returns a ptr
1662 static const struct rte_memzone *
1663 ring_dma_zone_reserve(struct rte_eth_dev *dev, const char *ring_name,
1664 uint16_t queue_id, uint32_t ring_size, int socket_id)
1666 char z_name[RTE_MEMZONE_NAMESIZE];
1667 const struct rte_memzone *mz;
1669 snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d",
1670 dev->driver->pci_drv.name, ring_name,
1671 dev->data->port_id, queue_id);
1673 mz = rte_memzone_lookup(z_name);
1677 #ifdef RTE_LIBRTE_XEN_DOM0
1678 return rte_memzone_reserve_bounded(z_name, ring_size,
1679 socket_id, 0, IXGBE_ALIGN, RTE_PGSIZE_2M);
1681 return rte_memzone_reserve_aligned(z_name, ring_size,
1682 socket_id, 0, IXGBE_ALIGN);
1687 ixgbe_tx_queue_release_mbufs(struct ixgbe_tx_queue *txq)
1691 if (txq->sw_ring != NULL) {
1692 for (i = 0; i < txq->nb_tx_desc; i++) {
1693 if (txq->sw_ring[i].mbuf != NULL) {
1694 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
1695 txq->sw_ring[i].mbuf = NULL;
1702 ixgbe_tx_free_swring(struct ixgbe_tx_queue *txq)
1705 txq->sw_ring != NULL)
1706 rte_free(txq->sw_ring);
1710 ixgbe_tx_queue_release(struct ixgbe_tx_queue *txq)
1712 if (txq != NULL && txq->ops != NULL) {
1713 txq->ops->release_mbufs(txq);
1714 txq->ops->free_swring(txq);
1720 ixgbe_dev_tx_queue_release(void *txq)
1722 ixgbe_tx_queue_release(txq);
1725 /* (Re)set dynamic ixgbe_tx_queue fields to defaults */
1727 ixgbe_reset_tx_queue(struct ixgbe_tx_queue *txq)
1729 static const union ixgbe_adv_tx_desc zeroed_desc = { .read = {
1731 struct ixgbe_tx_entry *txe = txq->sw_ring;
1734 /* Zero out HW ring memory */
1735 for (i = 0; i < txq->nb_tx_desc; i++) {
1736 txq->tx_ring[i] = zeroed_desc;
1739 /* Initialize SW ring entries */
1740 prev = (uint16_t) (txq->nb_tx_desc - 1);
1741 for (i = 0; i < txq->nb_tx_desc; i++) {
1742 volatile union ixgbe_adv_tx_desc *txd = &txq->tx_ring[i];
1743 txd->wb.status = IXGBE_TXD_STAT_DD;
1746 txe[prev].next_id = i;
1750 txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
1751 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
1754 txq->nb_tx_used = 0;
1756 * Always allow 1 descriptor to be un-allocated to avoid
1757 * a H/W race condition
1759 txq->last_desc_cleaned = (uint16_t)(txq->nb_tx_desc - 1);
1760 txq->nb_tx_free = (uint16_t)(txq->nb_tx_desc - 1);
1762 memset((void*)&txq->ctx_cache, 0,
1763 IXGBE_CTX_NUM * sizeof(struct ixgbe_advctx_info));
1766 static const struct ixgbe_txq_ops def_txq_ops = {
1767 .release_mbufs = ixgbe_tx_queue_release_mbufs,
1768 .free_swring = ixgbe_tx_free_swring,
1769 .reset = ixgbe_reset_tx_queue,
1772 /* Takes an ethdev and a queue and sets up the tx function to be used based on
1773 * the queue parameters. Used in tx_queue_setup by primary process and then
1774 * in dev_init by secondary process when attaching to an existing ethdev.
1777 ixgbe_set_tx_function(struct rte_eth_dev *dev, struct ixgbe_tx_queue *txq)
1779 /* Use a simple Tx queue (no offloads, no multi segs) if possible */
1780 if (((txq->txq_flags & IXGBE_SIMPLE_FLAGS) == IXGBE_SIMPLE_FLAGS)
1781 && (txq->tx_rs_thresh >= RTE_PMD_IXGBE_TX_MAX_BURST)) {
1782 PMD_INIT_LOG(INFO, "Using simple tx code path");
1783 #ifdef RTE_IXGBE_INC_VECTOR
1784 if (txq->tx_rs_thresh <= RTE_IXGBE_TX_MAX_FREE_BUF_SZ &&
1785 (rte_eal_process_type() != RTE_PROC_PRIMARY ||
1786 ixgbe_txq_vec_setup(txq) == 0)) {
1787 PMD_INIT_LOG(INFO, "Vector tx enabled.");
1788 dev->tx_pkt_burst = ixgbe_xmit_pkts_vec;
1791 dev->tx_pkt_burst = ixgbe_xmit_pkts_simple;
1793 PMD_INIT_LOG(INFO, "Using full-featured tx code path");
1795 " - txq_flags = %lx " "[IXGBE_SIMPLE_FLAGS=%lx]",
1796 (unsigned long)txq->txq_flags,
1797 (unsigned long)IXGBE_SIMPLE_FLAGS);
1799 " - tx_rs_thresh = %lu " "[RTE_PMD_IXGBE_TX_MAX_BURST=%lu]",
1800 (unsigned long)txq->tx_rs_thresh,
1801 (unsigned long)RTE_PMD_IXGBE_TX_MAX_BURST);
1802 dev->tx_pkt_burst = ixgbe_xmit_pkts;
1807 ixgbe_dev_tx_queue_setup(struct rte_eth_dev *dev,
1810 unsigned int socket_id,
1811 const struct rte_eth_txconf *tx_conf)
1813 const struct rte_memzone *tz;
1814 struct ixgbe_tx_queue *txq;
1815 struct ixgbe_hw *hw;
1816 uint16_t tx_rs_thresh, tx_free_thresh;
1818 PMD_INIT_FUNC_TRACE();
1819 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1822 * Validate number of transmit descriptors.
1823 * It must not exceed hardware maximum, and must be multiple
1826 if (((nb_desc * sizeof(union ixgbe_adv_tx_desc)) % IXGBE_ALIGN) != 0 ||
1827 (nb_desc > IXGBE_MAX_RING_DESC) ||
1828 (nb_desc < IXGBE_MIN_RING_DESC)) {
1833 * The following two parameters control the setting of the RS bit on
1834 * transmit descriptors.
1835 * TX descriptors will have their RS bit set after txq->tx_rs_thresh
1836 * descriptors have been used.
1837 * The TX descriptor ring will be cleaned after txq->tx_free_thresh
1838 * descriptors are used or if the number of descriptors required
1839 * to transmit a packet is greater than the number of free TX
1841 * The following constraints must be satisfied:
1842 * tx_rs_thresh must be greater than 0.
1843 * tx_rs_thresh must be less than the size of the ring minus 2.
1844 * tx_rs_thresh must be less than or equal to tx_free_thresh.
1845 * tx_rs_thresh must be a divisor of the ring size.
1846 * tx_free_thresh must be greater than 0.
1847 * tx_free_thresh must be less than the size of the ring minus 3.
1848 * One descriptor in the TX ring is used as a sentinel to avoid a
1849 * H/W race condition, hence the maximum threshold constraints.
1850 * When set to zero use default values.
1852 tx_rs_thresh = (uint16_t)((tx_conf->tx_rs_thresh) ?
1853 tx_conf->tx_rs_thresh : DEFAULT_TX_RS_THRESH);
1854 tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ?
1855 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH);
1856 if (tx_rs_thresh >= (nb_desc - 2)) {
1857 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than the number "
1858 "of TX descriptors minus 2. (tx_rs_thresh=%u "
1859 "port=%d queue=%d)", (unsigned int)tx_rs_thresh,
1860 (int)dev->data->port_id, (int)queue_idx);
1863 if (tx_free_thresh >= (nb_desc - 3)) {
1864 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than the "
1865 "tx_free_thresh must be less than the number of "
1866 "TX descriptors minus 3. (tx_free_thresh=%u "
1867 "port=%d queue=%d)",
1868 (unsigned int)tx_free_thresh,
1869 (int)dev->data->port_id, (int)queue_idx);
1872 if (tx_rs_thresh > tx_free_thresh) {
1873 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than or equal to "
1874 "tx_free_thresh. (tx_free_thresh=%u "
1875 "tx_rs_thresh=%u port=%d queue=%d)",
1876 (unsigned int)tx_free_thresh,
1877 (unsigned int)tx_rs_thresh,
1878 (int)dev->data->port_id,
1882 if ((nb_desc % tx_rs_thresh) != 0) {
1883 PMD_INIT_LOG(ERR, "tx_rs_thresh must be a divisor of the "
1884 "number of TX descriptors. (tx_rs_thresh=%u "
1885 "port=%d queue=%d)", (unsigned int)tx_rs_thresh,
1886 (int)dev->data->port_id, (int)queue_idx);
1891 * If rs_bit_thresh is greater than 1, then TX WTHRESH should be
1892 * set to 0. If WTHRESH is greater than zero, the RS bit is ignored
1893 * by the NIC and all descriptors are written back after the NIC
1894 * accumulates WTHRESH descriptors.
1896 if ((tx_rs_thresh > 1) && (tx_conf->tx_thresh.wthresh != 0)) {
1897 PMD_INIT_LOG(ERR, "TX WTHRESH must be set to 0 if "
1898 "tx_rs_thresh is greater than 1. (tx_rs_thresh=%u "
1899 "port=%d queue=%d)", (unsigned int)tx_rs_thresh,
1900 (int)dev->data->port_id, (int)queue_idx);
1904 /* Free memory prior to re-allocation if needed... */
1905 if (dev->data->tx_queues[queue_idx] != NULL) {
1906 ixgbe_tx_queue_release(dev->data->tx_queues[queue_idx]);
1907 dev->data->tx_queues[queue_idx] = NULL;
1910 /* First allocate the tx queue data structure */
1911 txq = rte_zmalloc_socket("ethdev TX queue", sizeof(struct ixgbe_tx_queue),
1912 RTE_CACHE_LINE_SIZE, socket_id);
1917 * Allocate TX ring hardware descriptors. A memzone large enough to
1918 * handle the maximum ring size is allocated in order to allow for
1919 * resizing in later calls to the queue setup function.
1921 tz = ring_dma_zone_reserve(dev, "tx_ring", queue_idx,
1922 sizeof(union ixgbe_adv_tx_desc) * IXGBE_MAX_RING_DESC,
1925 ixgbe_tx_queue_release(txq);
1929 txq->nb_tx_desc = nb_desc;
1930 txq->tx_rs_thresh = tx_rs_thresh;
1931 txq->tx_free_thresh = tx_free_thresh;
1932 txq->pthresh = tx_conf->tx_thresh.pthresh;
1933 txq->hthresh = tx_conf->tx_thresh.hthresh;
1934 txq->wthresh = tx_conf->tx_thresh.wthresh;
1935 txq->queue_id = queue_idx;
1936 txq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ?
1937 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx);
1938 txq->port_id = dev->data->port_id;
1939 txq->txq_flags = tx_conf->txq_flags;
1940 txq->ops = &def_txq_ops;
1941 txq->tx_deferred_start = tx_conf->tx_deferred_start;
1944 * Modification to set VFTDT for virtual function if vf is detected
1946 if (hw->mac.type == ixgbe_mac_82599_vf ||
1947 hw->mac.type == ixgbe_mac_X540_vf ||
1948 hw->mac.type == ixgbe_mac_X550_vf ||
1949 hw->mac.type == ixgbe_mac_X550EM_x_vf)
1950 txq->tdt_reg_addr = IXGBE_PCI_REG_ADDR(hw, IXGBE_VFTDT(queue_idx));
1952 txq->tdt_reg_addr = IXGBE_PCI_REG_ADDR(hw, IXGBE_TDT(txq->reg_idx));
1953 #ifndef RTE_LIBRTE_XEN_DOM0
1954 txq->tx_ring_phys_addr = (uint64_t) tz->phys_addr;
1956 txq->tx_ring_phys_addr = rte_mem_phy2mch(tz->memseg_id, tz->phys_addr);
1958 txq->tx_ring = (union ixgbe_adv_tx_desc *) tz->addr;
1960 /* Allocate software ring */
1961 txq->sw_ring = rte_zmalloc_socket("txq->sw_ring",
1962 sizeof(struct ixgbe_tx_entry) * nb_desc,
1963 RTE_CACHE_LINE_SIZE, socket_id);
1964 if (txq->sw_ring == NULL) {
1965 ixgbe_tx_queue_release(txq);
1968 PMD_INIT_LOG(DEBUG, "sw_ring=%p hw_ring=%p dma_addr=0x%"PRIx64,
1969 txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr);
1971 /* set up vector or scalar TX function as appropriate */
1972 ixgbe_set_tx_function(dev, txq);
1974 txq->ops->reset(txq);
1976 dev->data->tx_queues[queue_idx] = txq;
1983 ixgbe_rx_queue_release_mbufs(struct ixgbe_rx_queue *rxq)
1987 if (rxq->sw_ring != NULL) {
1988 for (i = 0; i < rxq->nb_rx_desc; i++) {
1989 if (rxq->sw_ring[i].mbuf != NULL) {
1990 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
1991 rxq->sw_ring[i].mbuf = NULL;
1994 #ifdef RTE_LIBRTE_IXGBE_RX_ALLOW_BULK_ALLOC
1995 if (rxq->rx_nb_avail) {
1996 for (i = 0; i < rxq->rx_nb_avail; ++i) {
1997 struct rte_mbuf *mb;
1998 mb = rxq->rx_stage[rxq->rx_next_avail + i];
1999 rte_pktmbuf_free_seg(mb);
2001 rxq->rx_nb_avail = 0;
2008 ixgbe_rx_queue_release(struct ixgbe_rx_queue *rxq)
2011 ixgbe_rx_queue_release_mbufs(rxq);
2012 rte_free(rxq->sw_ring);
2018 ixgbe_dev_rx_queue_release(void *rxq)
2020 ixgbe_rx_queue_release(rxq);
2024 * Check if Rx Burst Bulk Alloc function can be used.
2026 * 0: the preconditions are satisfied and the bulk allocation function
2028 * -EINVAL: the preconditions are NOT satisfied and the default Rx burst
2029 * function must be used.
2032 #ifdef RTE_LIBRTE_IXGBE_RX_ALLOW_BULK_ALLOC
2033 check_rx_burst_bulk_alloc_preconditions(struct ixgbe_rx_queue *rxq)
2035 check_rx_burst_bulk_alloc_preconditions(__rte_unused struct ixgbe_rx_queue *rxq)
2041 * Make sure the following pre-conditions are satisfied:
2042 * rxq->rx_free_thresh >= RTE_PMD_IXGBE_RX_MAX_BURST
2043 * rxq->rx_free_thresh < rxq->nb_rx_desc
2044 * (rxq->nb_rx_desc % rxq->rx_free_thresh) == 0
2045 * rxq->nb_rx_desc<(IXGBE_MAX_RING_DESC-RTE_PMD_IXGBE_RX_MAX_BURST)
2046 * Scattered packets are not supported. This should be checked
2047 * outside of this function.
2049 #ifdef RTE_LIBRTE_IXGBE_RX_ALLOW_BULK_ALLOC
2050 if (!(rxq->rx_free_thresh >= RTE_PMD_IXGBE_RX_MAX_BURST)) {
2051 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
2052 "rxq->rx_free_thresh=%d, "
2053 "RTE_PMD_IXGBE_RX_MAX_BURST=%d",
2054 rxq->rx_free_thresh, RTE_PMD_IXGBE_RX_MAX_BURST);
2056 } else if (!(rxq->rx_free_thresh < rxq->nb_rx_desc)) {
2057 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
2058 "rxq->rx_free_thresh=%d, "
2059 "rxq->nb_rx_desc=%d",
2060 rxq->rx_free_thresh, rxq->nb_rx_desc);
2062 } else if (!((rxq->nb_rx_desc % rxq->rx_free_thresh) == 0)) {
2063 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
2064 "rxq->nb_rx_desc=%d, "
2065 "rxq->rx_free_thresh=%d",
2066 rxq->nb_rx_desc, rxq->rx_free_thresh);
2068 } else if (!(rxq->nb_rx_desc <
2069 (IXGBE_MAX_RING_DESC - RTE_PMD_IXGBE_RX_MAX_BURST))) {
2070 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
2071 "rxq->nb_rx_desc=%d, "
2072 "IXGBE_MAX_RING_DESC=%d, "
2073 "RTE_PMD_IXGBE_RX_MAX_BURST=%d",
2074 rxq->nb_rx_desc, IXGBE_MAX_RING_DESC,
2075 RTE_PMD_IXGBE_RX_MAX_BURST);
2085 /* Reset dynamic ixgbe_rx_queue fields back to defaults */
2087 ixgbe_reset_rx_queue(struct ixgbe_hw *hw, struct ixgbe_rx_queue *rxq)
2089 static const union ixgbe_adv_rx_desc zeroed_desc = { .read = {
2092 uint16_t len = rxq->nb_rx_desc;
2095 * By default, the Rx queue setup function allocates enough memory for
2096 * IXGBE_MAX_RING_DESC. The Rx Burst bulk allocation function requires
2097 * extra memory at the end of the descriptor ring to be zero'd out. A
2098 * pre-condition for using the Rx burst bulk alloc function is that the
2099 * number of descriptors is less than or equal to
2100 * (IXGBE_MAX_RING_DESC - RTE_PMD_IXGBE_RX_MAX_BURST). Check all the
2101 * constraints here to see if we need to zero out memory after the end
2102 * of the H/W descriptor ring.
2104 if (hw->rx_bulk_alloc_allowed)
2105 /* zero out extra memory */
2106 len += RTE_PMD_IXGBE_RX_MAX_BURST;
2109 * Zero out HW ring memory. Zero out extra memory at the end of
2110 * the H/W ring so look-ahead logic in Rx Burst bulk alloc function
2111 * reads extra memory as zeros.
2113 for (i = 0; i < len; i++) {
2114 rxq->rx_ring[i] = zeroed_desc;
2117 #ifdef RTE_LIBRTE_IXGBE_RX_ALLOW_BULK_ALLOC
2119 * initialize extra software ring entries. Space for these extra
2120 * entries is always allocated
2122 memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf));
2123 for (i = rxq->nb_rx_desc; i < len; ++i) {
2124 rxq->sw_ring[i].mbuf = &rxq->fake_mbuf;
2127 rxq->rx_nb_avail = 0;
2128 rxq->rx_next_avail = 0;
2129 rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1);
2130 #endif /* RTE_LIBRTE_IXGBE_RX_ALLOW_BULK_ALLOC */
2132 rxq->nb_rx_hold = 0;
2133 rxq->pkt_first_seg = NULL;
2134 rxq->pkt_last_seg = NULL;
2138 ixgbe_dev_rx_queue_setup(struct rte_eth_dev *dev,
2141 unsigned int socket_id,
2142 const struct rte_eth_rxconf *rx_conf,
2143 struct rte_mempool *mp)
2145 const struct rte_memzone *rz;
2146 struct ixgbe_rx_queue *rxq;
2147 struct ixgbe_hw *hw;
2150 PMD_INIT_FUNC_TRACE();
2151 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2154 * Validate number of receive descriptors.
2155 * It must not exceed hardware maximum, and must be multiple
2158 if (((nb_desc * sizeof(union ixgbe_adv_rx_desc)) % IXGBE_ALIGN) != 0 ||
2159 (nb_desc > IXGBE_MAX_RING_DESC) ||
2160 (nb_desc < IXGBE_MIN_RING_DESC)) {
2164 /* Free memory prior to re-allocation if needed... */
2165 if (dev->data->rx_queues[queue_idx] != NULL) {
2166 ixgbe_rx_queue_release(dev->data->rx_queues[queue_idx]);
2167 dev->data->rx_queues[queue_idx] = NULL;
2170 /* First allocate the rx queue data structure */
2171 rxq = rte_zmalloc_socket("ethdev RX queue", sizeof(struct ixgbe_rx_queue),
2172 RTE_CACHE_LINE_SIZE, socket_id);
2176 rxq->nb_rx_desc = nb_desc;
2177 rxq->rx_free_thresh = rx_conf->rx_free_thresh;
2178 rxq->queue_id = queue_idx;
2179 rxq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ?
2180 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx);
2181 rxq->port_id = dev->data->port_id;
2182 rxq->crc_len = (uint8_t) ((dev->data->dev_conf.rxmode.hw_strip_crc) ?
2184 rxq->drop_en = rx_conf->rx_drop_en;
2185 rxq->rx_deferred_start = rx_conf->rx_deferred_start;
2188 * Allocate RX ring hardware descriptors. A memzone large enough to
2189 * handle the maximum ring size is allocated in order to allow for
2190 * resizing in later calls to the queue setup function.
2192 rz = ring_dma_zone_reserve(dev, "rx_ring", queue_idx,
2193 RX_RING_SZ, socket_id);
2195 ixgbe_rx_queue_release(rxq);
2200 * Zero init all the descriptors in the ring.
2202 memset (rz->addr, 0, RX_RING_SZ);
2205 * Modified to setup VFRDT for Virtual Function
2207 if (hw->mac.type == ixgbe_mac_82599_vf ||
2208 hw->mac.type == ixgbe_mac_X540_vf ||
2209 hw->mac.type == ixgbe_mac_X550_vf ||
2210 hw->mac.type == ixgbe_mac_X550EM_x_vf) {
2212 IXGBE_PCI_REG_ADDR(hw, IXGBE_VFRDT(queue_idx));
2214 IXGBE_PCI_REG_ADDR(hw, IXGBE_VFRDH(queue_idx));
2218 IXGBE_PCI_REG_ADDR(hw, IXGBE_RDT(rxq->reg_idx));
2220 IXGBE_PCI_REG_ADDR(hw, IXGBE_RDH(rxq->reg_idx));
2222 #ifndef RTE_LIBRTE_XEN_DOM0
2223 rxq->rx_ring_phys_addr = (uint64_t) rz->phys_addr;
2225 rxq->rx_ring_phys_addr = rte_mem_phy2mch(rz->memseg_id, rz->phys_addr);
2227 rxq->rx_ring = (union ixgbe_adv_rx_desc *) rz->addr;
2230 * Certain constraints must be met in order to use the bulk buffer
2231 * allocation Rx burst function. If any of Rx queues doesn't meet them
2232 * the feature should be disabled for the whole port.
2234 if (check_rx_burst_bulk_alloc_preconditions(rxq)) {
2235 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Rx Bulk Alloc "
2236 "preconditions - canceling the feature for "
2237 "the whole port[%d]",
2238 rxq->queue_id, rxq->port_id);
2239 hw->rx_bulk_alloc_allowed = false;
2243 * Allocate software ring. Allow for space at the end of the
2244 * S/W ring to make sure look-ahead logic in bulk alloc Rx burst
2245 * function does not access an invalid memory region.
2248 if (hw->rx_bulk_alloc_allowed)
2249 len += RTE_PMD_IXGBE_RX_MAX_BURST;
2251 rxq->sw_ring = rte_zmalloc_socket("rxq->sw_ring",
2252 sizeof(struct ixgbe_rx_entry) * len,
2253 RTE_CACHE_LINE_SIZE, socket_id);
2254 if (rxq->sw_ring == NULL) {
2255 ixgbe_rx_queue_release(rxq);
2258 PMD_INIT_LOG(DEBUG, "sw_ring=%p hw_ring=%p dma_addr=0x%"PRIx64,
2259 rxq->sw_ring, rxq->rx_ring, rxq->rx_ring_phys_addr);
2261 if (!rte_is_power_of_2(nb_desc)) {
2262 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
2263 "preconditions - canceling the feature for "
2264 "the whole port[%d]",
2265 rxq->queue_id, rxq->port_id);
2266 hw->rx_vec_allowed = false;
2268 ixgbe_rxq_vec_setup(rxq);
2270 dev->data->rx_queues[queue_idx] = rxq;
2272 ixgbe_reset_rx_queue(hw, rxq);
2278 ixgbe_dev_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id)
2280 #define IXGBE_RXQ_SCAN_INTERVAL 4
2281 volatile union ixgbe_adv_rx_desc *rxdp;
2282 struct ixgbe_rx_queue *rxq;
2285 if (rx_queue_id >= dev->data->nb_rx_queues) {
2286 PMD_RX_LOG(ERR, "Invalid RX queue id=%d", rx_queue_id);
2290 rxq = dev->data->rx_queues[rx_queue_id];
2291 rxdp = &(rxq->rx_ring[rxq->rx_tail]);
2293 while ((desc < rxq->nb_rx_desc) &&
2294 (rxdp->wb.upper.status_error & IXGBE_RXDADV_STAT_DD)) {
2295 desc += IXGBE_RXQ_SCAN_INTERVAL;
2296 rxdp += IXGBE_RXQ_SCAN_INTERVAL;
2297 if (rxq->rx_tail + desc >= rxq->nb_rx_desc)
2298 rxdp = &(rxq->rx_ring[rxq->rx_tail +
2299 desc - rxq->nb_rx_desc]);
2306 ixgbe_dev_rx_descriptor_done(void *rx_queue, uint16_t offset)
2308 volatile union ixgbe_adv_rx_desc *rxdp;
2309 struct ixgbe_rx_queue *rxq = rx_queue;
2312 if (unlikely(offset >= rxq->nb_rx_desc))
2314 desc = rxq->rx_tail + offset;
2315 if (desc >= rxq->nb_rx_desc)
2316 desc -= rxq->nb_rx_desc;
2318 rxdp = &rxq->rx_ring[desc];
2319 return !!(rxdp->wb.upper.status_error & IXGBE_RXDADV_STAT_DD);
2323 ixgbe_dev_clear_queues(struct rte_eth_dev *dev)
2326 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2328 PMD_INIT_FUNC_TRACE();
2330 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2331 struct ixgbe_tx_queue *txq = dev->data->tx_queues[i];
2333 txq->ops->release_mbufs(txq);
2334 txq->ops->reset(txq);
2338 for (i = 0; i < dev->data->nb_rx_queues; i++) {
2339 struct ixgbe_rx_queue *rxq = dev->data->rx_queues[i];
2341 ixgbe_rx_queue_release_mbufs(rxq);
2342 ixgbe_reset_rx_queue(hw, rxq);
2347 /*********************************************************************
2349 * Device RX/TX init functions
2351 **********************************************************************/
2354 * Receive Side Scaling (RSS)
2355 * See section 7.1.2.8 in the following document:
2356 * "Intel 82599 10 GbE Controller Datasheet" - Revision 2.1 October 2009
2359 * The source and destination IP addresses of the IP header and the source
2360 * and destination ports of TCP/UDP headers, if any, of received packets are
2361 * hashed against a configurable random key to compute a 32-bit RSS hash result.
2362 * The seven (7) LSBs of the 32-bit hash result are used as an index into a
2363 * 128-entry redirection table (RETA). Each entry of the RETA provides a 3-bit
2364 * RSS output index which is used as the RX queue index where to store the
2366 * The following output is supplied in the RX write-back descriptor:
2367 * - 32-bit result of the Microsoft RSS hash function,
2368 * - 4-bit RSS type field.
2372 * RSS random key supplied in section 7.1.2.8.3 of the Intel 82599 datasheet.
2373 * Used as the default key.
2375 static uint8_t rss_intel_key[40] = {
2376 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
2377 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
2378 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
2379 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
2380 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
2384 ixgbe_rss_disable(struct rte_eth_dev *dev)
2386 struct ixgbe_hw *hw;
2389 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2390 mrqc = IXGBE_READ_REG(hw, IXGBE_MRQC);
2391 mrqc &= ~IXGBE_MRQC_RSSEN;
2392 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2396 ixgbe_hw_rss_hash_set(struct ixgbe_hw *hw, struct rte_eth_rss_conf *rss_conf)
2404 hash_key = rss_conf->rss_key;
2405 if (hash_key != NULL) {
2406 /* Fill in RSS hash key */
2407 for (i = 0; i < 10; i++) {
2408 rss_key = hash_key[(i * 4)];
2409 rss_key |= hash_key[(i * 4) + 1] << 8;
2410 rss_key |= hash_key[(i * 4) + 2] << 16;
2411 rss_key |= hash_key[(i * 4) + 3] << 24;
2412 IXGBE_WRITE_REG_ARRAY(hw, IXGBE_RSSRK(0), i, rss_key);
2416 /* Set configured hashing protocols in MRQC register */
2417 rss_hf = rss_conf->rss_hf;
2418 mrqc = IXGBE_MRQC_RSSEN; /* Enable RSS */
2419 if (rss_hf & ETH_RSS_IPV4)
2420 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4;
2421 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
2422 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP;
2423 if (rss_hf & ETH_RSS_IPV6)
2424 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6;
2425 if (rss_hf & ETH_RSS_IPV6_EX)
2426 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX;
2427 if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP)
2428 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP;
2429 if (rss_hf & ETH_RSS_IPV6_TCP_EX)
2430 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP;
2431 if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP)
2432 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP;
2433 if (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP)
2434 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP;
2435 if (rss_hf & ETH_RSS_IPV6_UDP_EX)
2436 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2437 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2441 ixgbe_dev_rss_hash_update(struct rte_eth_dev *dev,
2442 struct rte_eth_rss_conf *rss_conf)
2444 struct ixgbe_hw *hw;
2448 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2451 * Excerpt from section 7.1.2.8 Receive-Side Scaling (RSS):
2452 * "RSS enabling cannot be done dynamically while it must be
2453 * preceded by a software reset"
2454 * Before changing anything, first check that the update RSS operation
2455 * does not attempt to disable RSS, if RSS was enabled at
2456 * initialization time, or does not attempt to enable RSS, if RSS was
2457 * disabled at initialization time.
2459 rss_hf = rss_conf->rss_hf & IXGBE_RSS_OFFLOAD_ALL;
2460 mrqc = IXGBE_READ_REG(hw, IXGBE_MRQC);
2461 if (!(mrqc & IXGBE_MRQC_RSSEN)) { /* RSS disabled */
2462 if (rss_hf != 0) /* Enable RSS */
2464 return 0; /* Nothing to do */
2467 if (rss_hf == 0) /* Disable RSS */
2469 ixgbe_hw_rss_hash_set(hw, rss_conf);
2474 ixgbe_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
2475 struct rte_eth_rss_conf *rss_conf)
2477 struct ixgbe_hw *hw;
2484 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2485 hash_key = rss_conf->rss_key;
2486 if (hash_key != NULL) {
2487 /* Return RSS hash key */
2488 for (i = 0; i < 10; i++) {
2489 rss_key = IXGBE_READ_REG_ARRAY(hw, IXGBE_RSSRK(0), i);
2490 hash_key[(i * 4)] = rss_key & 0x000000FF;
2491 hash_key[(i * 4) + 1] = (rss_key >> 8) & 0x000000FF;
2492 hash_key[(i * 4) + 2] = (rss_key >> 16) & 0x000000FF;
2493 hash_key[(i * 4) + 3] = (rss_key >> 24) & 0x000000FF;
2497 /* Get RSS functions configured in MRQC register */
2498 mrqc = IXGBE_READ_REG(hw, IXGBE_MRQC);
2499 if ((mrqc & IXGBE_MRQC_RSSEN) == 0) { /* RSS is disabled */
2500 rss_conf->rss_hf = 0;
2504 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV4)
2505 rss_hf |= ETH_RSS_IPV4;
2506 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV4_TCP)
2507 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP;
2508 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6)
2509 rss_hf |= ETH_RSS_IPV6;
2510 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_EX)
2511 rss_hf |= ETH_RSS_IPV6_EX;
2512 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_TCP)
2513 rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP;
2514 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP)
2515 rss_hf |= ETH_RSS_IPV6_TCP_EX;
2516 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV4_UDP)
2517 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP;
2518 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_UDP)
2519 rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP;
2520 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP)
2521 rss_hf |= ETH_RSS_IPV6_UDP_EX;
2522 rss_conf->rss_hf = rss_hf;
2527 ixgbe_rss_configure(struct rte_eth_dev *dev)
2529 struct rte_eth_rss_conf rss_conf;
2530 struct ixgbe_hw *hw;
2535 PMD_INIT_FUNC_TRACE();
2536 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2539 * Fill in redirection table
2540 * The byte-swap is needed because NIC registers are in
2541 * little-endian order.
2544 for (i = 0, j = 0; i < 128; i++, j++) {
2545 if (j == dev->data->nb_rx_queues)
2547 reta = (reta << 8) | j;
2549 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2),
2554 * Configure the RSS key and the RSS protocols used to compute
2555 * the RSS hash of input packets.
2557 rss_conf = dev->data->dev_conf.rx_adv_conf.rss_conf;
2558 if ((rss_conf.rss_hf & IXGBE_RSS_OFFLOAD_ALL) == 0) {
2559 ixgbe_rss_disable(dev);
2562 if (rss_conf.rss_key == NULL)
2563 rss_conf.rss_key = rss_intel_key; /* Default hash key */
2564 ixgbe_hw_rss_hash_set(hw, &rss_conf);
2567 #define NUM_VFTA_REGISTERS 128
2568 #define NIC_RX_BUFFER_SIZE 0x200
2571 ixgbe_vmdq_dcb_configure(struct rte_eth_dev *dev)
2573 struct rte_eth_vmdq_dcb_conf *cfg;
2574 struct ixgbe_hw *hw;
2575 enum rte_eth_nb_pools num_pools;
2576 uint32_t mrqc, vt_ctl, queue_mapping, vlanctrl;
2578 uint8_t nb_tcs; /* number of traffic classes */
2581 PMD_INIT_FUNC_TRACE();
2582 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2583 cfg = &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf;
2584 num_pools = cfg->nb_queue_pools;
2585 /* Check we have a valid number of pools */
2586 if (num_pools != ETH_16_POOLS && num_pools != ETH_32_POOLS) {
2587 ixgbe_rss_disable(dev);
2590 /* 16 pools -> 8 traffic classes, 32 pools -> 4 traffic classes */
2591 nb_tcs = (uint8_t)(ETH_VMDQ_DCB_NUM_QUEUES / (int)num_pools);
2595 * split rx buffer up into sections, each for 1 traffic class
2597 pbsize = (uint16_t)(NIC_RX_BUFFER_SIZE / nb_tcs);
2598 for (i = 0 ; i < nb_tcs; i++) {
2599 uint32_t rxpbsize = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(i));
2600 rxpbsize &= (~(0x3FF << IXGBE_RXPBSIZE_SHIFT));
2601 /* clear 10 bits. */
2602 rxpbsize |= (pbsize << IXGBE_RXPBSIZE_SHIFT); /* set value */
2603 IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), rxpbsize);
2605 /* zero alloc all unused TCs */
2606 for (i = nb_tcs; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
2607 uint32_t rxpbsize = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(i));
2608 rxpbsize &= (~( 0x3FF << IXGBE_RXPBSIZE_SHIFT ));
2609 /* clear 10 bits. */
2610 IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), rxpbsize);
2613 /* MRQC: enable vmdq and dcb */
2614 mrqc = ((num_pools == ETH_16_POOLS) ? \
2615 IXGBE_MRQC_VMDQRT8TCEN : IXGBE_MRQC_VMDQRT4TCEN );
2616 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2618 /* PFVTCTL: turn on virtualisation and set the default pool */
2619 vt_ctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
2620 if (cfg->enable_default_pool) {
2621 vt_ctl |= (cfg->default_pool << IXGBE_VT_CTL_POOL_SHIFT);
2623 vt_ctl |= IXGBE_VT_CTL_DIS_DEFPL;
2626 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vt_ctl);
2628 /* RTRUP2TC: mapping user priorities to traffic classes (TCs) */
2630 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++)
2632 * mapping is done with 3 bits per priority,
2633 * so shift by i*3 each time
2635 queue_mapping |= ((cfg->dcb_queue[i] & 0x07) << (i * 3));
2637 IXGBE_WRITE_REG(hw, IXGBE_RTRUP2TC, queue_mapping);
2639 /* RTRPCS: DCB related */
2640 IXGBE_WRITE_REG(hw, IXGBE_RTRPCS, IXGBE_RMCS_RRM);
2642 /* VLNCTRL: enable vlan filtering and allow all vlan tags through */
2643 vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
2644 vlanctrl |= IXGBE_VLNCTRL_VFE ; /* enable vlan filters */
2645 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl);
2647 /* VFTA - enable all vlan filters */
2648 for (i = 0; i < NUM_VFTA_REGISTERS; i++) {
2649 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), 0xFFFFFFFF);
2652 /* VFRE: pool enabling for receive - 16 or 32 */
2653 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), \
2654 num_pools == ETH_16_POOLS ? 0xFFFF : 0xFFFFFFFF);
2657 * MPSAR - allow pools to read specific mac addresses
2658 * In this case, all pools should be able to read from mac addr 0
2660 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(0), 0xFFFFFFFF);
2661 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(0), 0xFFFFFFFF);
2663 /* PFVLVF, PFVLVFB: set up filters for vlan tags as configured */
2664 for (i = 0; i < cfg->nb_pool_maps; i++) {
2665 /* set vlan id in VF register and set the valid bit */
2666 IXGBE_WRITE_REG(hw, IXGBE_VLVF(i), (IXGBE_VLVF_VIEN | \
2667 (cfg->pool_map[i].vlan_id & 0xFFF)));
2669 * Put the allowed pools in VFB reg. As we only have 16 or 32
2670 * pools, we only need to use the first half of the register
2673 IXGBE_WRITE_REG(hw, IXGBE_VLVFB(i*2), cfg->pool_map[i].pools);
2678 * ixgbe_dcb_config_tx_hw_config - Configure general DCB TX parameters
2679 * @hw: pointer to hardware structure
2680 * @dcb_config: pointer to ixgbe_dcb_config structure
2683 ixgbe_dcb_tx_hw_config(struct ixgbe_hw *hw,
2684 struct ixgbe_dcb_config *dcb_config)
2689 PMD_INIT_FUNC_TRACE();
2690 if (hw->mac.type != ixgbe_mac_82598EB) {
2691 /* Disable the Tx desc arbiter so that MTQC can be changed */
2692 reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2693 reg |= IXGBE_RTTDCS_ARBDIS;
2694 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg);
2696 /* Enable DCB for Tx with 8 TCs */
2697 if (dcb_config->num_tcs.pg_tcs == 8) {
2698 reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;
2701 reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
2703 if (dcb_config->vt_mode)
2704 reg |= IXGBE_MTQC_VT_ENA;
2705 IXGBE_WRITE_REG(hw, IXGBE_MTQC, reg);
2707 /* Disable drop for all queues */
2708 for (q = 0; q < 128; q++)
2709 IXGBE_WRITE_REG(hw, IXGBE_QDE,
2710 (IXGBE_QDE_WRITE | (q << IXGBE_QDE_IDX_SHIFT)));
2712 /* Enable the Tx desc arbiter */
2713 reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2714 reg &= ~IXGBE_RTTDCS_ARBDIS;
2715 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg);
2717 /* Enable Security TX Buffer IFG for DCB */
2718 reg = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG);
2719 reg |= IXGBE_SECTX_DCB;
2720 IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, reg);
2726 * ixgbe_vmdq_dcb_hw_tx_config - Configure general VMDQ+DCB TX parameters
2727 * @dev: pointer to rte_eth_dev structure
2728 * @dcb_config: pointer to ixgbe_dcb_config structure
2731 ixgbe_vmdq_dcb_hw_tx_config(struct rte_eth_dev *dev,
2732 struct ixgbe_dcb_config *dcb_config)
2734 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
2735 &dev->data->dev_conf.tx_adv_conf.vmdq_dcb_tx_conf;
2736 struct ixgbe_hw *hw =
2737 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2739 PMD_INIT_FUNC_TRACE();
2740 if (hw->mac.type != ixgbe_mac_82598EB)
2741 /*PF VF Transmit Enable*/
2742 IXGBE_WRITE_REG(hw, IXGBE_VFTE(0),
2743 vmdq_tx_conf->nb_queue_pools == ETH_16_POOLS ? 0xFFFF : 0xFFFFFFFF);
2745 /*Configure general DCB TX parameters*/
2746 ixgbe_dcb_tx_hw_config(hw,dcb_config);
2751 ixgbe_vmdq_dcb_rx_config(struct rte_eth_dev *dev,
2752 struct ixgbe_dcb_config *dcb_config)
2754 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
2755 &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf;
2756 struct ixgbe_dcb_tc_config *tc;
2759 /* convert rte_eth_conf.rx_adv_conf to struct ixgbe_dcb_config */
2760 if (vmdq_rx_conf->nb_queue_pools == ETH_16_POOLS ) {
2761 dcb_config->num_tcs.pg_tcs = ETH_8_TCS;
2762 dcb_config->num_tcs.pfc_tcs = ETH_8_TCS;
2765 dcb_config->num_tcs.pg_tcs = ETH_4_TCS;
2766 dcb_config->num_tcs.pfc_tcs = ETH_4_TCS;
2768 /* User Priority to Traffic Class mapping */
2769 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
2770 j = vmdq_rx_conf->dcb_queue[i];
2771 tc = &dcb_config->tc_config[j];
2772 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap =
2778 ixgbe_dcb_vt_tx_config(struct rte_eth_dev *dev,
2779 struct ixgbe_dcb_config *dcb_config)
2781 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
2782 &dev->data->dev_conf.tx_adv_conf.vmdq_dcb_tx_conf;
2783 struct ixgbe_dcb_tc_config *tc;
2786 /* convert rte_eth_conf.rx_adv_conf to struct ixgbe_dcb_config */
2787 if (vmdq_tx_conf->nb_queue_pools == ETH_16_POOLS ) {
2788 dcb_config->num_tcs.pg_tcs = ETH_8_TCS;
2789 dcb_config->num_tcs.pfc_tcs = ETH_8_TCS;
2792 dcb_config->num_tcs.pg_tcs = ETH_4_TCS;
2793 dcb_config->num_tcs.pfc_tcs = ETH_4_TCS;
2796 /* User Priority to Traffic Class mapping */
2797 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
2798 j = vmdq_tx_conf->dcb_queue[i];
2799 tc = &dcb_config->tc_config[j];
2800 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap =
2807 ixgbe_dcb_rx_config(struct rte_eth_dev *dev,
2808 struct ixgbe_dcb_config *dcb_config)
2810 struct rte_eth_dcb_rx_conf *rx_conf =
2811 &dev->data->dev_conf.rx_adv_conf.dcb_rx_conf;
2812 struct ixgbe_dcb_tc_config *tc;
2815 dcb_config->num_tcs.pg_tcs = (uint8_t)rx_conf->nb_tcs;
2816 dcb_config->num_tcs.pfc_tcs = (uint8_t)rx_conf->nb_tcs;
2818 /* User Priority to Traffic Class mapping */
2819 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
2820 j = rx_conf->dcb_queue[i];
2821 tc = &dcb_config->tc_config[j];
2822 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap =
2828 ixgbe_dcb_tx_config(struct rte_eth_dev *dev,
2829 struct ixgbe_dcb_config *dcb_config)
2831 struct rte_eth_dcb_tx_conf *tx_conf =
2832 &dev->data->dev_conf.tx_adv_conf.dcb_tx_conf;
2833 struct ixgbe_dcb_tc_config *tc;
2836 dcb_config->num_tcs.pg_tcs = (uint8_t)tx_conf->nb_tcs;
2837 dcb_config->num_tcs.pfc_tcs = (uint8_t)tx_conf->nb_tcs;
2839 /* User Priority to Traffic Class mapping */
2840 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
2841 j = tx_conf->dcb_queue[i];
2842 tc = &dcb_config->tc_config[j];
2843 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap =
2849 * ixgbe_dcb_rx_hw_config - Configure general DCB RX HW parameters
2850 * @hw: pointer to hardware structure
2851 * @dcb_config: pointer to ixgbe_dcb_config structure
2854 ixgbe_dcb_rx_hw_config(struct ixgbe_hw *hw,
2855 struct ixgbe_dcb_config *dcb_config)
2861 PMD_INIT_FUNC_TRACE();
2863 * Disable the arbiter before changing parameters
2864 * (always enable recycle mode; WSP)
2866 reg = IXGBE_RTRPCS_RRM | IXGBE_RTRPCS_RAC | IXGBE_RTRPCS_ARBDIS;
2867 IXGBE_WRITE_REG(hw, IXGBE_RTRPCS, reg);
2869 if (hw->mac.type != ixgbe_mac_82598EB) {
2870 reg = IXGBE_READ_REG(hw, IXGBE_MRQC);
2871 if (dcb_config->num_tcs.pg_tcs == 4) {
2872 if (dcb_config->vt_mode)
2873 reg = (reg & ~IXGBE_MRQC_MRQE_MASK) |
2874 IXGBE_MRQC_VMDQRT4TCEN;
2876 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, 0);
2877 reg = (reg & ~IXGBE_MRQC_MRQE_MASK) |
2881 if (dcb_config->num_tcs.pg_tcs == 8) {
2882 if (dcb_config->vt_mode)
2883 reg = (reg & ~IXGBE_MRQC_MRQE_MASK) |
2884 IXGBE_MRQC_VMDQRT8TCEN;
2886 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, 0);
2887 reg = (reg & ~IXGBE_MRQC_MRQE_MASK) |
2892 IXGBE_WRITE_REG(hw, IXGBE_MRQC, reg);
2895 /* VLNCTRL: enable vlan filtering and allow all vlan tags through */
2896 vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
2897 vlanctrl |= IXGBE_VLNCTRL_VFE ; /* enable vlan filters */
2898 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl);
2900 /* VFTA - enable all vlan filters */
2901 for (i = 0; i < NUM_VFTA_REGISTERS; i++) {
2902 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), 0xFFFFFFFF);
2906 * Configure Rx packet plane (recycle mode; WSP) and
2909 reg = IXGBE_RTRPCS_RRM | IXGBE_RTRPCS_RAC;
2910 IXGBE_WRITE_REG(hw, IXGBE_RTRPCS, reg);
2916 ixgbe_dcb_hw_arbite_rx_config(struct ixgbe_hw *hw, uint16_t *refill,
2917 uint16_t *max,uint8_t *bwg_id, uint8_t *tsa, uint8_t *map)
2919 switch (hw->mac.type) {
2920 case ixgbe_mac_82598EB:
2921 ixgbe_dcb_config_rx_arbiter_82598(hw, refill, max, tsa);
2923 case ixgbe_mac_82599EB:
2924 case ixgbe_mac_X540:
2925 case ixgbe_mac_X550:
2926 case ixgbe_mac_X550EM_x:
2927 ixgbe_dcb_config_rx_arbiter_82599(hw, refill, max, bwg_id,
2936 ixgbe_dcb_hw_arbite_tx_config(struct ixgbe_hw *hw, uint16_t *refill, uint16_t *max,
2937 uint8_t *bwg_id, uint8_t *tsa, uint8_t *map)
2939 switch (hw->mac.type) {
2940 case ixgbe_mac_82598EB:
2941 ixgbe_dcb_config_tx_desc_arbiter_82598(hw, refill, max, bwg_id,tsa);
2942 ixgbe_dcb_config_tx_data_arbiter_82598(hw, refill, max, bwg_id,tsa);
2944 case ixgbe_mac_82599EB:
2945 case ixgbe_mac_X540:
2946 case ixgbe_mac_X550:
2947 case ixgbe_mac_X550EM_x:
2948 ixgbe_dcb_config_tx_desc_arbiter_82599(hw, refill, max, bwg_id,tsa);
2949 ixgbe_dcb_config_tx_data_arbiter_82599(hw, refill, max, bwg_id,tsa, map);
2956 #define DCB_RX_CONFIG 1
2957 #define DCB_TX_CONFIG 1
2958 #define DCB_TX_PB 1024
2960 * ixgbe_dcb_hw_configure - Enable DCB and configure
2961 * general DCB in VT mode and non-VT mode parameters
2962 * @dev: pointer to rte_eth_dev structure
2963 * @dcb_config: pointer to ixgbe_dcb_config structure
2966 ixgbe_dcb_hw_configure(struct rte_eth_dev *dev,
2967 struct ixgbe_dcb_config *dcb_config)
2970 uint8_t i,pfc_en,nb_tcs;
2972 uint8_t config_dcb_rx = 0;
2973 uint8_t config_dcb_tx = 0;
2974 uint8_t tsa[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0};
2975 uint8_t bwgid[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0};
2976 uint16_t refill[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0};
2977 uint16_t max[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0};
2978 uint8_t map[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0};
2979 struct ixgbe_dcb_tc_config *tc;
2980 uint32_t max_frame = dev->data->mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2981 struct ixgbe_hw *hw =
2982 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2984 switch(dev->data->dev_conf.rxmode.mq_mode){
2985 case ETH_MQ_RX_VMDQ_DCB:
2986 dcb_config->vt_mode = true;
2987 if (hw->mac.type != ixgbe_mac_82598EB) {
2988 config_dcb_rx = DCB_RX_CONFIG;
2990 *get dcb and VT rx configuration parameters
2993 ixgbe_vmdq_dcb_rx_config(dev,dcb_config);
2994 /*Configure general VMDQ and DCB RX parameters*/
2995 ixgbe_vmdq_dcb_configure(dev);
2999 dcb_config->vt_mode = false;
3000 config_dcb_rx = DCB_RX_CONFIG;
3001 /* Get dcb TX configuration parameters from rte_eth_conf */
3002 ixgbe_dcb_rx_config(dev,dcb_config);
3003 /*Configure general DCB RX parameters*/
3004 ixgbe_dcb_rx_hw_config(hw, dcb_config);
3007 PMD_INIT_LOG(ERR, "Incorrect DCB RX mode configuration");
3010 switch (dev->data->dev_conf.txmode.mq_mode) {
3011 case ETH_MQ_TX_VMDQ_DCB:
3012 dcb_config->vt_mode = true;
3013 config_dcb_tx = DCB_TX_CONFIG;
3014 /* get DCB and VT TX configuration parameters from rte_eth_conf */
3015 ixgbe_dcb_vt_tx_config(dev,dcb_config);
3016 /*Configure general VMDQ and DCB TX parameters*/
3017 ixgbe_vmdq_dcb_hw_tx_config(dev,dcb_config);
3021 dcb_config->vt_mode = false;
3022 config_dcb_tx = DCB_TX_CONFIG;
3023 /*get DCB TX configuration parameters from rte_eth_conf*/
3024 ixgbe_dcb_tx_config(dev,dcb_config);
3025 /*Configure general DCB TX parameters*/
3026 ixgbe_dcb_tx_hw_config(hw, dcb_config);
3029 PMD_INIT_LOG(ERR, "Incorrect DCB TX mode configuration");
3033 nb_tcs = dcb_config->num_tcs.pfc_tcs;
3035 ixgbe_dcb_unpack_map_cee(dcb_config, IXGBE_DCB_RX_CONFIG, map);
3036 if(nb_tcs == ETH_4_TCS) {
3037 /* Avoid un-configured priority mapping to TC0 */
3039 uint8_t mask = 0xFF;
3040 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES - 4; i++)
3041 mask = (uint8_t)(mask & (~ (1 << map[i])));
3042 for (i = 0; mask && (i < IXGBE_DCB_MAX_TRAFFIC_CLASS); i++) {
3043 if ((mask & 0x1) && (j < ETH_DCB_NUM_USER_PRIORITIES))
3047 /* Re-configure 4 TCs BW */
3048 for (i = 0; i < nb_tcs; i++) {
3049 tc = &dcb_config->tc_config[i];
3050 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent =
3051 (uint8_t)(100 / nb_tcs);
3052 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent =
3053 (uint8_t)(100 / nb_tcs);
3055 for (; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
3056 tc = &dcb_config->tc_config[i];
3057 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent = 0;
3058 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent = 0;
3063 /* Set RX buffer size */
3064 pbsize = (uint16_t)(NIC_RX_BUFFER_SIZE / nb_tcs);
3065 uint32_t rxpbsize = pbsize << IXGBE_RXPBSIZE_SHIFT;
3066 for (i = 0 ; i < nb_tcs; i++) {
3067 IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), rxpbsize);
3069 /* zero alloc all unused TCs */
3070 for (; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3071 IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), 0);
3075 /* Only support an equally distributed Tx packet buffer strategy. */
3076 uint32_t txpktsize = IXGBE_TXPBSIZE_MAX / nb_tcs;
3077 uint32_t txpbthresh = (txpktsize / DCB_TX_PB) - IXGBE_TXPKT_SIZE_MAX;
3078 for (i = 0; i < nb_tcs; i++) {
3079 IXGBE_WRITE_REG(hw, IXGBE_TXPBSIZE(i), txpktsize);
3080 IXGBE_WRITE_REG(hw, IXGBE_TXPBTHRESH(i), txpbthresh);
3082 /* Clear unused TCs, if any, to zero buffer size*/
3083 for (; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3084 IXGBE_WRITE_REG(hw, IXGBE_TXPBSIZE(i), 0);
3085 IXGBE_WRITE_REG(hw, IXGBE_TXPBTHRESH(i), 0);
3089 /*Calculates traffic class credits*/
3090 ixgbe_dcb_calculate_tc_credits_cee(hw, dcb_config,max_frame,
3091 IXGBE_DCB_TX_CONFIG);
3092 ixgbe_dcb_calculate_tc_credits_cee(hw, dcb_config,max_frame,
3093 IXGBE_DCB_RX_CONFIG);
3096 /* Unpack CEE standard containers */
3097 ixgbe_dcb_unpack_refill_cee(dcb_config, IXGBE_DCB_RX_CONFIG, refill);
3098 ixgbe_dcb_unpack_max_cee(dcb_config, max);
3099 ixgbe_dcb_unpack_bwgid_cee(dcb_config, IXGBE_DCB_RX_CONFIG, bwgid);
3100 ixgbe_dcb_unpack_tsa_cee(dcb_config, IXGBE_DCB_RX_CONFIG, tsa);
3101 /* Configure PG(ETS) RX */
3102 ixgbe_dcb_hw_arbite_rx_config(hw,refill,max,bwgid,tsa,map);
3106 /* Unpack CEE standard containers */
3107 ixgbe_dcb_unpack_refill_cee(dcb_config, IXGBE_DCB_TX_CONFIG, refill);
3108 ixgbe_dcb_unpack_max_cee(dcb_config, max);
3109 ixgbe_dcb_unpack_bwgid_cee(dcb_config, IXGBE_DCB_TX_CONFIG, bwgid);
3110 ixgbe_dcb_unpack_tsa_cee(dcb_config, IXGBE_DCB_TX_CONFIG, tsa);
3111 /* Configure PG(ETS) TX */
3112 ixgbe_dcb_hw_arbite_tx_config(hw,refill,max,bwgid,tsa,map);
3115 /*Configure queue statistics registers*/
3116 ixgbe_dcb_config_tc_stats_82599(hw, dcb_config);
3118 /* Check if the PFC is supported */
3119 if(dev->data->dev_conf.dcb_capability_en & ETH_DCB_PFC_SUPPORT) {
3120 pbsize = (uint16_t) (NIC_RX_BUFFER_SIZE / nb_tcs);
3121 for (i = 0; i < nb_tcs; i++) {
3123 * If the TC count is 8,and the default high_water is 48,
3124 * the low_water is 16 as default.
3126 hw->fc.high_water[i] = (pbsize * 3 ) / 4;
3127 hw->fc.low_water[i] = pbsize / 4;
3128 /* Enable pfc for this TC */
3129 tc = &dcb_config->tc_config[i];
3130 tc->pfc = ixgbe_dcb_pfc_enabled;
3132 ixgbe_dcb_unpack_pfc_cee(dcb_config, map, &pfc_en);
3133 if(dcb_config->num_tcs.pfc_tcs == ETH_4_TCS)
3135 ret = ixgbe_dcb_config_pfc(hw, pfc_en, map);
3142 * ixgbe_configure_dcb - Configure DCB Hardware
3143 * @dev: pointer to rte_eth_dev
3145 void ixgbe_configure_dcb(struct rte_eth_dev *dev)
3147 struct ixgbe_dcb_config *dcb_cfg =
3148 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private);
3149 struct rte_eth_conf *dev_conf = &(dev->data->dev_conf);
3151 PMD_INIT_FUNC_TRACE();
3153 /* check support mq_mode for DCB */
3154 if ((dev_conf->rxmode.mq_mode != ETH_MQ_RX_VMDQ_DCB) &&
3155 (dev_conf->rxmode.mq_mode != ETH_MQ_RX_DCB))
3158 if (dev->data->nb_rx_queues != ETH_DCB_NUM_QUEUES)
3161 /** Configure DCB hardware **/
3162 ixgbe_dcb_hw_configure(dev,dcb_cfg);
3168 * VMDq only support for 10 GbE NIC.
3171 ixgbe_vmdq_rx_hw_configure(struct rte_eth_dev *dev)
3173 struct rte_eth_vmdq_rx_conf *cfg;
3174 struct ixgbe_hw *hw;
3175 enum rte_eth_nb_pools num_pools;
3176 uint32_t mrqc, vt_ctl, vlanctrl;
3180 PMD_INIT_FUNC_TRACE();
3181 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3182 cfg = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
3183 num_pools = cfg->nb_queue_pools;
3185 ixgbe_rss_disable(dev);
3187 /* MRQC: enable vmdq */
3188 mrqc = IXGBE_MRQC_VMDQEN;
3189 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
3191 /* PFVTCTL: turn on virtualisation and set the default pool */
3192 vt_ctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
3193 if (cfg->enable_default_pool)
3194 vt_ctl |= (cfg->default_pool << IXGBE_VT_CTL_POOL_SHIFT);
3196 vt_ctl |= IXGBE_VT_CTL_DIS_DEFPL;
3198 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vt_ctl);
3200 for (i = 0; i < (int)num_pools; i++) {
3201 vmolr = ixgbe_convert_vm_rx_mask_to_val(cfg->rx_mode, vmolr);
3202 IXGBE_WRITE_REG(hw, IXGBE_VMOLR(i), vmolr);
3205 /* VLNCTRL: enable vlan filtering and allow all vlan tags through */
3206 vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
3207 vlanctrl |= IXGBE_VLNCTRL_VFE ; /* enable vlan filters */
3208 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl);
3210 /* VFTA - enable all vlan filters */
3211 for (i = 0; i < NUM_VFTA_REGISTERS; i++)
3212 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), UINT32_MAX);
3214 /* VFRE: pool enabling for receive - 64 */
3215 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), UINT32_MAX);
3216 if (num_pools == ETH_64_POOLS)
3217 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), UINT32_MAX);
3220 * MPSAR - allow pools to read specific mac addresses
3221 * In this case, all pools should be able to read from mac addr 0
3223 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(0), UINT32_MAX);
3224 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(0), UINT32_MAX);
3226 /* PFVLVF, PFVLVFB: set up filters for vlan tags as configured */
3227 for (i = 0; i < cfg->nb_pool_maps; i++) {
3228 /* set vlan id in VF register and set the valid bit */
3229 IXGBE_WRITE_REG(hw, IXGBE_VLVF(i), (IXGBE_VLVF_VIEN | \
3230 (cfg->pool_map[i].vlan_id & IXGBE_RXD_VLAN_ID_MASK)));
3232 * Put the allowed pools in VFB reg. As we only have 16 or 64
3233 * pools, we only need to use the first half of the register
3236 if (((cfg->pool_map[i].pools >> 32) & UINT32_MAX) == 0)
3237 IXGBE_WRITE_REG(hw, IXGBE_VLVFB(i*2), \
3238 (cfg->pool_map[i].pools & UINT32_MAX));
3240 IXGBE_WRITE_REG(hw, IXGBE_VLVFB((i*2+1)), \
3241 ((cfg->pool_map[i].pools >> 32) \
3246 /* PFDMA Tx General Switch Control Enables VMDQ loopback */
3247 if (cfg->enable_loop_back) {
3248 IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN);
3249 for (i = 0; i < RTE_IXGBE_VMTXSW_REGISTER_COUNT; i++)
3250 IXGBE_WRITE_REG(hw, IXGBE_VMTXSW(i), UINT32_MAX);
3253 IXGBE_WRITE_FLUSH(hw);
3257 * ixgbe_dcb_config_tx_hw_config - Configure general VMDq TX parameters
3258 * @hw: pointer to hardware structure
3261 ixgbe_vmdq_tx_hw_configure(struct ixgbe_hw *hw)
3266 PMD_INIT_FUNC_TRACE();
3267 /*PF VF Transmit Enable*/
3268 IXGBE_WRITE_REG(hw, IXGBE_VFTE(0), UINT32_MAX);
3269 IXGBE_WRITE_REG(hw, IXGBE_VFTE(1), UINT32_MAX);
3271 /* Disable the Tx desc arbiter so that MTQC can be changed */
3272 reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3273 reg |= IXGBE_RTTDCS_ARBDIS;
3274 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg);
3276 reg = IXGBE_MTQC_VT_ENA | IXGBE_MTQC_64VF;
3277 IXGBE_WRITE_REG(hw, IXGBE_MTQC, reg);
3279 /* Disable drop for all queues */
3280 for (q = 0; q < IXGBE_MAX_RX_QUEUE_NUM; q++)
3281 IXGBE_WRITE_REG(hw, IXGBE_QDE,
3282 (IXGBE_QDE_WRITE | (q << IXGBE_QDE_IDX_SHIFT)));
3284 /* Enable the Tx desc arbiter */
3285 reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3286 reg &= ~IXGBE_RTTDCS_ARBDIS;
3287 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg);
3289 IXGBE_WRITE_FLUSH(hw);
3295 ixgbe_alloc_rx_queue_mbufs(struct ixgbe_rx_queue *rxq)
3297 struct ixgbe_rx_entry *rxe = rxq->sw_ring;
3301 /* Initialize software ring entries */
3302 for (i = 0; i < rxq->nb_rx_desc; i++) {
3303 volatile union ixgbe_adv_rx_desc *rxd;
3304 struct rte_mbuf *mbuf = rte_rxmbuf_alloc(rxq->mb_pool);
3306 PMD_INIT_LOG(ERR, "RX mbuf alloc failed queue_id=%u",
3307 (unsigned) rxq->queue_id);
3311 rte_mbuf_refcnt_set(mbuf, 1);
3313 mbuf->data_off = RTE_PKTMBUF_HEADROOM;
3315 mbuf->port = rxq->port_id;
3318 rte_cpu_to_le_64(RTE_MBUF_DATA_DMA_ADDR_DEFAULT(mbuf));
3319 rxd = &rxq->rx_ring[i];
3320 rxd->read.hdr_addr = dma_addr;
3321 rxd->read.pkt_addr = dma_addr;
3329 ixgbe_config_vf_rss(struct rte_eth_dev *dev)
3331 struct ixgbe_hw *hw;
3334 ixgbe_rss_configure(dev);
3336 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3338 /* MRQC: enable VF RSS */
3339 mrqc = IXGBE_READ_REG(hw, IXGBE_MRQC);
3340 mrqc &= ~IXGBE_MRQC_MRQE_MASK;
3341 switch (RTE_ETH_DEV_SRIOV(dev).active) {
3343 mrqc |= IXGBE_MRQC_VMDQRSS64EN;
3347 mrqc |= IXGBE_MRQC_VMDQRSS32EN;
3351 PMD_INIT_LOG(ERR, "Invalid pool number in IOV mode with VMDQ RSS");
3355 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
3361 ixgbe_config_vf_default(struct rte_eth_dev *dev)
3363 struct ixgbe_hw *hw =
3364 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3366 switch (RTE_ETH_DEV_SRIOV(dev).active) {
3368 IXGBE_WRITE_REG(hw, IXGBE_MRQC,
3373 IXGBE_WRITE_REG(hw, IXGBE_MRQC,
3374 IXGBE_MRQC_VMDQRT4TCEN);
3378 IXGBE_WRITE_REG(hw, IXGBE_MRQC,
3379 IXGBE_MRQC_VMDQRT8TCEN);
3383 "invalid pool number in IOV mode");
3390 ixgbe_dev_mq_rx_configure(struct rte_eth_dev *dev)
3392 struct ixgbe_hw *hw =
3393 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3395 if (hw->mac.type == ixgbe_mac_82598EB)
3398 if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
3400 * SRIOV inactive scheme
3401 * any DCB/RSS w/o VMDq multi-queue setting
3403 switch (dev->data->dev_conf.rxmode.mq_mode) {
3405 ixgbe_rss_configure(dev);
3408 case ETH_MQ_RX_VMDQ_DCB:
3409 ixgbe_vmdq_dcb_configure(dev);
3412 case ETH_MQ_RX_VMDQ_ONLY:
3413 ixgbe_vmdq_rx_hw_configure(dev);
3416 case ETH_MQ_RX_NONE:
3417 /* if mq_mode is none, disable rss mode.*/
3418 default: ixgbe_rss_disable(dev);
3422 * SRIOV active scheme
3423 * Support RSS together with VMDq & SRIOV
3425 switch (dev->data->dev_conf.rxmode.mq_mode) {
3427 case ETH_MQ_RX_VMDQ_RSS:
3428 ixgbe_config_vf_rss(dev);
3431 /* FIXME if support DCB/RSS together with VMDq & SRIOV */
3432 case ETH_MQ_RX_VMDQ_DCB:
3433 case ETH_MQ_RX_VMDQ_DCB_RSS:
3435 "Could not support DCB with VMDq & SRIOV");
3438 ixgbe_config_vf_default(dev);
3447 ixgbe_dev_mq_tx_configure(struct rte_eth_dev *dev)
3449 struct ixgbe_hw *hw =
3450 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3454 if (hw->mac.type == ixgbe_mac_82598EB)
3457 /* disable arbiter before setting MTQC */
3458 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3459 rttdcs |= IXGBE_RTTDCS_ARBDIS;
3460 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3462 if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
3464 * SRIOV inactive scheme
3465 * any DCB w/o VMDq multi-queue setting
3467 if (dev->data->dev_conf.txmode.mq_mode == ETH_MQ_TX_VMDQ_ONLY)
3468 ixgbe_vmdq_tx_hw_configure(hw);
3470 mtqc = IXGBE_MTQC_64Q_1PB;
3471 IXGBE_WRITE_REG(hw, IXGBE_MTQC, mtqc);
3474 switch (RTE_ETH_DEV_SRIOV(dev).active) {
3477 * SRIOV active scheme
3478 * FIXME if support DCB together with VMDq & SRIOV
3481 mtqc = IXGBE_MTQC_VT_ENA | IXGBE_MTQC_64VF;
3484 mtqc = IXGBE_MTQC_VT_ENA | IXGBE_MTQC_32VF;
3487 mtqc = IXGBE_MTQC_VT_ENA | IXGBE_MTQC_RT_ENA |
3491 mtqc = IXGBE_MTQC_64Q_1PB;
3492 PMD_INIT_LOG(ERR, "invalid pool number in IOV mode");
3494 IXGBE_WRITE_REG(hw, IXGBE_MTQC, mtqc);
3497 /* re-enable arbiter */
3498 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
3499 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3504 void ixgbe_set_rx_function(struct rte_eth_dev *dev)
3506 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3509 * In order to allow Vector Rx there are a few configuration
3510 * conditions to be met and Rx Bulk Allocation should be allowed.
3512 if (ixgbe_rx_vec_dev_conf_condition_check(dev) ||
3513 !hw->rx_bulk_alloc_allowed) {
3514 PMD_INIT_LOG(DEBUG, "Port[%d] doesn't meet Vector Rx "
3515 "preconditions or RTE_IXGBE_INC_VECTOR is "
3517 dev->data->port_id);
3519 hw->rx_vec_allowed = false;
3522 if (dev->data->scattered_rx) {
3524 * Set the non-LRO scattered callback: there are Vector and
3525 * single allocation versions.
3527 if (hw->rx_vec_allowed) {
3528 PMD_INIT_LOG(DEBUG, "Using Vector Scattered Rx "
3529 "callback (port=%d).",
3530 dev->data->port_id);
3532 dev->rx_pkt_burst = ixgbe_recv_scattered_pkts_vec;
3534 PMD_INIT_LOG(DEBUG, "Using Regualr (non-vector) "
3535 "Scattered Rx callback "
3537 dev->data->port_id);
3539 dev->rx_pkt_burst = ixgbe_recv_scattered_pkts;
3542 * Below we set "simple" callbacks according to port/queues parameters.
3543 * If parameters allow we are going to choose between the following
3547 * - Single buffer allocation (the simplest one)
3549 } else if (hw->rx_vec_allowed) {
3550 PMD_INIT_LOG(INFO, "Vector rx enabled, please make sure RX "
3551 "burst size no less than 32.");
3553 dev->rx_pkt_burst = ixgbe_recv_pkts_vec;
3554 } else if (hw->rx_bulk_alloc_allowed) {
3555 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are "
3556 "satisfied. Rx Burst Bulk Alloc function "
3557 "will be used on port=%d.",
3558 dev->data->port_id);
3560 dev->rx_pkt_burst = ixgbe_recv_pkts_bulk_alloc;
3562 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are not "
3563 "satisfied, or Scattered Rx is requested, "
3564 "or RTE_LIBRTE_IXGBE_RX_ALLOW_BULK_ALLOC "
3565 "is not enabled (port=%d).",
3566 dev->data->port_id);
3568 dev->rx_pkt_burst = ixgbe_recv_pkts;
3573 * Initializes Receive Unit.
3576 ixgbe_dev_rx_init(struct rte_eth_dev *dev)
3578 struct ixgbe_hw *hw;
3579 struct ixgbe_rx_queue *rxq;
3580 struct rte_pktmbuf_pool_private *mbp_priv;
3591 struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode;
3593 PMD_INIT_FUNC_TRACE();
3594 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3597 * Make sure receives are disabled while setting
3598 * up the RX context (registers, descriptor rings, etc.).
3600 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
3601 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl & ~IXGBE_RXCTRL_RXEN);
3603 /* Enable receipt of broadcasted frames */
3604 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
3605 fctrl |= IXGBE_FCTRL_BAM;
3606 fctrl |= IXGBE_FCTRL_DPF;
3607 fctrl |= IXGBE_FCTRL_PMCF;
3608 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
3611 * Configure CRC stripping, if any.
3613 hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
3614 if (rx_conf->hw_strip_crc)
3615 hlreg0 |= IXGBE_HLREG0_RXCRCSTRP;
3617 hlreg0 &= ~IXGBE_HLREG0_RXCRCSTRP;
3620 * Configure jumbo frame support, if any.
3622 if (rx_conf->jumbo_frame == 1) {
3623 hlreg0 |= IXGBE_HLREG0_JUMBOEN;
3624 maxfrs = IXGBE_READ_REG(hw, IXGBE_MAXFRS);
3625 maxfrs &= 0x0000FFFF;
3626 maxfrs |= (rx_conf->max_rx_pkt_len << 16);
3627 IXGBE_WRITE_REG(hw, IXGBE_MAXFRS, maxfrs);
3629 hlreg0 &= ~IXGBE_HLREG0_JUMBOEN;
3632 * If loopback mode is configured for 82599, set LPBK bit.
3634 if (hw->mac.type == ixgbe_mac_82599EB &&
3635 dev->data->dev_conf.lpbk_mode == IXGBE_LPBK_82599_TX_RX)
3636 hlreg0 |= IXGBE_HLREG0_LPBK;
3638 hlreg0 &= ~IXGBE_HLREG0_LPBK;
3640 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);
3642 /* Setup RX queues */
3643 for (i = 0; i < dev->data->nb_rx_queues; i++) {
3644 rxq = dev->data->rx_queues[i];
3647 * Reset crc_len in case it was changed after queue setup by a
3648 * call to configure.
3650 rxq->crc_len = rx_conf->hw_strip_crc ? 0 : ETHER_CRC_LEN;
3652 /* Setup the Base and Length of the Rx Descriptor Rings */
3653 bus_addr = rxq->rx_ring_phys_addr;
3654 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(rxq->reg_idx),
3655 (uint32_t)(bus_addr & 0x00000000ffffffffULL));
3656 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(rxq->reg_idx),
3657 (uint32_t)(bus_addr >> 32));
3658 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(rxq->reg_idx),
3659 rxq->nb_rx_desc * sizeof(union ixgbe_adv_rx_desc));
3660 IXGBE_WRITE_REG(hw, IXGBE_RDH(rxq->reg_idx), 0);
3661 IXGBE_WRITE_REG(hw, IXGBE_RDT(rxq->reg_idx), 0);
3663 /* Configure the SRRCTL register */
3664 #ifdef RTE_HEADER_SPLIT_ENABLE
3666 * Configure Header Split
3668 if (rx_conf->header_split) {
3669 if (hw->mac.type == ixgbe_mac_82599EB) {
3670 /* Must setup the PSRTYPE register */
3672 psrtype = IXGBE_PSRTYPE_TCPHDR |
3673 IXGBE_PSRTYPE_UDPHDR |
3674 IXGBE_PSRTYPE_IPV4HDR |
3675 IXGBE_PSRTYPE_IPV6HDR;
3676 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(rxq->reg_idx), psrtype);
3678 srrctl = ((rx_conf->split_hdr_size <<
3679 IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) &
3680 IXGBE_SRRCTL_BSIZEHDR_MASK);
3681 srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
3684 srrctl = IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
3686 /* Set if packets are dropped when no descriptors available */
3688 srrctl |= IXGBE_SRRCTL_DROP_EN;
3691 * Configure the RX buffer size in the BSIZEPACKET field of
3692 * the SRRCTL register of the queue.
3693 * The value is in 1 KB resolution. Valid values can be from
3696 mbp_priv = rte_mempool_get_priv(rxq->mb_pool);
3697 buf_size = (uint16_t) (mbp_priv->mbuf_data_room_size -
3698 RTE_PKTMBUF_HEADROOM);
3699 srrctl |= ((buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) &
3700 IXGBE_SRRCTL_BSIZEPKT_MASK);
3701 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxq->reg_idx), srrctl);
3703 buf_size = (uint16_t) ((srrctl & IXGBE_SRRCTL_BSIZEPKT_MASK) <<
3704 IXGBE_SRRCTL_BSIZEPKT_SHIFT);
3706 /* It adds dual VLAN length for supporting dual VLAN */
3707 if (dev->data->dev_conf.rxmode.max_rx_pkt_len +
3708 2 * IXGBE_VLAN_TAG_SIZE > buf_size)
3709 dev->data->scattered_rx = 1;
3712 if (rx_conf->enable_scatter)
3713 dev->data->scattered_rx = 1;
3715 ixgbe_set_rx_function(dev);
3718 * Device configured with multiple RX queues.
3720 ixgbe_dev_mq_rx_configure(dev);
3723 * Setup the Checksum Register.
3724 * Disable Full-Packet Checksum which is mutually exclusive with RSS.
3725 * Enable IP/L4 checkum computation by hardware if requested to do so.
3727 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
3728 rxcsum |= IXGBE_RXCSUM_PCSD;
3729 if (rx_conf->hw_ip_checksum)
3730 rxcsum |= IXGBE_RXCSUM_IPPCSE;
3732 rxcsum &= ~IXGBE_RXCSUM_IPPCSE;
3734 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
3736 if (hw->mac.type == ixgbe_mac_82599EB ||
3737 hw->mac.type == ixgbe_mac_X540) {
3738 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3739 if (rx_conf->hw_strip_crc)
3740 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3742 rdrxctl &= ~IXGBE_RDRXCTL_CRCSTRIP;
3743 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3744 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3751 * Initializes Transmit Unit.
3754 ixgbe_dev_tx_init(struct rte_eth_dev *dev)
3756 struct ixgbe_hw *hw;
3757 struct ixgbe_tx_queue *txq;
3763 PMD_INIT_FUNC_TRACE();
3764 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3766 /* Enable TX CRC (checksum offload requirement) and hw padding
3767 * (TSO requirement) */
3768 hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
3769 hlreg0 |= (IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN);
3770 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);
3772 /* Setup the Base and Length of the Tx Descriptor Rings */
3773 for (i = 0; i < dev->data->nb_tx_queues; i++) {
3774 txq = dev->data->tx_queues[i];
3776 bus_addr = txq->tx_ring_phys_addr;
3777 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(txq->reg_idx),
3778 (uint32_t)(bus_addr & 0x00000000ffffffffULL));
3779 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(txq->reg_idx),
3780 (uint32_t)(bus_addr >> 32));
3781 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(txq->reg_idx),
3782 txq->nb_tx_desc * sizeof(union ixgbe_adv_tx_desc));
3783 /* Setup the HW Tx Head and TX Tail descriptor pointers */
3784 IXGBE_WRITE_REG(hw, IXGBE_TDH(txq->reg_idx), 0);
3785 IXGBE_WRITE_REG(hw, IXGBE_TDT(txq->reg_idx), 0);
3788 * Disable Tx Head Writeback RO bit, since this hoses
3789 * bookkeeping if things aren't delivered in order.
3791 switch (hw->mac.type) {
3792 case ixgbe_mac_82598EB:
3793 txctrl = IXGBE_READ_REG(hw,
3794 IXGBE_DCA_TXCTRL(txq->reg_idx));
3795 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
3796 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(txq->reg_idx),
3800 case ixgbe_mac_82599EB:
3801 case ixgbe_mac_X540:
3802 case ixgbe_mac_X550:
3803 case ixgbe_mac_X550EM_x:
3805 txctrl = IXGBE_READ_REG(hw,
3806 IXGBE_DCA_TXCTRL_82599(txq->reg_idx));
3807 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
3808 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(txq->reg_idx),
3814 /* Device configured with multiple TX queues. */
3815 ixgbe_dev_mq_tx_configure(dev);
3819 * Set up link for 82599 loopback mode Tx->Rx.
3822 ixgbe_setup_loopback_link_82599(struct ixgbe_hw *hw)
3824 PMD_INIT_FUNC_TRACE();
3826 if (ixgbe_verify_lesm_fw_enabled_82599(hw)) {
3827 if (hw->mac.ops.acquire_swfw_sync(hw, IXGBE_GSSR_MAC_CSR_SM) !=
3829 PMD_INIT_LOG(ERR, "Could not enable loopback mode");
3838 IXGBE_AUTOC_LMS_10G_LINK_NO_AN | IXGBE_AUTOC_FLU);
3839 ixgbe_reset_pipeline_82599(hw);
3841 hw->mac.ops.release_swfw_sync(hw, IXGBE_GSSR_MAC_CSR_SM);
3847 * Start Transmit and Receive Units.
3850 ixgbe_dev_rxtx_start(struct rte_eth_dev *dev)
3852 struct ixgbe_hw *hw;
3853 struct ixgbe_tx_queue *txq;
3854 struct ixgbe_rx_queue *rxq;
3861 PMD_INIT_FUNC_TRACE();
3862 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3864 for (i = 0; i < dev->data->nb_tx_queues; i++) {
3865 txq = dev->data->tx_queues[i];
3866 /* Setup Transmit Threshold Registers */
3867 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txq->reg_idx));
3868 txdctl |= txq->pthresh & 0x7F;
3869 txdctl |= ((txq->hthresh & 0x7F) << 8);
3870 txdctl |= ((txq->wthresh & 0x7F) << 16);
3871 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txq->reg_idx), txdctl);
3874 if (hw->mac.type != ixgbe_mac_82598EB) {
3875 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
3876 dmatxctl |= IXGBE_DMATXCTL_TE;
3877 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
3880 for (i = 0; i < dev->data->nb_tx_queues; i++) {
3881 txq = dev->data->tx_queues[i];
3882 if (!txq->tx_deferred_start) {
3883 ret = ixgbe_dev_tx_queue_start(dev, i);
3889 for (i = 0; i < dev->data->nb_rx_queues; i++) {
3890 rxq = dev->data->rx_queues[i];
3891 if (!rxq->rx_deferred_start) {
3892 ret = ixgbe_dev_rx_queue_start(dev, i);
3898 /* Enable Receive engine */
3899 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
3900 if (hw->mac.type == ixgbe_mac_82598EB)
3901 rxctrl |= IXGBE_RXCTRL_DMBYPS;
3902 rxctrl |= IXGBE_RXCTRL_RXEN;
3903 hw->mac.ops.enable_rx_dma(hw, rxctrl);
3905 /* If loopback mode is enabled for 82599, set up the link accordingly */
3906 if (hw->mac.type == ixgbe_mac_82599EB &&
3907 dev->data->dev_conf.lpbk_mode == IXGBE_LPBK_82599_TX_RX)
3908 ixgbe_setup_loopback_link_82599(hw);
3914 * Start Receive Units for specified queue.
3917 ixgbe_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
3919 struct ixgbe_hw *hw;
3920 struct ixgbe_rx_queue *rxq;
3924 PMD_INIT_FUNC_TRACE();
3925 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3927 if (rx_queue_id < dev->data->nb_rx_queues) {
3928 rxq = dev->data->rx_queues[rx_queue_id];
3930 /* Allocate buffers for descriptor rings */
3931 if (ixgbe_alloc_rx_queue_mbufs(rxq) != 0) {
3932 PMD_INIT_LOG(ERR, "Could not alloc mbuf for queue:%d",
3936 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
3937 rxdctl |= IXGBE_RXDCTL_ENABLE;
3938 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxq->reg_idx), rxdctl);
3940 /* Wait until RX Enable ready */
3941 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
3944 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
3945 } while (--poll_ms && !(rxdctl & IXGBE_RXDCTL_ENABLE));
3947 PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d",
3950 IXGBE_WRITE_REG(hw, IXGBE_RDH(rxq->reg_idx), 0);
3951 IXGBE_WRITE_REG(hw, IXGBE_RDT(rxq->reg_idx), rxq->nb_rx_desc - 1);
3959 * Stop Receive Units for specified queue.
3962 ixgbe_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
3964 struct ixgbe_hw *hw;
3965 struct ixgbe_rx_queue *rxq;
3969 PMD_INIT_FUNC_TRACE();
3970 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3972 if (rx_queue_id < dev->data->nb_rx_queues) {
3973 rxq = dev->data->rx_queues[rx_queue_id];
3975 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
3976 rxdctl &= ~IXGBE_RXDCTL_ENABLE;
3977 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxq->reg_idx), rxdctl);
3979 /* Wait until RX Enable ready */
3980 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
3983 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
3984 } while (--poll_ms && (rxdctl | IXGBE_RXDCTL_ENABLE));
3986 PMD_INIT_LOG(ERR, "Could not disable Rx Queue %d",
3989 rte_delay_us(RTE_IXGBE_WAIT_100_US);
3991 ixgbe_rx_queue_release_mbufs(rxq);
3992 ixgbe_reset_rx_queue(hw, rxq);
4001 * Start Transmit Units for specified queue.
4004 ixgbe_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
4006 struct ixgbe_hw *hw;
4007 struct ixgbe_tx_queue *txq;
4011 PMD_INIT_FUNC_TRACE();
4012 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4014 if (tx_queue_id < dev->data->nb_tx_queues) {
4015 txq = dev->data->tx_queues[tx_queue_id];
4016 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txq->reg_idx));
4017 txdctl |= IXGBE_TXDCTL_ENABLE;
4018 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txq->reg_idx), txdctl);
4020 /* Wait until TX Enable ready */
4021 if (hw->mac.type == ixgbe_mac_82599EB) {
4022 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
4025 txdctl = IXGBE_READ_REG(hw,
4026 IXGBE_TXDCTL(txq->reg_idx));
4027 } while (--poll_ms && !(txdctl & IXGBE_TXDCTL_ENABLE));
4029 PMD_INIT_LOG(ERR, "Could not enable "
4030 "Tx Queue %d", tx_queue_id);
4033 IXGBE_WRITE_REG(hw, IXGBE_TDH(txq->reg_idx), 0);
4034 IXGBE_WRITE_REG(hw, IXGBE_TDT(txq->reg_idx), 0);
4042 * Stop Transmit Units for specified queue.
4045 ixgbe_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
4047 struct ixgbe_hw *hw;
4048 struct ixgbe_tx_queue *txq;
4050 uint32_t txtdh, txtdt;
4053 PMD_INIT_FUNC_TRACE();
4054 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4056 if (tx_queue_id < dev->data->nb_tx_queues) {
4057 txq = dev->data->tx_queues[tx_queue_id];
4059 /* Wait until TX queue is empty */
4060 if (hw->mac.type == ixgbe_mac_82599EB) {
4061 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
4063 rte_delay_us(RTE_IXGBE_WAIT_100_US);
4064 txtdh = IXGBE_READ_REG(hw,
4065 IXGBE_TDH(txq->reg_idx));
4066 txtdt = IXGBE_READ_REG(hw,
4067 IXGBE_TDT(txq->reg_idx));
4068 } while (--poll_ms && (txtdh != txtdt));
4070 PMD_INIT_LOG(ERR, "Tx Queue %d is not empty "
4071 "when stopping.", tx_queue_id);
4074 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txq->reg_idx));
4075 txdctl &= ~IXGBE_TXDCTL_ENABLE;
4076 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txq->reg_idx), txdctl);
4078 /* Wait until TX Enable ready */
4079 if (hw->mac.type == ixgbe_mac_82599EB) {
4080 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
4083 txdctl = IXGBE_READ_REG(hw,
4084 IXGBE_TXDCTL(txq->reg_idx));
4085 } while (--poll_ms && (txdctl | IXGBE_TXDCTL_ENABLE));
4087 PMD_INIT_LOG(ERR, "Could not disable "
4088 "Tx Queue %d", tx_queue_id);
4091 if (txq->ops != NULL) {
4092 txq->ops->release_mbufs(txq);
4093 txq->ops->reset(txq);
4102 * [VF] Initializes Receive Unit.
4105 ixgbevf_dev_rx_init(struct rte_eth_dev *dev)
4107 struct ixgbe_hw *hw;
4108 struct ixgbe_rx_queue *rxq;
4109 struct rte_pktmbuf_pool_private *mbp_priv;
4111 uint32_t srrctl, psrtype = 0;
4116 PMD_INIT_FUNC_TRACE();
4117 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4119 if (rte_is_power_of_2(dev->data->nb_rx_queues) == 0) {
4120 PMD_INIT_LOG(ERR, "The number of Rx queue invalid, "
4121 "it should be power of 2");
4125 if (dev->data->nb_rx_queues > hw->mac.max_rx_queues) {
4126 PMD_INIT_LOG(ERR, "The number of Rx queue invalid, "
4127 "it should be equal to or less than %d",
4128 hw->mac.max_rx_queues);
4133 * When the VF driver issues a IXGBE_VF_RESET request, the PF driver
4134 * disables the VF receipt of packets if the PF MTU is > 1500.
4135 * This is done to deal with 82599 limitations that imposes
4136 * the PF and all VFs to share the same MTU.
4137 * Then, the PF driver enables again the VF receipt of packet when
4138 * the VF driver issues a IXGBE_VF_SET_LPE request.
4139 * In the meantime, the VF device cannot be used, even if the VF driver
4140 * and the Guest VM network stack are ready to accept packets with a
4141 * size up to the PF MTU.
4142 * As a work-around to this PF behaviour, force the call to
4143 * ixgbevf_rlpml_set_vf even if jumbo frames are not used. This way,
4144 * VF packets received can work in all cases.
4146 ixgbevf_rlpml_set_vf(hw,
4147 (uint16_t)dev->data->dev_conf.rxmode.max_rx_pkt_len);
4149 /* Setup RX queues */
4150 dev->rx_pkt_burst = ixgbe_recv_pkts;
4151 for (i = 0; i < dev->data->nb_rx_queues; i++) {
4152 rxq = dev->data->rx_queues[i];
4154 /* Allocate buffers for descriptor rings */
4155 ret = ixgbe_alloc_rx_queue_mbufs(rxq);
4159 /* Setup the Base and Length of the Rx Descriptor Rings */
4160 bus_addr = rxq->rx_ring_phys_addr;
4162 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(i),
4163 (uint32_t)(bus_addr & 0x00000000ffffffffULL));
4164 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(i),
4165 (uint32_t)(bus_addr >> 32));
4166 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(i),
4167 rxq->nb_rx_desc * sizeof(union ixgbe_adv_rx_desc));
4168 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(i), 0);
4169 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(i), 0);
4172 /* Configure the SRRCTL register */
4173 #ifdef RTE_HEADER_SPLIT_ENABLE
4175 * Configure Header Split
4177 if (dev->data->dev_conf.rxmode.header_split) {
4178 srrctl = ((dev->data->dev_conf.rxmode.split_hdr_size <<
4179 IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) &
4180 IXGBE_SRRCTL_BSIZEHDR_MASK);
4181 srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
4184 srrctl = IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
4186 /* Set if packets are dropped when no descriptors available */
4188 srrctl |= IXGBE_SRRCTL_DROP_EN;
4191 * Configure the RX buffer size in the BSIZEPACKET field of
4192 * the SRRCTL register of the queue.
4193 * The value is in 1 KB resolution. Valid values can be from
4196 mbp_priv = rte_mempool_get_priv(rxq->mb_pool);
4197 buf_size = (uint16_t) (mbp_priv->mbuf_data_room_size -
4198 RTE_PKTMBUF_HEADROOM);
4199 srrctl |= ((buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) &
4200 IXGBE_SRRCTL_BSIZEPKT_MASK);
4203 * VF modification to write virtual function SRRCTL register
4205 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(i), srrctl);
4207 buf_size = (uint16_t) ((srrctl & IXGBE_SRRCTL_BSIZEPKT_MASK) <<
4208 IXGBE_SRRCTL_BSIZEPKT_SHIFT);
4210 if (dev->data->dev_conf.rxmode.enable_scatter ||
4211 /* It adds dual VLAN length for supporting dual VLAN */
4212 (dev->data->dev_conf.rxmode.max_rx_pkt_len +
4213 2 * IXGBE_VLAN_TAG_SIZE) > buf_size) {
4214 if (!dev->data->scattered_rx)
4215 PMD_INIT_LOG(DEBUG, "forcing scatter mode");
4216 dev->data->scattered_rx = 1;
4217 #ifdef RTE_IXGBE_INC_VECTOR
4218 if (rte_is_power_of_2(rxq->nb_rx_desc))
4220 ixgbe_recv_scattered_pkts_vec;
4223 dev->rx_pkt_burst = ixgbe_recv_scattered_pkts;
4227 #ifdef RTE_HEADER_SPLIT_ENABLE
4228 if (dev->data->dev_conf.rxmode.header_split)
4229 /* Must setup the PSRTYPE register */
4230 psrtype = IXGBE_PSRTYPE_TCPHDR |
4231 IXGBE_PSRTYPE_UDPHDR |
4232 IXGBE_PSRTYPE_IPV4HDR |
4233 IXGBE_PSRTYPE_IPV6HDR;
4236 /* Set RQPL for VF RSS according to max Rx queue */
4237 psrtype |= (dev->data->nb_rx_queues >> 1) <<
4238 IXGBE_PSRTYPE_RQPL_SHIFT;
4239 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype);
4245 * [VF] Initializes Transmit Unit.
4248 ixgbevf_dev_tx_init(struct rte_eth_dev *dev)
4250 struct ixgbe_hw *hw;
4251 struct ixgbe_tx_queue *txq;
4256 PMD_INIT_FUNC_TRACE();
4257 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4259 /* Setup the Base and Length of the Tx Descriptor Rings */
4260 for (i = 0; i < dev->data->nb_tx_queues; i++) {
4261 txq = dev->data->tx_queues[i];
4262 bus_addr = txq->tx_ring_phys_addr;
4263 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(i),
4264 (uint32_t)(bus_addr & 0x00000000ffffffffULL));
4265 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(i),
4266 (uint32_t)(bus_addr >> 32));
4267 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(i),
4268 txq->nb_tx_desc * sizeof(union ixgbe_adv_tx_desc));
4269 /* Setup the HW Tx Head and TX Tail descriptor pointers */
4270 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(i), 0);
4271 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(i), 0);
4274 * Disable Tx Head Writeback RO bit, since this hoses
4275 * bookkeeping if things aren't delivered in order.
4277 txctrl = IXGBE_READ_REG(hw,
4278 IXGBE_VFDCA_TXCTRL(i));
4279 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
4280 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(i),
4286 * [VF] Start Transmit and Receive Units.
4289 ixgbevf_dev_rxtx_start(struct rte_eth_dev *dev)
4291 struct ixgbe_hw *hw;
4292 struct ixgbe_tx_queue *txq;
4293 struct ixgbe_rx_queue *rxq;
4299 PMD_INIT_FUNC_TRACE();
4300 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4302 for (i = 0; i < dev->data->nb_tx_queues; i++) {
4303 txq = dev->data->tx_queues[i];
4304 /* Setup Transmit Threshold Registers */
4305 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i));
4306 txdctl |= txq->pthresh & 0x7F;
4307 txdctl |= ((txq->hthresh & 0x7F) << 8);
4308 txdctl |= ((txq->wthresh & 0x7F) << 16);
4309 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl);
4312 for (i = 0; i < dev->data->nb_tx_queues; i++) {
4314 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i));
4315 txdctl |= IXGBE_TXDCTL_ENABLE;
4316 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl);
4319 /* Wait until TX Enable ready */
4322 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i));
4323 } while (--poll_ms && !(txdctl & IXGBE_TXDCTL_ENABLE));
4325 PMD_INIT_LOG(ERR, "Could not enable Tx Queue %d", i);
4327 for (i = 0; i < dev->data->nb_rx_queues; i++) {
4329 rxq = dev->data->rx_queues[i];
4331 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i));
4332 rxdctl |= IXGBE_RXDCTL_ENABLE;
4333 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), rxdctl);
4335 /* Wait until RX Enable ready */
4339 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i));
4340 } while (--poll_ms && !(rxdctl & IXGBE_RXDCTL_ENABLE));
4342 PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", i);
4344 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(i), rxq->nb_rx_desc - 1);
4349 /* Stubs needed for linkage when CONFIG_RTE_IXGBE_INC_VECTOR is set to 'n' */
4350 int __attribute__((weak))
4351 ixgbe_rx_vec_dev_conf_condition_check(struct rte_eth_dev __rte_unused *dev)
4356 uint16_t __attribute__((weak))
4357 ixgbe_recv_pkts_vec(
4358 void __rte_unused *rx_queue,
4359 struct rte_mbuf __rte_unused **rx_pkts,
4360 uint16_t __rte_unused nb_pkts)
4365 uint16_t __attribute__((weak))
4366 ixgbe_recv_scattered_pkts_vec(
4367 void __rte_unused *rx_queue,
4368 struct rte_mbuf __rte_unused **rx_pkts,
4369 uint16_t __rte_unused nb_pkts)
4374 int __attribute__((weak))
4375 ixgbe_rxq_vec_setup(struct ixgbe_rx_queue __rte_unused *rxq)