drivers/net/ixgbe/ixgbe_rxtx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2016 Intel Corporation.
   3  * Copyright 2014 6WIND S.A.
   4  */
   5
   6 #include <sys/queue.h>
   7
   8 #include <stdio.h>
   9 #include <stdlib.h>
  10 #include <string.h>
  11 #include <errno.h>
  12 #include <stdint.h>
  13 #include <stdarg.h>
  14 #include <unistd.h>
  15 #include <inttypes.h>
  16
  17 #include <rte_byteorder.h>
  18 #include <rte_common.h>
  19 #include <rte_cycles.h>
  20 #include <rte_log.h>
  21 #include <rte_debug.h>
  22 #include <rte_interrupts.h>
  23 #include <rte_pci.h>
  24 #include <rte_memory.h>
  25 #include <rte_memzone.h>
  26 #include <rte_launch.h>
  27 #include <rte_eal.h>
  28 #include <rte_per_lcore.h>
  29 #include <rte_lcore.h>
  30 #include <rte_atomic.h>
  31 #include <rte_branch_prediction.h>
  32 #include <rte_mempool.h>
  33 #include <rte_malloc.h>
  34 #include <rte_mbuf.h>
  35 #include <rte_ether.h>
  36 #include <rte_ethdev_driver.h>
  37 #include <rte_prefetch.h>
  38 #include <rte_udp.h>
  39 #include <rte_tcp.h>
  40 #include <rte_sctp.h>
  41 #include <rte_string_fns.h>
  42 #include <rte_errno.h>
  43 #include <rte_ip.h>
  44 #include <rte_net.h>
  45
  46 #include "ixgbe_logs.h"
  47 #include "base/ixgbe_api.h"
  48 #include "base/ixgbe_vf.h"
  49 #include "ixgbe_ethdev.h"
  50 #include "base/ixgbe_dcb.h"
  51 #include "base/ixgbe_common.h"
  52 #include "ixgbe_rxtx.h"
  53
  54 #ifdef RTE_LIBRTE_IEEE1588
  55 #define IXGBE_TX_IEEE1588_TMST PKT_TX_IEEE1588_TMST
  56 #else
  57 #define IXGBE_TX_IEEE1588_TMST 0
  58 #endif
  59 /* Bit Mask to indicate what bits required for building TX context */
  60 #define IXGBE_TX_OFFLOAD_MASK (                  \
  61                 PKT_TX_VLAN_PKT |                \
  62                 PKT_TX_IP_CKSUM |                \
  63                 PKT_TX_L4_MASK |                 \
  64                 PKT_TX_TCP_SEG |                 \
  65                 PKT_TX_MACSEC |                  \
  66                 PKT_TX_OUTER_IP_CKSUM |          \
  67                 PKT_TX_SEC_OFFLOAD |     \
  68                 IXGBE_TX_IEEE1588_TMST)
  69
  70 #define IXGBE_TX_OFFLOAD_NOTSUP_MASK \
  71                 (PKT_TX_OFFLOAD_MASK ^ IXGBE_TX_OFFLOAD_MASK)
  72
  73 #if 1
  74 #define RTE_PMD_USE_PREFETCH
  75 #endif
  76
  77 #ifdef RTE_PMD_USE_PREFETCH
  78 /*
  79  * Prefetch a cache line into all cache levels.
  80  */
  81 #define rte_ixgbe_prefetch(p)   rte_prefetch0(p)
  82 #else
  83 #define rte_ixgbe_prefetch(p)   do {} while (0)
  84 #endif
  85
  86 #ifdef RTE_IXGBE_INC_VECTOR
  87 uint16_t ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
  88                                     uint16_t nb_pkts);
  89 #endif
  90
  91 /*********************************************************************
  92  *
  93  *  TX functions
  94  *
  95  **********************************************************************/
  96
  97 /*
  98  * Check for descriptors with their DD bit set and free mbufs.
  99  * Return the total number of buffers freed.
 100  */
 101 static __rte_always_inline int
 102 ixgbe_tx_free_bufs(struct ixgbe_tx_queue *txq)
 103 {
 104         struct ixgbe_tx_entry *txep;
 105         uint32_t status;
 106         int i, nb_free = 0;
 107         struct rte_mbuf *m, *free[RTE_IXGBE_TX_MAX_FREE_BUF_SZ];
 108
 109         /* check DD bit on threshold descriptor */
 110         status = txq->tx_ring[txq->tx_next_dd].wb.status;
 111         if (!(status & rte_cpu_to_le_32(IXGBE_ADVTXD_STAT_DD)))
 112                 return 0;
 113
 114         /*
 115          * first buffer to free from S/W ring is at index
 116          * tx_next_dd - (tx_rs_thresh-1)
 117          */
 118         txep = &(txq->sw_ring[txq->tx_next_dd - (txq->tx_rs_thresh - 1)]);
 119
 120         for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) {
 121                 /* free buffers one at a time */
 122                 m = rte_pktmbuf_prefree_seg(txep->mbuf);
 123                 txep->mbuf = NULL;
 124
 125                 if (unlikely(m == NULL))
 126                         continue;
 127
 128                 if (nb_free >= RTE_IXGBE_TX_MAX_FREE_BUF_SZ ||
 129                     (nb_free > 0 && m->pool != free[0]->pool)) {
 130                         rte_mempool_put_bulk(free[0]->pool,
 131                                              (void **)free, nb_free);
 132                         nb_free = 0;
 133                 }
 134
 135                 free[nb_free++] = m;
 136         }
 137
 138         if (nb_free > 0)
 139                 rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free);
 140
 141         /* buffers were freed, update counters */
 142         txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh);
 143         txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh);
 144         if (txq->tx_next_dd >= txq->nb_tx_desc)
 145                 txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
 146
 147         return txq->tx_rs_thresh;
 148 }
 149
 150 /* Populate 4 descriptors with data from 4 mbufs */
 151 static inline void
 152 tx4(volatile union ixgbe_adv_tx_desc *txdp, struct rte_mbuf **pkts)
 153 {
 154         uint64_t buf_dma_addr;
 155         uint32_t pkt_len;
 156         int i;
 157
 158         for (i = 0; i < 4; ++i, ++txdp, ++pkts) {
 159                 buf_dma_addr = rte_mbuf_data_iova(*pkts);
 160                 pkt_len = (*pkts)->data_len;
 161
 162                 /* write data to descriptor */
 163                 txdp->read.buffer_addr = rte_cpu_to_le_64(buf_dma_addr);
 164
 165                 txdp->read.cmd_type_len =
 166                         rte_cpu_to_le_32((uint32_t)DCMD_DTYP_FLAGS | pkt_len);
 167
 168                 txdp->read.olinfo_status =
 169                         rte_cpu_to_le_32(pkt_len << IXGBE_ADVTXD_PAYLEN_SHIFT);
 170
 171                 rte_prefetch0(&(*pkts)->pool);
 172         }
 173 }
 174
 175 /* Populate 1 descriptor with data from 1 mbuf */
 176 static inline void
 177 tx1(volatile union ixgbe_adv_tx_desc *txdp, struct rte_mbuf **pkts)
 178 {
 179         uint64_t buf_dma_addr;
 180         uint32_t pkt_len;
 181
 182         buf_dma_addr = rte_mbuf_data_iova(*pkts);
 183         pkt_len = (*pkts)->data_len;
 184
 185         /* write data to descriptor */
 186         txdp->read.buffer_addr = rte_cpu_to_le_64(buf_dma_addr);
 187         txdp->read.cmd_type_len =
 188                         rte_cpu_to_le_32((uint32_t)DCMD_DTYP_FLAGS | pkt_len);
 189         txdp->read.olinfo_status =
 190                         rte_cpu_to_le_32(pkt_len << IXGBE_ADVTXD_PAYLEN_SHIFT);
 191         rte_prefetch0(&(*pkts)->pool);
 192 }
 193
 194 /*
 195  * Fill H/W descriptor ring with mbuf data.
 196  * Copy mbuf pointers to the S/W ring.
 197  */
 198 static inline void
 199 ixgbe_tx_fill_hw_ring(struct ixgbe_tx_queue *txq, struct rte_mbuf **pkts,
 200                       uint16_t nb_pkts)
 201 {
 202         volatile union ixgbe_adv_tx_desc *txdp = &(txq->tx_ring[txq->tx_tail]);
 203         struct ixgbe_tx_entry *txep = &(txq->sw_ring[txq->tx_tail]);
 204         const int N_PER_LOOP = 4;
 205         const int N_PER_LOOP_MASK = N_PER_LOOP-1;
 206         int mainpart, leftover;
 207         int i, j;
 208
 209         /*
 210          * Process most of the packets in chunks of N pkts.  Any
 211          * leftover packets will get processed one at a time.
 212          */
 213         mainpart = (nb_pkts & ((uint32_t) ~N_PER_LOOP_MASK));
 214         leftover = (nb_pkts & ((uint32_t)  N_PER_LOOP_MASK));
 215         for (i = 0; i < mainpart; i += N_PER_LOOP) {
 216                 /* Copy N mbuf pointers to the S/W ring */
 217                 for (j = 0; j < N_PER_LOOP; ++j) {
 218                         (txep + i + j)->mbuf = *(pkts + i + j);
 219                 }
 220                 tx4(txdp + i, pkts + i);
 221         }
 222
 223         if (unlikely(leftover > 0)) {
 224                 for (i = 0; i < leftover; ++i) {
 225                         (txep + mainpart + i)->mbuf = *(pkts + mainpart + i);
 226                         tx1(txdp + mainpart + i, pkts + mainpart + i);
 227                 }
 228         }
 229 }
 230
 231 static inline uint16_t
 232 tx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 233              uint16_t nb_pkts)
 234 {
 235         struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue;
 236         volatile union ixgbe_adv_tx_desc *tx_r = txq->tx_ring;
 237         uint16_t n = 0;
 238
 239         /*
 240          * Begin scanning the H/W ring for done descriptors when the
 241          * number of available descriptors drops below tx_free_thresh.  For
 242          * each done descriptor, free the associated buffer.
 243          */
 244         if (txq->nb_tx_free < txq->tx_free_thresh)
 245                 ixgbe_tx_free_bufs(txq);
 246
 247         /* Only use descriptors that are available */
 248         nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts);
 249         if (unlikely(nb_pkts == 0))
 250                 return 0;
 251
 252         /* Use exactly nb_pkts descriptors */
 253         txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts);
 254
 255         /*
 256          * At this point, we know there are enough descriptors in the
 257          * ring to transmit all the packets.  This assumes that each
 258          * mbuf contains a single segment, and that no new offloads
 259          * are expected, which would require a new context descriptor.
 260          */
 261
 262         /*
 263          * See if we're going to wrap-around. If so, handle the top
 264          * of the descriptor ring first, then do the bottom.  If not,
 265          * the processing looks just like the "bottom" part anyway...
 266          */
 267         if ((txq->tx_tail + nb_pkts) > txq->nb_tx_desc) {
 268                 n = (uint16_t)(txq->nb_tx_desc - txq->tx_tail);
 269                 ixgbe_tx_fill_hw_ring(txq, tx_pkts, n);
 270
 271                 /*
 272                  * We know that the last descriptor in the ring will need to
 273                  * have its RS bit set because tx_rs_thresh has to be
 274                  * a divisor of the ring size
 275                  */
 276                 tx_r[txq->tx_next_rs].read.cmd_type_len |=
 277                         rte_cpu_to_le_32(IXGBE_ADVTXD_DCMD_RS);
 278                 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
 279
 280                 txq->tx_tail = 0;
 281         }
 282
 283         /* Fill H/W descriptor ring with mbuf data */
 284         ixgbe_tx_fill_hw_ring(txq, tx_pkts + n, (uint16_t)(nb_pkts - n));
 285         txq->tx_tail = (uint16_t)(txq->tx_tail + (nb_pkts - n));
 286
 287         /*
 288          * Determine if RS bit should be set
 289          * This is what we actually want:
 290          *   if ((txq->tx_tail - 1) >= txq->tx_next_rs)
 291          * but instead of subtracting 1 and doing >=, we can just do
 292          * greater than without subtracting.
 293          */
 294         if (txq->tx_tail > txq->tx_next_rs) {
 295                 tx_r[txq->tx_next_rs].read.cmd_type_len |=
 296                         rte_cpu_to_le_32(IXGBE_ADVTXD_DCMD_RS);
 297                 txq->tx_next_rs = (uint16_t)(txq->tx_next_rs +
 298                                                 txq->tx_rs_thresh);
 299                 if (txq->tx_next_rs >= txq->nb_tx_desc)
 300                         txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
 301         }
 302
 303         /*
 304          * Check for wrap-around. This would only happen if we used
 305          * up to the last descriptor in the ring, no more, no less.
 306          */
 307         if (txq->tx_tail >= txq->nb_tx_desc)
 308                 txq->tx_tail = 0;
 309
 310         /* update tail pointer */
 311         rte_wmb();
 312         IXGBE_PCI_REG_WRITE_RELAXED(txq->tdt_reg_addr, txq->tx_tail);
 313
 314         return nb_pkts;
 315 }
 316
 317 uint16_t
 318 ixgbe_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts,
 319                        uint16_t nb_pkts)
 320 {
 321         uint16_t nb_tx;
 322
 323         /* Try to transmit at least chunks of TX_MAX_BURST pkts */
 324         if (likely(nb_pkts <= RTE_PMD_IXGBE_TX_MAX_BURST))
 325                 return tx_xmit_pkts(tx_queue, tx_pkts, nb_pkts);
 326
 327         /* transmit more than the max burst, in chunks of TX_MAX_BURST */
 328         nb_tx = 0;
 329         while (nb_pkts) {
 330                 uint16_t ret, n;
 331
 332                 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_IXGBE_TX_MAX_BURST);
 333                 ret = tx_xmit_pkts(tx_queue, &(tx_pkts[nb_tx]), n);
 334                 nb_tx = (uint16_t)(nb_tx + ret);
 335                 nb_pkts = (uint16_t)(nb_pkts - ret);
 336                 if (ret < n)
 337                         break;
 338         }
 339
 340         return nb_tx;
 341 }
 342
 343 #ifdef RTE_IXGBE_INC_VECTOR
 344 static uint16_t
 345 ixgbe_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
 346                     uint16_t nb_pkts)
 347 {
 348         uint16_t nb_tx = 0;
 349         struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue;
 350
 351         while (nb_pkts) {
 352                 uint16_t ret, num;
 353
 354                 num = (uint16_t)RTE_MIN(nb_pkts, txq->tx_rs_thresh);
 355                 ret = ixgbe_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
 356                                                  num);
 357                 nb_tx += ret;
 358                 nb_pkts -= ret;
 359                 if (ret < num)
 360                         break;
 361         }
 362
 363         return nb_tx;
 364 }
 365 #endif
 366
 367 static inline void
 368 ixgbe_set_xmit_ctx(struct ixgbe_tx_queue *txq,
 369                 volatile struct ixgbe_adv_tx_context_desc *ctx_txd,
 370                 uint64_t ol_flags, union ixgbe_tx_offload tx_offload,
 371                 __rte_unused uint64_t *mdata)
 372 {
 373         uint32_t type_tucmd_mlhl;
 374         uint32_t mss_l4len_idx = 0;
 375         uint32_t ctx_idx;
 376         uint32_t vlan_macip_lens;
 377         union ixgbe_tx_offload tx_offload_mask;
 378         uint32_t seqnum_seed = 0;
 379
 380         ctx_idx = txq->ctx_curr;
 381         tx_offload_mask.data[0] = 0;
 382         tx_offload_mask.data[1] = 0;
 383         type_tucmd_mlhl = 0;
 384
 385         /* Specify which HW CTX to upload. */
 386         mss_l4len_idx |= (ctx_idx << IXGBE_ADVTXD_IDX_SHIFT);
 387
 388         if (ol_flags & PKT_TX_VLAN_PKT) {
 389                 tx_offload_mask.vlan_tci |= ~0;
 390         }
 391
 392         /* check if TCP segmentation required for this packet */
 393         if (ol_flags & PKT_TX_TCP_SEG) {
 394                 /* implies IP cksum in IPv4 */
 395                 if (ol_flags & PKT_TX_IP_CKSUM)
 396                         type_tucmd_mlhl = IXGBE_ADVTXD_TUCMD_IPV4 |
 397                                 IXGBE_ADVTXD_TUCMD_L4T_TCP |
 398                                 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
 399                 else
 400                         type_tucmd_mlhl = IXGBE_ADVTXD_TUCMD_IPV6 |
 401                                 IXGBE_ADVTXD_TUCMD_L4T_TCP |
 402                                 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
 403
 404                 tx_offload_mask.l2_len |= ~0;
 405                 tx_offload_mask.l3_len |= ~0;
 406                 tx_offload_mask.l4_len |= ~0;
 407                 tx_offload_mask.tso_segsz |= ~0;
 408                 mss_l4len_idx |= tx_offload.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT;
 409                 mss_l4len_idx |= tx_offload.l4_len << IXGBE_ADVTXD_L4LEN_SHIFT;
 410         } else { /* no TSO, check if hardware checksum is needed */
 411                 if (ol_flags & PKT_TX_IP_CKSUM) {
 412                         type_tucmd_mlhl = IXGBE_ADVTXD_TUCMD_IPV4;
 413                         tx_offload_mask.l2_len |= ~0;
 414                         tx_offload_mask.l3_len |= ~0;
 415                 }
 416
 417                 switch (ol_flags & PKT_TX_L4_MASK) {
 418                 case PKT_TX_UDP_CKSUM:
 419                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP |
 420                                 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
 421                         mss_l4len_idx |= sizeof(struct udp_hdr) << IXGBE_ADVTXD_L4LEN_SHIFT;
 422                         tx_offload_mask.l2_len |= ~0;
 423                         tx_offload_mask.l3_len |= ~0;
 424                         break;
 425                 case PKT_TX_TCP_CKSUM:
 426                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP |
 427                                 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
 428                         mss_l4len_idx |= sizeof(struct tcp_hdr) << IXGBE_ADVTXD_L4LEN_SHIFT;
 429                         tx_offload_mask.l2_len |= ~0;
 430                         tx_offload_mask.l3_len |= ~0;
 431                         break;
 432                 case PKT_TX_SCTP_CKSUM:
 433                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP |
 434                                 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
 435                         mss_l4len_idx |= sizeof(struct sctp_hdr) << IXGBE_ADVTXD_L4LEN_SHIFT;
 436                         tx_offload_mask.l2_len |= ~0;
 437                         tx_offload_mask.l3_len |= ~0;
 438                         break;
 439                 default:
 440                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_RSV |
 441                                 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT;
 442                         break;
 443                 }
 444         }
 445
 446         if (ol_flags & PKT_TX_OUTER_IP_CKSUM) {
 447                 tx_offload_mask.outer_l2_len |= ~0;
 448                 tx_offload_mask.outer_l3_len |= ~0;
 449                 tx_offload_mask.l2_len |= ~0;
 450                 seqnum_seed |= tx_offload.outer_l3_len
 451                                << IXGBE_ADVTXD_OUTER_IPLEN;
 452                 seqnum_seed |= tx_offload.l2_len
 453                                << IXGBE_ADVTXD_TUNNEL_LEN;
 454         }
 455 #ifdef RTE_LIBRTE_SECURITY
 456         if (ol_flags & PKT_TX_SEC_OFFLOAD) {
 457                 union ixgbe_crypto_tx_desc_md *md =
 458                                 (union ixgbe_crypto_tx_desc_md *)mdata;
 459                 seqnum_seed |=
 460                         (IXGBE_ADVTXD_IPSEC_SA_INDEX_MASK & md->sa_idx);
 461                 type_tucmd_mlhl |= md->enc ?
 462                                 (IXGBE_ADVTXD_TUCMD_IPSEC_TYPE_ESP |
 463                                 IXGBE_ADVTXD_TUCMD_IPSEC_ENCRYPT_EN) : 0;
 464                 type_tucmd_mlhl |=
 465                         (md->pad_len & IXGBE_ADVTXD_IPSEC_ESP_LEN_MASK);
 466                 tx_offload_mask.sa_idx |= ~0;
 467                 tx_offload_mask.sec_pad_len |= ~0;
 468         }
 469 #endif
 470
 471         txq->ctx_cache[ctx_idx].flags = ol_flags;
 472         txq->ctx_cache[ctx_idx].tx_offload.data[0]  =
 473                 tx_offload_mask.data[0] & tx_offload.data[0];
 474         txq->ctx_cache[ctx_idx].tx_offload.data[1]  =
 475                 tx_offload_mask.data[1] & tx_offload.data[1];
 476         txq->ctx_cache[ctx_idx].tx_offload_mask    = tx_offload_mask;
 477
 478         ctx_txd->type_tucmd_mlhl = rte_cpu_to_le_32(type_tucmd_mlhl);
 479         vlan_macip_lens = tx_offload.l3_len;
 480         if (ol_flags & PKT_TX_OUTER_IP_CKSUM)
 481                 vlan_macip_lens |= (tx_offload.outer_l2_len <<
 482                                     IXGBE_ADVTXD_MACLEN_SHIFT);
 483         else
 484                 vlan_macip_lens |= (tx_offload.l2_len <<
 485                                     IXGBE_ADVTXD_MACLEN_SHIFT);
 486         vlan_macip_lens |= ((uint32_t)tx_offload.vlan_tci << IXGBE_ADVTXD_VLAN_SHIFT);
 487         ctx_txd->vlan_macip_lens = rte_cpu_to_le_32(vlan_macip_lens);
 488         ctx_txd->mss_l4len_idx   = rte_cpu_to_le_32(mss_l4len_idx);
 489         ctx_txd->seqnum_seed     = seqnum_seed;
 490 }
 491
 492 /*
 493  * Check which hardware context can be used. Use the existing match
 494  * or create a new context descriptor.
 495  */
 496 static inline uint32_t
 497 what_advctx_update(struct ixgbe_tx_queue *txq, uint64_t flags,
 498                    union ixgbe_tx_offload tx_offload)
 499 {
 500         /* If match with the current used context */
 501         if (likely((txq->ctx_cache[txq->ctx_curr].flags == flags) &&
 502                    (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] ==
 503                     (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0]
 504                      & tx_offload.data[0])) &&
 505                    (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] ==
 506                     (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1]
 507                      & tx_offload.data[1]))))
 508                 return txq->ctx_curr;
 509
 510         /* What if match with the next context  */
 511         txq->ctx_curr ^= 1;
 512         if (likely((txq->ctx_cache[txq->ctx_curr].flags == flags) &&
 513                    (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] ==
 514                     (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0]
 515                      & tx_offload.data[0])) &&
 516                    (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] ==
 517                     (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1]
 518                      & tx_offload.data[1]))))
 519                 return txq->ctx_curr;
 520
 521         /* Mismatch, use the previous context */
 522         return IXGBE_CTX_NUM;
 523 }
 524
 525 static inline uint32_t
 526 tx_desc_cksum_flags_to_olinfo(uint64_t ol_flags)
 527 {
 528         uint32_t tmp = 0;
 529
 530         if ((ol_flags & PKT_TX_L4_MASK) != PKT_TX_L4_NO_CKSUM)
 531                 tmp |= IXGBE_ADVTXD_POPTS_TXSM;
 532         if (ol_flags & PKT_TX_IP_CKSUM)
 533                 tmp |= IXGBE_ADVTXD_POPTS_IXSM;
 534         if (ol_flags & PKT_TX_TCP_SEG)
 535                 tmp |= IXGBE_ADVTXD_POPTS_TXSM;
 536         return tmp;
 537 }
 538
 539 static inline uint32_t
 540 tx_desc_ol_flags_to_cmdtype(uint64_t ol_flags)
 541 {
 542         uint32_t cmdtype = 0;
 543
 544         if (ol_flags & PKT_TX_VLAN_PKT)
 545                 cmdtype |= IXGBE_ADVTXD_DCMD_VLE;
 546         if (ol_flags & PKT_TX_TCP_SEG)
 547                 cmdtype |= IXGBE_ADVTXD_DCMD_TSE;
 548         if (ol_flags & PKT_TX_OUTER_IP_CKSUM)
 549                 cmdtype |= (1 << IXGBE_ADVTXD_OUTERIPCS_SHIFT);
 550         if (ol_flags & PKT_TX_MACSEC)
 551                 cmdtype |= IXGBE_ADVTXD_MAC_LINKSEC;
 552         return cmdtype;
 553 }
 554
 555 /* Default RS bit threshold values */
 556 #ifndef DEFAULT_TX_RS_THRESH
 557 #define DEFAULT_TX_RS_THRESH   32
 558 #endif
 559 #ifndef DEFAULT_TX_FREE_THRESH
 560 #define DEFAULT_TX_FREE_THRESH 32
 561 #endif
 562
 563 /* Reset transmit descriptors after they have been used */
 564 static inline int
 565 ixgbe_xmit_cleanup(struct ixgbe_tx_queue *txq)
 566 {
 567         struct ixgbe_tx_entry *sw_ring = txq->sw_ring;
 568         volatile union ixgbe_adv_tx_desc *txr = txq->tx_ring;
 569         uint16_t last_desc_cleaned = txq->last_desc_cleaned;
 570         uint16_t nb_tx_desc = txq->nb_tx_desc;
 571         uint16_t desc_to_clean_to;
 572         uint16_t nb_tx_to_clean;
 573         uint32_t status;
 574
 575         /* Determine the last descriptor needing to be cleaned */
 576         desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->tx_rs_thresh);
 577         if (desc_to_clean_to >= nb_tx_desc)
 578                 desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc);
 579
 580         /* Check to make sure the last descriptor to clean is done */
 581         desc_to_clean_to = sw_ring[desc_to_clean_to].last_id;
 582         status = txr[desc_to_clean_to].wb.status;
 583         if (!(status & rte_cpu_to_le_32(IXGBE_TXD_STAT_DD))) {
 584                 PMD_TX_FREE_LOG(DEBUG,
 585                                 "TX descriptor %4u is not done"
 586                                 "(port=%d queue=%d)",
 587                                 desc_to_clean_to,
 588                                 txq->port_id, txq->queue_id);
 589                 /* Failed to clean any descriptors, better luck next time */
 590                 return -(1);
 591         }
 592
 593         /* Figure out how many descriptors will be cleaned */
 594         if (last_desc_cleaned > desc_to_clean_to)
 595                 nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) +
 596                                                         desc_to_clean_to);
 597         else
 598                 nb_tx_to_clean = (uint16_t)(desc_to_clean_to -
 599                                                 last_desc_cleaned);
 600
 601         PMD_TX_FREE_LOG(DEBUG,
 602                         "Cleaning %4u TX descriptors: %4u to %4u "
 603                         "(port=%d queue=%d)",
 604                         nb_tx_to_clean, last_desc_cleaned, desc_to_clean_to,
 605                         txq->port_id, txq->queue_id);
 606
 607         /*
 608          * The last descriptor to clean is done, so that means all the
 609          * descriptors from the last descriptor that was cleaned
 610          * up to the last descriptor with the RS bit set
 611          * are done. Only reset the threshold descriptor.
 612          */
 613         txr[desc_to_clean_to].wb.status = 0;
 614
 615         /* Update the txq to reflect the last descriptor that was cleaned */
 616         txq->last_desc_cleaned = desc_to_clean_to;
 617         txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + nb_tx_to_clean);
 618
 619         /* No Error */
 620         return 0;
 621 }
 622
 623 uint16_t
 624 ixgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 625                 uint16_t nb_pkts)
 626 {
 627         struct ixgbe_tx_queue *txq;
 628         struct ixgbe_tx_entry *sw_ring;
 629         struct ixgbe_tx_entry *txe, *txn;
 630         volatile union ixgbe_adv_tx_desc *txr;
 631         volatile union ixgbe_adv_tx_desc *txd, *txp;
 632         struct rte_mbuf     *tx_pkt;
 633         struct rte_mbuf     *m_seg;
 634         uint64_t buf_dma_addr;
 635         uint32_t olinfo_status;
 636         uint32_t cmd_type_len;
 637         uint32_t pkt_len;
 638         uint16_t slen;
 639         uint64_t ol_flags;
 640         uint16_t tx_id;
 641         uint16_t tx_last;
 642         uint16_t nb_tx;
 643         uint16_t nb_used;
 644         uint64_t tx_ol_req;
 645         uint32_t ctx = 0;
 646         uint32_t new_ctx;
 647         union ixgbe_tx_offload tx_offload;
 648 #ifdef RTE_LIBRTE_SECURITY
 649         uint8_t use_ipsec;
 650 #endif
 651
 652         tx_offload.data[0] = 0;
 653         tx_offload.data[1] = 0;
 654         txq = tx_queue;
 655         sw_ring = txq->sw_ring;
 656         txr     = txq->tx_ring;
 657         tx_id   = txq->tx_tail;
 658         txe = &sw_ring[tx_id];
 659         txp = NULL;
 660
 661         /* Determine if the descriptor ring needs to be cleaned. */
 662         if (txq->nb_tx_free < txq->tx_free_thresh)
 663                 ixgbe_xmit_cleanup(txq);
 664
 665         rte_prefetch0(&txe->mbuf->pool);
 666
 667         /* TX loop */
 668         for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
 669                 new_ctx = 0;
 670                 tx_pkt = *tx_pkts++;
 671                 pkt_len = tx_pkt->pkt_len;
 672
 673                 /*
 674                  * Determine how many (if any) context descriptors
 675                  * are needed for offload functionality.
 676                  */
 677                 ol_flags = tx_pkt->ol_flags;
 678 #ifdef RTE_LIBRTE_SECURITY
 679                 use_ipsec = txq->using_ipsec && (ol_flags & PKT_TX_SEC_OFFLOAD);
 680 #endif
 681
 682                 /* If hardware offload required */
 683                 tx_ol_req = ol_flags & IXGBE_TX_OFFLOAD_MASK;
 684                 if (tx_ol_req) {
 685                         tx_offload.l2_len = tx_pkt->l2_len;
 686                         tx_offload.l3_len = tx_pkt->l3_len;
 687                         tx_offload.l4_len = tx_pkt->l4_len;
 688                         tx_offload.vlan_tci = tx_pkt->vlan_tci;
 689                         tx_offload.tso_segsz = tx_pkt->tso_segsz;
 690                         tx_offload.outer_l2_len = tx_pkt->outer_l2_len;
 691                         tx_offload.outer_l3_len = tx_pkt->outer_l3_len;
 692 #ifdef RTE_LIBRTE_SECURITY
 693                         if (use_ipsec) {
 694                                 union ixgbe_crypto_tx_desc_md *ipsec_mdata =
 695                                         (union ixgbe_crypto_tx_desc_md *)
 696                                                         &tx_pkt->udata64;
 697                                 tx_offload.sa_idx = ipsec_mdata->sa_idx;
 698                                 tx_offload.sec_pad_len = ipsec_mdata->pad_len;
 699                         }
 700 #endif
 701
 702                         /* If new context need be built or reuse the exist ctx. */
 703                         ctx = what_advctx_update(txq, tx_ol_req,
 704                                 tx_offload);
 705                         /* Only allocate context descriptor if required*/
 706                         new_ctx = (ctx == IXGBE_CTX_NUM);
 707                         ctx = txq->ctx_curr;
 708                 }
 709
 710                 /*
 711                  * Keep track of how many descriptors are used this loop
 712                  * This will always be the number of segments + the number of
 713                  * Context descriptors required to transmit the packet
 714                  */
 715                 nb_used = (uint16_t)(tx_pkt->nb_segs + new_ctx);
 716
 717                 if (txp != NULL &&
 718                                 nb_used + txq->nb_tx_used >= txq->tx_rs_thresh)
 719                         /* set RS on the previous packet in the burst */
 720                         txp->read.cmd_type_len |=
 721                                 rte_cpu_to_le_32(IXGBE_TXD_CMD_RS);
 722
 723                 /*
 724                  * The number of descriptors that must be allocated for a
 725                  * packet is the number of segments of that packet, plus 1
 726                  * Context Descriptor for the hardware offload, if any.
 727                  * Determine the last TX descriptor to allocate in the TX ring
 728                  * for the packet, starting from the current position (tx_id)
 729                  * in the ring.
 730                  */
 731                 tx_last = (uint16_t) (tx_id + nb_used - 1);
 732
 733                 /* Circular ring */
 734                 if (tx_last >= txq->nb_tx_desc)
 735                         tx_last = (uint16_t) (tx_last - txq->nb_tx_desc);
 736
 737                 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u pktlen=%u"
 738                            " tx_first=%u tx_last=%u",
 739                            (unsigned) txq->port_id,
 740                            (unsigned) txq->queue_id,
 741                            (unsigned) pkt_len,
 742                            (unsigned) tx_id,
 743                            (unsigned) tx_last);
 744
 745                 /*
 746                  * Make sure there are enough TX descriptors available to
 747                  * transmit the entire packet.
 748                  * nb_used better be less than or equal to txq->tx_rs_thresh
 749                  */
 750                 if (nb_used > txq->nb_tx_free) {
 751                         PMD_TX_FREE_LOG(DEBUG,
 752                                         "Not enough free TX descriptors "
 753                                         "nb_used=%4u nb_free=%4u "
 754                                         "(port=%d queue=%d)",
 755                                         nb_used, txq->nb_tx_free,
 756                                         txq->port_id, txq->queue_id);
 757
 758                         if (ixgbe_xmit_cleanup(txq) != 0) {
 759                                 /* Could not clean any descriptors */
 760                                 if (nb_tx == 0)
 761                                         return 0;
 762                                 goto end_of_tx;
 763                         }
 764
 765                         /* nb_used better be <= txq->tx_rs_thresh */
 766                         if (unlikely(nb_used > txq->tx_rs_thresh)) {
 767                                 PMD_TX_FREE_LOG(DEBUG,
 768                                         "The number of descriptors needed to "
 769                                         "transmit the packet exceeds the "
 770                                         "RS bit threshold. This will impact "
 771                                         "performance."
 772                                         "nb_used=%4u nb_free=%4u "
 773                                         "tx_rs_thresh=%4u. "
 774                                         "(port=%d queue=%d)",
 775                                         nb_used, txq->nb_tx_free,
 776                                         txq->tx_rs_thresh,
 777                                         txq->port_id, txq->queue_id);
 778                                 /*
 779                                  * Loop here until there are enough TX
 780                                  * descriptors or until the ring cannot be
 781                                  * cleaned.
 782                                  */
 783                                 while (nb_used > txq->nb_tx_free) {
 784                                         if (ixgbe_xmit_cleanup(txq) != 0) {
 785                                                 /*
 786                                                  * Could not clean any
 787                                                  * descriptors
 788                                                  */
 789                                                 if (nb_tx == 0)
 790                                                         return 0;
 791                                                 goto end_of_tx;
 792                                         }
 793                                 }
 794                         }
 795                 }
 796
 797                 /*
 798                  * By now there are enough free TX descriptors to transmit
 799                  * the packet.
 800                  */
 801
 802                 /*
 803                  * Set common flags of all TX Data Descriptors.
 804                  *
 805                  * The following bits must be set in all Data Descriptors:
 806                  *   - IXGBE_ADVTXD_DTYP_DATA
 807                  *   - IXGBE_ADVTXD_DCMD_DEXT
 808                  *
 809                  * The following bits must be set in the first Data Descriptor
 810                  * and are ignored in the other ones:
 811                  *   - IXGBE_ADVTXD_DCMD_IFCS
 812                  *   - IXGBE_ADVTXD_MAC_1588
 813                  *   - IXGBE_ADVTXD_DCMD_VLE
 814                  *
 815                  * The following bits must only be set in the last Data
 816                  * Descriptor:
 817                  *   - IXGBE_TXD_CMD_EOP
 818                  *
 819                  * The following bits can be set in any Data Descriptor, but
 820                  * are only set in the last Data Descriptor:
 821                  *   - IXGBE_TXD_CMD_RS
 822                  */
 823                 cmd_type_len = IXGBE_ADVTXD_DTYP_DATA |
 824                         IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT;
 825
 826 #ifdef RTE_LIBRTE_IEEE1588
 827                 if (ol_flags & PKT_TX_IEEE1588_TMST)
 828                         cmd_type_len |= IXGBE_ADVTXD_MAC_1588;
 829 #endif
 830
 831                 olinfo_status = 0;
 832                 if (tx_ol_req) {
 833
 834                         if (ol_flags & PKT_TX_TCP_SEG) {
 835                                 /* when TSO is on, paylen in descriptor is the
 836                                  * not the packet len but the tcp payload len */
 837                                 pkt_len -= (tx_offload.l2_len +
 838                                         tx_offload.l3_len + tx_offload.l4_len);
 839                         }
 840
 841                         /*
 842                          * Setup the TX Advanced Context Descriptor if required
 843                          */
 844                         if (new_ctx) {
 845                                 volatile struct ixgbe_adv_tx_context_desc *
 846                                     ctx_txd;
 847
 848                                 ctx_txd = (volatile struct
 849                                     ixgbe_adv_tx_context_desc *)
 850                                     &txr[tx_id];
 851
 852                                 txn = &sw_ring[txe->next_id];
 853                                 rte_prefetch0(&txn->mbuf->pool);
 854
 855                                 if (txe->mbuf != NULL) {
 856                                         rte_pktmbuf_free_seg(txe->mbuf);
 857                                         txe->mbuf = NULL;
 858                                 }
 859
 860                                 ixgbe_set_xmit_ctx(txq, ctx_txd, tx_ol_req,
 861                                         tx_offload, &tx_pkt->udata64);
 862
 863                                 txe->last_id = tx_last;
 864                                 tx_id = txe->next_id;
 865                                 txe = txn;
 866                         }
 867
 868                         /*
 869                          * Setup the TX Advanced Data Descriptor,
 870                          * This path will go through
 871                          * whatever new/reuse the context descriptor
 872                          */
 873                         cmd_type_len  |= tx_desc_ol_flags_to_cmdtype(ol_flags);
 874                         olinfo_status |= tx_desc_cksum_flags_to_olinfo(ol_flags);
 875                         olinfo_status |= ctx << IXGBE_ADVTXD_IDX_SHIFT;
 876                 }
 877
 878                 olinfo_status |= (pkt_len << IXGBE_ADVTXD_PAYLEN_SHIFT);
 879 #ifdef RTE_LIBRTE_SECURITY
 880                 if (use_ipsec)
 881                         olinfo_status |= IXGBE_ADVTXD_POPTS_IPSEC;
 882 #endif
 883
 884                 m_seg = tx_pkt;
 885                 do {
 886                         txd = &txr[tx_id];
 887                         txn = &sw_ring[txe->next_id];
 888                         rte_prefetch0(&txn->mbuf->pool);
 889
 890                         if (txe->mbuf != NULL)
 891                                 rte_pktmbuf_free_seg(txe->mbuf);
 892                         txe->mbuf = m_seg;
 893
 894                         /*
 895                          * Set up Transmit Data Descriptor.
 896                          */
 897                         slen = m_seg->data_len;
 898                         buf_dma_addr = rte_mbuf_data_iova(m_seg);
 899                         txd->read.buffer_addr =
 900                                 rte_cpu_to_le_64(buf_dma_addr);
 901                         txd->read.cmd_type_len =
 902                                 rte_cpu_to_le_32(cmd_type_len | slen);
 903                         txd->read.olinfo_status =
 904                                 rte_cpu_to_le_32(olinfo_status);
 905                         txe->last_id = tx_last;
 906                         tx_id = txe->next_id;
 907                         txe = txn;
 908                         m_seg = m_seg->next;
 909                 } while (m_seg != NULL);
 910
 911                 /*
 912                  * The last packet data descriptor needs End Of Packet (EOP)
 913                  */
 914                 cmd_type_len |= IXGBE_TXD_CMD_EOP;
 915                 txq->nb_tx_used = (uint16_t)(txq->nb_tx_used + nb_used);
 916                 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_used);
 917
 918                 /* Set RS bit only on threshold packets' last descriptor */
 919                 if (txq->nb_tx_used >= txq->tx_rs_thresh) {
 920                         PMD_TX_FREE_LOG(DEBUG,
 921                                         "Setting RS bit on TXD id="
 922                                         "%4u (port=%d queue=%d)",
 923                                         tx_last, txq->port_id, txq->queue_id);
 924
 925                         cmd_type_len |= IXGBE_TXD_CMD_RS;
 926
 927                         /* Update txq RS bit counters */
 928                         txq->nb_tx_used = 0;
 929                         txp = NULL;
 930                 } else
 931                         txp = txd;
 932
 933                 txd->read.cmd_type_len |= rte_cpu_to_le_32(cmd_type_len);
 934         }
 935
 936 end_of_tx:
 937         /* set RS on last packet in the burst */
 938         if (txp != NULL)
 939                 txp->read.cmd_type_len |= rte_cpu_to_le_32(IXGBE_TXD_CMD_RS);
 940
 941         rte_wmb();
 942
 943         /*
 944          * Set the Transmit Descriptor Tail (TDT)
 945          */
 946         PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u",
 947                    (unsigned) txq->port_id, (unsigned) txq->queue_id,
 948                    (unsigned) tx_id, (unsigned) nb_tx);
 949         IXGBE_PCI_REG_WRITE_RELAXED(txq->tdt_reg_addr, tx_id);
 950         txq->tx_tail = tx_id;
 951
 952         return nb_tx;
 953 }
 954
 955 /*********************************************************************
 956  *
 957  *  TX prep functions
 958  *
 959  **********************************************************************/
 960 uint16_t
 961 ixgbe_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 962 {
 963         int i, ret;
 964         uint64_t ol_flags;
 965         struct rte_mbuf *m;
 966         struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue;
 967
 968         for (i = 0; i < nb_pkts; i++) {
 969                 m = tx_pkts[i];
 970                 ol_flags = m->ol_flags;
 971
 972                 /**
 973                  * Check if packet meets requirements for number of segments
 974                  *
 975                  * NOTE: for ixgbe it's always (40 - WTHRESH) for both TSO and
 976                  *       non-TSO
 977                  */
 978
 979                 if (m->nb_segs > IXGBE_TX_MAX_SEG - txq->wthresh) {
 980                         rte_errno = -EINVAL;
 981                         return i;
 982                 }
 983
 984                 if (ol_flags & IXGBE_TX_OFFLOAD_NOTSUP_MASK) {
 985                         rte_errno = -ENOTSUP;
 986                         return i;
 987                 }
 988
 989 #ifdef RTE_LIBRTE_ETHDEV_DEBUG
 990                 ret = rte_validate_tx_offload(m);
 991                 if (ret != 0) {
 992                         rte_errno = ret;
 993                         return i;
 994                 }
 995 #endif
 996                 ret = rte_net_intel_cksum_prepare(m);
 997                 if (ret != 0) {
 998                         rte_errno = ret;
 999                         return i;
1000                 }
1001         }
1002
1003         return i;
1004 }
1005
1006 /*********************************************************************
1007  *
1008  *  RX functions
1009  *
1010  **********************************************************************/
1011
1012 #define IXGBE_PACKET_TYPE_ETHER                         0X00
1013 #define IXGBE_PACKET_TYPE_IPV4                          0X01
1014 #define IXGBE_PACKET_TYPE_IPV4_TCP                      0X11
1015 #define IXGBE_PACKET_TYPE_IPV4_UDP                      0X21
1016 #define IXGBE_PACKET_TYPE_IPV4_SCTP                     0X41
1017 #define IXGBE_PACKET_TYPE_IPV4_EXT                      0X03
1018 #define IXGBE_PACKET_TYPE_IPV4_EXT_TCP                  0X13
1019 #define IXGBE_PACKET_TYPE_IPV4_EXT_UDP                  0X23
1020 #define IXGBE_PACKET_TYPE_IPV4_EXT_SCTP                 0X43
1021 #define IXGBE_PACKET_TYPE_IPV6                          0X04
1022 #define IXGBE_PACKET_TYPE_IPV6_TCP                      0X14
1023 #define IXGBE_PACKET_TYPE_IPV6_UDP                      0X24
1024 #define IXGBE_PACKET_TYPE_IPV6_SCTP                     0X44
1025 #define IXGBE_PACKET_TYPE_IPV6_EXT                      0X0C
1026 #define IXGBE_PACKET_TYPE_IPV6_EXT_TCP                  0X1C
1027 #define IXGBE_PACKET_TYPE_IPV6_EXT_UDP                  0X2C
1028 #define IXGBE_PACKET_TYPE_IPV6_EXT_SCTP                 0X4C
1029 #define IXGBE_PACKET_TYPE_IPV4_IPV6                     0X05
1030 #define IXGBE_PACKET_TYPE_IPV4_IPV6_TCP                 0X15
1031 #define IXGBE_PACKET_TYPE_IPV4_IPV6_UDP                 0X25
1032 #define IXGBE_PACKET_TYPE_IPV4_IPV6_SCTP                0X45
1033 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6                 0X07
1034 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_TCP             0X17
1035 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_UDP             0X27
1036 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_SCTP            0X47
1037 #define IXGBE_PACKET_TYPE_IPV4_IPV6_EXT                 0X0D
1038 #define IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_TCP             0X1D
1039 #define IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_UDP             0X2D
1040 #define IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_SCTP            0X4D
1041 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT             0X0F
1042 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_TCP         0X1F
1043 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_UDP         0X2F
1044 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_SCTP        0X4F
1045
1046 #define IXGBE_PACKET_TYPE_NVGRE                   0X00
1047 #define IXGBE_PACKET_TYPE_NVGRE_IPV4              0X01
1048 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_TCP          0X11
1049 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_UDP          0X21
1050 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_SCTP         0X41
1051 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT          0X03
1052 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_TCP      0X13
1053 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_UDP      0X23
1054 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_SCTP     0X43
1055 #define IXGBE_PACKET_TYPE_NVGRE_IPV6              0X04
1056 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_TCP          0X14
1057 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_UDP          0X24
1058 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_SCTP         0X44
1059 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT          0X0C
1060 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_TCP      0X1C
1061 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_UDP      0X2C
1062 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_SCTP     0X4C
1063 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6         0X05
1064 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_TCP     0X15
1065 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_UDP     0X25
1066 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT     0X0D
1067 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_TCP 0X1D
1068 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_UDP 0X2D
1069
1070 #define IXGBE_PACKET_TYPE_VXLAN                   0X80
1071 #define IXGBE_PACKET_TYPE_VXLAN_IPV4              0X81
1072 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_TCP          0x91
1073 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_UDP          0xA1
1074 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_SCTP         0xC1
1075 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT          0x83
1076 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_TCP      0X93
1077 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_UDP      0XA3
1078 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_SCTP     0XC3
1079 #define IXGBE_PACKET_TYPE_VXLAN_IPV6              0X84
1080 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_TCP          0X94
1081 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_UDP          0XA4
1082 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_SCTP         0XC4
1083 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT          0X8C
1084 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_TCP      0X9C
1085 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_UDP      0XAC
1086 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_SCTP     0XCC
1087 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6         0X85
1088 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_TCP     0X95
1089 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_UDP     0XA5
1090 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT     0X8D
1091 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_TCP 0X9D
1092 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_UDP 0XAD
1093
1094 /**
1095  * Use 2 different table for normal packet and tunnel packet
1096  * to save the space.
1097  */
1098 const uint32_t
1099         ptype_table[IXGBE_PACKET_TYPE_MAX] __rte_cache_aligned = {
1100         [IXGBE_PACKET_TYPE_ETHER] = RTE_PTYPE_L2_ETHER,
1101         [IXGBE_PACKET_TYPE_IPV4] = RTE_PTYPE_L2_ETHER |
1102                 RTE_PTYPE_L3_IPV4,
1103         [IXGBE_PACKET_TYPE_IPV4_TCP] = RTE_PTYPE_L2_ETHER |
1104                 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP,
1105         [IXGBE_PACKET_TYPE_IPV4_UDP] = RTE_PTYPE_L2_ETHER |
1106                 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP,
1107         [IXGBE_PACKET_TYPE_IPV4_SCTP] = RTE_PTYPE_L2_ETHER |
1108                 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_SCTP,
1109         [IXGBE_PACKET_TYPE_IPV4_EXT] = RTE_PTYPE_L2_ETHER |
1110                 RTE_PTYPE_L3_IPV4_EXT,
1111         [IXGBE_PACKET_TYPE_IPV4_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1112                 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_TCP,
1113         [IXGBE_PACKET_TYPE_IPV4_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1114                 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_UDP,
1115         [IXGBE_PACKET_TYPE_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1116                 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_SCTP,
1117         [IXGBE_PACKET_TYPE_IPV6] = RTE_PTYPE_L2_ETHER |
1118                 RTE_PTYPE_L3_IPV6,
1119         [IXGBE_PACKET_TYPE_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1120                 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP,
1121         [IXGBE_PACKET_TYPE_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1122                 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP,
1123         [IXGBE_PACKET_TYPE_IPV6_SCTP] = RTE_PTYPE_L2_ETHER |
1124                 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_SCTP,
1125         [IXGBE_PACKET_TYPE_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1126                 RTE_PTYPE_L3_IPV6_EXT,
1127         [IXGBE_PACKET_TYPE_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1128                 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_TCP,
1129         [IXGBE_PACKET_TYPE_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1130                 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_UDP,
1131         [IXGBE_PACKET_TYPE_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1132                 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_SCTP,
1133         [IXGBE_PACKET_TYPE_IPV4_IPV6] = RTE_PTYPE_L2_ETHER |
1134                 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1135                 RTE_PTYPE_INNER_L3_IPV6,
1136         [IXGBE_PACKET_TYPE_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1137                 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1138                 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP,
1139         [IXGBE_PACKET_TYPE_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1140                 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1141         RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP,
1142         [IXGBE_PACKET_TYPE_IPV4_IPV6_SCTP] = RTE_PTYPE_L2_ETHER |
1143                 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1144                 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_SCTP,
1145         [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6] = RTE_PTYPE_L2_ETHER |
1146                 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1147                 RTE_PTYPE_INNER_L3_IPV6,
1148         [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1149                 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1150                 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP,
1151         [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1152                 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1153                 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP,
1154         [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_SCTP] = RTE_PTYPE_L2_ETHER |
1155                 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1156                 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_SCTP,
1157         [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1158                 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1159                 RTE_PTYPE_INNER_L3_IPV6_EXT,
1160         [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1161                 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1162                 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP,
1163         [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1164                 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1165                 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP,
1166         [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1167                 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP |
1168                 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_SCTP,
1169         [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1170                 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1171                 RTE_PTYPE_INNER_L3_IPV6_EXT,
1172         [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1173                 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1174                 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP,
1175         [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1176                 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1177                 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP,
1178         [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_SCTP] =
1179                 RTE_PTYPE_L2_ETHER |
1180                 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP |
1181                 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_SCTP,
1182 };
1183
1184 const uint32_t
1185         ptype_table_tn[IXGBE_PACKET_TYPE_TN_MAX] __rte_cache_aligned = {
1186         [IXGBE_PACKET_TYPE_NVGRE] = RTE_PTYPE_L2_ETHER |
1187                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1188                 RTE_PTYPE_INNER_L2_ETHER,
1189         [IXGBE_PACKET_TYPE_NVGRE_IPV4] = RTE_PTYPE_L2_ETHER |
1190                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1191                 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1192         [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT] = RTE_PTYPE_L2_ETHER |
1193                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1194                 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT,
1195         [IXGBE_PACKET_TYPE_NVGRE_IPV6] = RTE_PTYPE_L2_ETHER |
1196                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1197                 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6,
1198         [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6] = RTE_PTYPE_L2_ETHER |
1199                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1200                 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1201         [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1202                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1203                 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT,
1204         [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1205                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1206                 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1207         [IXGBE_PACKET_TYPE_NVGRE_IPV4_TCP] = RTE_PTYPE_L2_ETHER |
1208                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1209                 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4 |
1210                 RTE_PTYPE_INNER_L4_TCP,
1211         [IXGBE_PACKET_TYPE_NVGRE_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1212                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1213                 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6 |
1214                 RTE_PTYPE_INNER_L4_TCP,
1215         [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1216                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1217                 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1218         [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1219                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1220                 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT |
1221                 RTE_PTYPE_INNER_L4_TCP,
1222         [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_TCP] =
1223                 RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
1224                 RTE_PTYPE_TUNNEL_GRE | RTE_PTYPE_INNER_L2_ETHER |
1225                 RTE_PTYPE_INNER_L3_IPV4,
1226         [IXGBE_PACKET_TYPE_NVGRE_IPV4_UDP] = RTE_PTYPE_L2_ETHER |
1227                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1228                 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4 |
1229                 RTE_PTYPE_INNER_L4_UDP,
1230         [IXGBE_PACKET_TYPE_NVGRE_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1231                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1232                 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6 |
1233                 RTE_PTYPE_INNER_L4_UDP,
1234         [IXGBE_PACKET_TYPE_NVGRE_IPV6_SCTP] = RTE_PTYPE_L2_ETHER |
1235                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1236                 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6 |
1237                 RTE_PTYPE_INNER_L4_SCTP,
1238         [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1239                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1240                 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1241         [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1242                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1243                 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT |
1244                 RTE_PTYPE_INNER_L4_UDP,
1245         [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1246                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1247                 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT |
1248                 RTE_PTYPE_INNER_L4_SCTP,
1249         [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_UDP] =
1250                 RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
1251                 RTE_PTYPE_TUNNEL_GRE | RTE_PTYPE_INNER_L2_ETHER |
1252                 RTE_PTYPE_INNER_L3_IPV4,
1253         [IXGBE_PACKET_TYPE_NVGRE_IPV4_SCTP] = RTE_PTYPE_L2_ETHER |
1254                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1255                 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4 |
1256                 RTE_PTYPE_INNER_L4_SCTP,
1257         [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1258                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1259                 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT |
1260                 RTE_PTYPE_INNER_L4_SCTP,
1261         [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1262                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1263                 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT |
1264                 RTE_PTYPE_INNER_L4_TCP,
1265         [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1266                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE |
1267                 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT |
1268                 RTE_PTYPE_INNER_L4_UDP,
1269
1270         [IXGBE_PACKET_TYPE_VXLAN] = RTE_PTYPE_L2_ETHER |
1271                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1272                 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER,
1273         [IXGBE_PACKET_TYPE_VXLAN_IPV4] = RTE_PTYPE_L2_ETHER |
1274                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1275                 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1276                 RTE_PTYPE_INNER_L3_IPV4,
1277         [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT] = RTE_PTYPE_L2_ETHER |
1278                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1279                 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1280                 RTE_PTYPE_INNER_L3_IPV4_EXT,
1281         [IXGBE_PACKET_TYPE_VXLAN_IPV6] = RTE_PTYPE_L2_ETHER |
1282                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1283                 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1284                 RTE_PTYPE_INNER_L3_IPV6,
1285         [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6] = RTE_PTYPE_L2_ETHER |
1286                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1287                 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1288                 RTE_PTYPE_INNER_L3_IPV4,
1289         [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1290                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1291                 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1292                 RTE_PTYPE_INNER_L3_IPV6_EXT,
1293         [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER |
1294                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1295                 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1296                 RTE_PTYPE_INNER_L3_IPV4,
1297         [IXGBE_PACKET_TYPE_VXLAN_IPV4_TCP] = RTE_PTYPE_L2_ETHER |
1298                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1299                 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1300                 RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L4_TCP,
1301         [IXGBE_PACKET_TYPE_VXLAN_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1302                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1303                 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1304                 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP,
1305         [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER |
1306                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1307                 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1308                 RTE_PTYPE_INNER_L3_IPV4,
1309         [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1310                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1311                 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1312                 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP,
1313         [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_TCP] =
1314                 RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
1315                 RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_VXLAN |
1316                 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1317         [IXGBE_PACKET_TYPE_VXLAN_IPV4_UDP] = RTE_PTYPE_L2_ETHER |
1318                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1319                 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1320                 RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L4_UDP,
1321         [IXGBE_PACKET_TYPE_VXLAN_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1322                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1323                 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1324                 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP,
1325         [IXGBE_PACKET_TYPE_VXLAN_IPV6_SCTP] = RTE_PTYPE_L2_ETHER |
1326                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1327                 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1328                 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_SCTP,
1329         [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER |
1330                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1331                 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1332                 RTE_PTYPE_INNER_L3_IPV4,
1333         [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1334                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1335                 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1336                 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP,
1337         [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1338                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1339                 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1340                 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_SCTP,
1341         [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_UDP] =
1342                 RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
1343                 RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_VXLAN |
1344                 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4,
1345         [IXGBE_PACKET_TYPE_VXLAN_IPV4_SCTP] = RTE_PTYPE_L2_ETHER |
1346                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1347                 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1348                 RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L4_SCTP,
1349         [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER |
1350                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1351                 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1352                 RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_SCTP,
1353         [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_TCP] = RTE_PTYPE_L2_ETHER |
1354                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1355                 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1356                 RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_TCP,
1357         [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_UDP] = RTE_PTYPE_L2_ETHER |
1358                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP |
1359                 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER |
1360                 RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_UDP,
1361 };
1362
1363 /* @note: fix ixgbe_dev_supported_ptypes_get() if any change here. */
1364 static inline uint32_t
1365 ixgbe_rxd_pkt_info_to_pkt_type(uint32_t pkt_info, uint16_t ptype_mask)
1366 {
1367
1368         if (unlikely(pkt_info & IXGBE_RXDADV_PKTTYPE_ETQF))
1369                 return RTE_PTYPE_UNKNOWN;
1370
1371         pkt_info = (pkt_info >> IXGBE_PACKET_TYPE_SHIFT) & ptype_mask;
1372
1373         /* For tunnel packet */
1374         if (pkt_info & IXGBE_PACKET_TYPE_TUNNEL_BIT) {
1375                 /* Remove the tunnel bit to save the space. */
1376                 pkt_info &= IXGBE_PACKET_TYPE_MASK_TUNNEL;
1377                 return ptype_table_tn[pkt_info];
1378         }
1379
1380         /**
1381          * For x550, if it's not tunnel,
1382          * tunnel type bit should be set to 0.
1383          * Reuse 82599's mask.
1384          */
1385         pkt_info &= IXGBE_PACKET_TYPE_MASK_82599;
1386
1387         return ptype_table[pkt_info];
1388 }
1389
1390 static inline uint64_t
1391 ixgbe_rxd_pkt_info_to_pkt_flags(uint16_t pkt_info)
1392 {
1393         static uint64_t ip_rss_types_map[16] __rte_cache_aligned = {
1394                 0, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH,
1395                 0, PKT_RX_RSS_HASH, 0, PKT_RX_RSS_HASH,
1396                 PKT_RX_RSS_HASH, 0, 0, 0,
1397                 0, 0, 0,  PKT_RX_FDIR,
1398         };
1399 #ifdef RTE_LIBRTE_IEEE1588
1400         static uint64_t ip_pkt_etqf_map[8] = {
1401                 0, 0, 0, PKT_RX_IEEE1588_PTP,
1402                 0, 0, 0, 0,
1403         };
1404
1405         if (likely(pkt_info & IXGBE_RXDADV_PKTTYPE_ETQF))
1406                 return ip_pkt_etqf_map[(pkt_info >> 4) & 0X07] |
1407                                 ip_rss_types_map[pkt_info & 0XF];
1408         else
1409                 return ip_rss_types_map[pkt_info & 0XF];
1410 #else
1411         return ip_rss_types_map[pkt_info & 0XF];
1412 #endif
1413 }
1414
1415 static inline uint64_t
1416 rx_desc_status_to_pkt_flags(uint32_t rx_status, uint64_t vlan_flags)
1417 {
1418         uint64_t pkt_flags;
1419
1420         /*
1421          * Check if VLAN present only.
1422          * Do not check whether L3/L4 rx checksum done by NIC or not,
1423          * That can be found from rte_eth_rxmode.hw_ip_checksum flag
1424          */
1425         pkt_flags = (rx_status & IXGBE_RXD_STAT_VP) ?  vlan_flags : 0;
1426
1427 #ifdef RTE_LIBRTE_IEEE1588
1428         if (rx_status & IXGBE_RXD_STAT_TMST)
1429                 pkt_flags = pkt_flags | PKT_RX_IEEE1588_TMST;
1430 #endif
1431         return pkt_flags;
1432 }
1433
1434 static inline uint64_t
1435 rx_desc_error_to_pkt_flags(uint32_t rx_status)
1436 {
1437         uint64_t pkt_flags;
1438
1439         /*
1440          * Bit 31: IPE, IPv4 checksum error
1441          * Bit 30: L4I, L4I integrity error
1442          */
1443         static uint64_t error_to_pkt_flags_map[4] = {
1444                 PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD,
1445                 PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD,
1446                 PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_GOOD,
1447                 PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD
1448         };
1449         pkt_flags = error_to_pkt_flags_map[(rx_status >>
1450                 IXGBE_RXDADV_ERR_CKSUM_BIT) & IXGBE_RXDADV_ERR_CKSUM_MSK];
1451
1452         if ((rx_status & IXGBE_RXD_STAT_OUTERIPCS) &&
1453             (rx_status & IXGBE_RXDADV_ERR_OUTERIPER)) {
1454                 pkt_flags |= PKT_RX_EIP_CKSUM_BAD;
1455         }
1456
1457 #ifdef RTE_LIBRTE_SECURITY
1458         if (rx_status & IXGBE_RXD_STAT_SECP) {
1459                 pkt_flags |= PKT_RX_SEC_OFFLOAD;
1460                 if (rx_status & IXGBE_RXDADV_LNKSEC_ERROR_BAD_SIG)
1461                         pkt_flags |= PKT_RX_SEC_OFFLOAD_FAILED;
1462         }
1463 #endif
1464
1465         return pkt_flags;
1466 }
1467
1468 /*
1469  * LOOK_AHEAD defines how many desc statuses to check beyond the
1470  * current descriptor.
1471  * It must be a pound define for optimal performance.
1472  * Do not change the value of LOOK_AHEAD, as the ixgbe_rx_scan_hw_ring
1473  * function only works with LOOK_AHEAD=8.
1474  */
1475 #define LOOK_AHEAD 8
1476 #if (LOOK_AHEAD != 8)
1477 #error "PMD IXGBE: LOOK_AHEAD must be 8\n"
1478 #endif
1479 static inline int
1480 ixgbe_rx_scan_hw_ring(struct ixgbe_rx_queue *rxq)
1481 {
1482         volatile union ixgbe_adv_rx_desc *rxdp;
1483         struct ixgbe_rx_entry *rxep;
1484         struct rte_mbuf *mb;
1485         uint16_t pkt_len;
1486         uint64_t pkt_flags;
1487         int nb_dd;
1488         uint32_t s[LOOK_AHEAD];
1489         uint32_t pkt_info[LOOK_AHEAD];
1490         int i, j, nb_rx = 0;
1491         uint32_t status;
1492         uint64_t vlan_flags = rxq->vlan_flags;
1493
1494         /* get references to current descriptor and S/W ring entry */
1495         rxdp = &rxq->rx_ring[rxq->rx_tail];
1496         rxep = &rxq->sw_ring[rxq->rx_tail];
1497
1498         status = rxdp->wb.upper.status_error;
1499         /* check to make sure there is at least 1 packet to receive */
1500         if (!(status & rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD)))
1501                 return 0;
1502
1503         /*
1504          * Scan LOOK_AHEAD descriptors at a time to determine which descriptors
1505          * reference packets that are ready to be received.
1506          */
1507         for (i = 0; i < RTE_PMD_IXGBE_RX_MAX_BURST;
1508              i += LOOK_AHEAD, rxdp += LOOK_AHEAD, rxep += LOOK_AHEAD) {
1509                 /* Read desc statuses backwards to avoid race condition */
1510                 for (j = 0; j < LOOK_AHEAD; j++)
1511                         s[j] = rte_le_to_cpu_32(rxdp[j].wb.upper.status_error);
1512
1513                 rte_smp_rmb();
1514
1515                 /* Compute how many status bits were set */
1516                 for (nb_dd = 0; nb_dd < LOOK_AHEAD &&
1517                                 (s[nb_dd] & IXGBE_RXDADV_STAT_DD); nb_dd++)
1518                         ;
1519
1520                 for (j = 0; j < nb_dd; j++)
1521                         pkt_info[j] = rte_le_to_cpu_32(rxdp[j].wb.lower.
1522                                                        lo_dword.data);
1523
1524                 nb_rx += nb_dd;
1525
1526                 /* Translate descriptor info to mbuf format */
1527                 for (j = 0; j < nb_dd; ++j) {
1528                         mb = rxep[j].mbuf;
1529                         pkt_len = rte_le_to_cpu_16(rxdp[j].wb.upper.length) -
1530                                   rxq->crc_len;
1531                         mb->data_len = pkt_len;
1532                         mb->pkt_len = pkt_len;
1533                         mb->vlan_tci = rte_le_to_cpu_16(rxdp[j].wb.upper.vlan);
1534
1535                         /* convert descriptor fields to rte mbuf flags */
1536                         pkt_flags = rx_desc_status_to_pkt_flags(s[j],
1537                                 vlan_flags);
1538                         pkt_flags |= rx_desc_error_to_pkt_flags(s[j]);
1539                         pkt_flags |= ixgbe_rxd_pkt_info_to_pkt_flags
1540                                         ((uint16_t)pkt_info[j]);
1541                         mb->ol_flags = pkt_flags;
1542                         mb->packet_type =
1543                                 ixgbe_rxd_pkt_info_to_pkt_type
1544                                         (pkt_info[j], rxq->pkt_type_mask);
1545
1546                         if (likely(pkt_flags & PKT_RX_RSS_HASH))
1547                                 mb->hash.rss = rte_le_to_cpu_32(
1548                                     rxdp[j].wb.lower.hi_dword.rss);
1549                         else if (pkt_flags & PKT_RX_FDIR) {
1550                                 mb->hash.fdir.hash = rte_le_to_cpu_16(
1551                                     rxdp[j].wb.lower.hi_dword.csum_ip.csum) &
1552                                     IXGBE_ATR_HASH_MASK;
1553                                 mb->hash.fdir.id = rte_le_to_cpu_16(
1554                                     rxdp[j].wb.lower.hi_dword.csum_ip.ip_id);
1555                         }
1556                 }
1557
1558                 /* Move mbuf pointers from the S/W ring to the stage */
1559                 for (j = 0; j < LOOK_AHEAD; ++j) {
1560                         rxq->rx_stage[i + j] = rxep[j].mbuf;
1561                 }
1562
1563                 /* stop if all requested packets could not be received */
1564                 if (nb_dd != LOOK_AHEAD)
1565                         break;
1566         }
1567
1568         /* clear software ring entries so we can cleanup correctly */
1569         for (i = 0; i < nb_rx; ++i) {
1570                 rxq->sw_ring[rxq->rx_tail + i].mbuf = NULL;
1571         }
1572
1573
1574         return nb_rx;
1575 }
1576
1577 static inline int
1578 ixgbe_rx_alloc_bufs(struct ixgbe_rx_queue *rxq, bool reset_mbuf)
1579 {
1580         volatile union ixgbe_adv_rx_desc *rxdp;
1581         struct ixgbe_rx_entry *rxep;
1582         struct rte_mbuf *mb;
1583         uint16_t alloc_idx;
1584         __le64 dma_addr;
1585         int diag, i;
1586
1587         /* allocate buffers in bulk directly into the S/W ring */
1588         alloc_idx = rxq->rx_free_trigger - (rxq->rx_free_thresh - 1);
1589         rxep = &rxq->sw_ring[alloc_idx];
1590         diag = rte_mempool_get_bulk(rxq->mb_pool, (void *)rxep,
1591                                     rxq->rx_free_thresh);
1592         if (unlikely(diag != 0))
1593                 return -ENOMEM;
1594
1595         rxdp = &rxq->rx_ring[alloc_idx];
1596         for (i = 0; i < rxq->rx_free_thresh; ++i) {
1597                 /* populate the static rte mbuf fields */
1598                 mb = rxep[i].mbuf;
1599                 if (reset_mbuf) {
1600                         mb->port = rxq->port_id;
1601                 }
1602
1603                 rte_mbuf_refcnt_set(mb, 1);
1604                 mb->data_off = RTE_PKTMBUF_HEADROOM;
1605
1606                 /* populate the descriptors */
1607                 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mb));
1608                 rxdp[i].read.hdr_addr = 0;
1609                 rxdp[i].read.pkt_addr = dma_addr;
1610         }
1611
1612         /* update state of internal queue structure */
1613         rxq->rx_free_trigger = rxq->rx_free_trigger + rxq->rx_free_thresh;
1614         if (rxq->rx_free_trigger >= rxq->nb_rx_desc)
1615                 rxq->rx_free_trigger = rxq->rx_free_thresh - 1;
1616
1617         /* no errors */
1618         return 0;
1619 }
1620
1621 static inline uint16_t
1622 ixgbe_rx_fill_from_stage(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
1623                          uint16_t nb_pkts)
1624 {
1625         struct rte_mbuf **stage = &rxq->rx_stage[rxq->rx_next_avail];
1626         int i;
1627
1628         /* how many packets are ready to return? */
1629         nb_pkts = (uint16_t)RTE_MIN(nb_pkts, rxq->rx_nb_avail);
1630
1631         /* copy mbuf pointers to the application's packet list */
1632         for (i = 0; i < nb_pkts; ++i)
1633                 rx_pkts[i] = stage[i];
1634
1635         /* update internal queue state */
1636         rxq->rx_nb_avail = (uint16_t)(rxq->rx_nb_avail - nb_pkts);
1637         rxq->rx_next_avail = (uint16_t)(rxq->rx_next_avail + nb_pkts);
1638
1639         return nb_pkts;
1640 }
1641
1642 static inline uint16_t
1643 rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
1644              uint16_t nb_pkts)
1645 {
1646         struct ixgbe_rx_queue *rxq = (struct ixgbe_rx_queue *)rx_queue;
1647         uint16_t nb_rx = 0;
1648
1649         /* Any previously recv'd pkts will be returned from the Rx stage */
1650         if (rxq->rx_nb_avail)
1651                 return ixgbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
1652
1653         /* Scan the H/W ring for packets to receive */
1654         nb_rx = (uint16_t)ixgbe_rx_scan_hw_ring(rxq);
1655
1656         /* update internal queue state */
1657         rxq->rx_next_avail = 0;
1658         rxq->rx_nb_avail = nb_rx;
1659         rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_rx);
1660
1661         /* if required, allocate new buffers to replenish descriptors */
1662         if (rxq->rx_tail > rxq->rx_free_trigger) {
1663                 uint16_t cur_free_trigger = rxq->rx_free_trigger;
1664
1665                 if (ixgbe_rx_alloc_bufs(rxq, true) != 0) {
1666                         int i, j;
1667
1668                         PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u "
1669                                    "queue_id=%u", (unsigned) rxq->port_id,
1670                                    (unsigned) rxq->queue_id);
1671
1672                         rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed +=
1673                                 rxq->rx_free_thresh;
1674
1675                         /*
1676                          * Need to rewind any previous receives if we cannot
1677                          * allocate new buffers to replenish the old ones.
1678                          */
1679                         rxq->rx_nb_avail = 0;
1680                         rxq->rx_tail = (uint16_t)(rxq->rx_tail - nb_rx);
1681                         for (i = 0, j = rxq->rx_tail; i < nb_rx; ++i, ++j)
1682                                 rxq->sw_ring[j].mbuf = rxq->rx_stage[i];
1683
1684                         return 0;
1685                 }
1686
1687                 /* update tail pointer */
1688                 rte_wmb();
1689                 IXGBE_PCI_REG_WRITE_RELAXED(rxq->rdt_reg_addr,
1690                                             cur_free_trigger);
1691         }
1692
1693         if (rxq->rx_tail >= rxq->nb_rx_desc)
1694                 rxq->rx_tail = 0;
1695
1696         /* received any packets this loop? */
1697         if (rxq->rx_nb_avail)
1698                 return ixgbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
1699
1700         return 0;
1701 }
1702
1703 /* split requests into chunks of size RTE_PMD_IXGBE_RX_MAX_BURST */
1704 uint16_t
1705 ixgbe_recv_pkts_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
1706                            uint16_t nb_pkts)
1707 {
1708         uint16_t nb_rx;
1709
1710         if (unlikely(nb_pkts == 0))
1711                 return 0;
1712
1713         if (likely(nb_pkts <= RTE_PMD_IXGBE_RX_MAX_BURST))
1714                 return rx_recv_pkts(rx_queue, rx_pkts, nb_pkts);
1715
1716         /* request is relatively large, chunk it up */
1717         nb_rx = 0;
1718         while (nb_pkts) {
1719                 uint16_t ret, n;
1720
1721                 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_IXGBE_RX_MAX_BURST);
1722                 ret = rx_recv_pkts(rx_queue, &rx_pkts[nb_rx], n);
1723                 nb_rx = (uint16_t)(nb_rx + ret);
1724                 nb_pkts = (uint16_t)(nb_pkts - ret);
1725                 if (ret < n)
1726                         break;
1727         }
1728
1729         return nb_rx;
1730 }
1731
1732 uint16_t
1733 ixgbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
1734                 uint16_t nb_pkts)
1735 {
1736         struct ixgbe_rx_queue *rxq;
1737         volatile union ixgbe_adv_rx_desc *rx_ring;
1738         volatile union ixgbe_adv_rx_desc *rxdp;
1739         struct ixgbe_rx_entry *sw_ring;
1740         struct ixgbe_rx_entry *rxe;
1741         struct rte_mbuf *rxm;
1742         struct rte_mbuf *nmb;
1743         union ixgbe_adv_rx_desc rxd;
1744         uint64_t dma_addr;
1745         uint32_t staterr;
1746         uint32_t pkt_info;
1747         uint16_t pkt_len;
1748         uint16_t rx_id;
1749         uint16_t nb_rx;
1750         uint16_t nb_hold;
1751         uint64_t pkt_flags;
1752         uint64_t vlan_flags;
1753
1754         nb_rx = 0;
1755         nb_hold = 0;
1756         rxq = rx_queue;
1757         rx_id = rxq->rx_tail;
1758         rx_ring = rxq->rx_ring;
1759         sw_ring = rxq->sw_ring;
1760         vlan_flags = rxq->vlan_flags;
1761         while (nb_rx < nb_pkts) {
1762                 /*
1763                  * The order of operations here is important as the DD status
1764                  * bit must not be read after any other descriptor fields.
1765                  * rx_ring and rxdp are pointing to volatile data so the order
1766                  * of accesses cannot be reordered by the compiler. If they were
1767                  * not volatile, they could be reordered which could lead to
1768                  * using invalid descriptor fields when read from rxd.
1769                  */
1770                 rxdp = &rx_ring[rx_id];
1771                 staterr = rxdp->wb.upper.status_error;
1772                 if (!(staterr & rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD)))
1773                         break;
1774                 rxd = *rxdp;
1775
1776                 /*
1777                  * End of packet.
1778                  *
1779                  * If the IXGBE_RXDADV_STAT_EOP flag is not set, the RX packet
1780                  * is likely to be invalid and to be dropped by the various
1781                  * validation checks performed by the network stack.
1782                  *
1783                  * Allocate a new mbuf to replenish the RX ring descriptor.
1784                  * If the allocation fails:
1785                  *    - arrange for that RX descriptor to be the first one
1786                  *      being parsed the next time the receive function is
1787                  *      invoked [on the same queue].
1788                  *
1789                  *    - Stop parsing the RX ring and return immediately.
1790                  *
1791                  * This policy do not drop the packet received in the RX
1792                  * descriptor for which the allocation of a new mbuf failed.
1793                  * Thus, it allows that packet to be later retrieved if
1794                  * mbuf have been freed in the mean time.
1795                  * As a side effect, holding RX descriptors instead of
1796                  * systematically giving them back to the NIC may lead to
1797                  * RX ring exhaustion situations.
1798                  * However, the NIC can gracefully prevent such situations
1799                  * to happen by sending specific "back-pressure" flow control
1800                  * frames to its peer(s).
1801                  */
1802                 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u "
1803                            "ext_err_stat=0x%08x pkt_len=%u",
1804                            (unsigned) rxq->port_id, (unsigned) rxq->queue_id,
1805                            (unsigned) rx_id, (unsigned) staterr,
1806                            (unsigned) rte_le_to_cpu_16(rxd.wb.upper.length));
1807
1808                 nmb = rte_mbuf_raw_alloc(rxq->mb_pool);
1809                 if (nmb == NULL) {
1810                         PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u "
1811                                    "queue_id=%u", (unsigned) rxq->port_id,
1812                                    (unsigned) rxq->queue_id);
1813                         rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed++;
1814                         break;
1815                 }
1816
1817                 nb_hold++;
1818                 rxe = &sw_ring[rx_id];
1819                 rx_id++;
1820                 if (rx_id == rxq->nb_rx_desc)
1821                         rx_id = 0;
1822
1823                 /* Prefetch next mbuf while processing current one. */
1824                 rte_ixgbe_prefetch(sw_ring[rx_id].mbuf);
1825
1826                 /*
1827                  * When next RX descriptor is on a cache-line boundary,
1828                  * prefetch the next 4 RX descriptors and the next 8 pointers
1829                  * to mbufs.
1830                  */
1831                 if ((rx_id & 0x3) == 0) {
1832                         rte_ixgbe_prefetch(&rx_ring[rx_id]);
1833                         rte_ixgbe_prefetch(&sw_ring[rx_id]);
1834                 }
1835
1836                 rxm = rxe->mbuf;
1837                 rxe->mbuf = nmb;
1838                 dma_addr =
1839                         rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
1840                 rxdp->read.hdr_addr = 0;
1841                 rxdp->read.pkt_addr = dma_addr;
1842
1843                 /*
1844                  * Initialize the returned mbuf.
1845                  * 1) setup generic mbuf fields:
1846                  *    - number of segments,
1847                  *    - next segment,
1848                  *    - packet length,
1849                  *    - RX port identifier.
1850                  * 2) integrate hardware offload data, if any:
1851                  *    - RSS flag & hash,
1852                  *    - IP checksum flag,
1853                  *    - VLAN TCI, if any,
1854                  *    - error flags.
1855                  */
1856                 pkt_len = (uint16_t) (rte_le_to_cpu_16(rxd.wb.upper.length) -
1857                                       rxq->crc_len);
1858                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1859                 rte_packet_prefetch((char *)rxm->buf_addr + rxm->data_off);
1860                 rxm->nb_segs = 1;
1861                 rxm->next = NULL;
1862                 rxm->pkt_len = pkt_len;
1863                 rxm->data_len = pkt_len;
1864                 rxm->port = rxq->port_id;
1865
1866                 pkt_info = rte_le_to_cpu_32(rxd.wb.lower.lo_dword.data);
1867                 /* Only valid if PKT_RX_VLAN set in pkt_flags */
1868                 rxm->vlan_tci = rte_le_to_cpu_16(rxd.wb.upper.vlan);
1869
1870                 pkt_flags = rx_desc_status_to_pkt_flags(staterr, vlan_flags);
1871                 pkt_flags = pkt_flags | rx_desc_error_to_pkt_flags(staterr);
1872                 pkt_flags = pkt_flags |
1873                         ixgbe_rxd_pkt_info_to_pkt_flags((uint16_t)pkt_info);
1874                 rxm->ol_flags = pkt_flags;
1875                 rxm->packet_type =
1876                         ixgbe_rxd_pkt_info_to_pkt_type(pkt_info,
1877                                                        rxq->pkt_type_mask);
1878
1879                 if (likely(pkt_flags & PKT_RX_RSS_HASH))
1880                         rxm->hash.rss = rte_le_to_cpu_32(
1881                                                 rxd.wb.lower.hi_dword.rss);
1882                 else if (pkt_flags & PKT_RX_FDIR) {
1883                         rxm->hash.fdir.hash = rte_le_to_cpu_16(
1884                                         rxd.wb.lower.hi_dword.csum_ip.csum) &
1885                                         IXGBE_ATR_HASH_MASK;
1886                         rxm->hash.fdir.id = rte_le_to_cpu_16(
1887                                         rxd.wb.lower.hi_dword.csum_ip.ip_id);
1888                 }
1889                 /*
1890                  * Store the mbuf address into the next entry of the array
1891                  * of returned packets.
1892                  */
1893                 rx_pkts[nb_rx++] = rxm;
1894         }
1895         rxq->rx_tail = rx_id;
1896
1897         /*
1898          * If the number of free RX descriptors is greater than the RX free
1899          * threshold of the queue, advance the Receive Descriptor Tail (RDT)
1900          * register.
1901          * Update the RDT with the value of the last processed RX descriptor
1902          * minus 1, to guarantee that the RDT register is never equal to the
1903          * RDH register, which creates a "full" ring situtation from the
1904          * hardware point of view...
1905          */
1906         nb_hold = (uint16_t) (nb_hold + rxq->nb_rx_hold);
1907         if (nb_hold > rxq->rx_free_thresh) {
1908                 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u "
1909                            "nb_hold=%u nb_rx=%u",
1910                            (unsigned) rxq->port_id, (unsigned) rxq->queue_id,
1911                            (unsigned) rx_id, (unsigned) nb_hold,
1912                            (unsigned) nb_rx);
1913                 rx_id = (uint16_t) ((rx_id == 0) ?
1914                                      (rxq->nb_rx_desc - 1) : (rx_id - 1));
1915                 IXGBE_PCI_REG_WRITE(rxq->rdt_reg_addr, rx_id);
1916                 nb_hold = 0;
1917         }
1918         rxq->nb_rx_hold = nb_hold;
1919         return nb_rx;
1920 }
1921
1922 /**
1923  * Detect an RSC descriptor.
1924  */
1925 static inline uint32_t
1926 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
1927 {
1928         return (rte_le_to_cpu_32(rx->wb.lower.lo_dword.data) &
1929                 IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
1930 }
1931
1932 /**
1933  * ixgbe_fill_cluster_head_buf - fill the first mbuf of the returned packet
1934  *
1935  * Fill the following info in the HEAD buffer of the Rx cluster:
1936  *    - RX port identifier
1937  *    - hardware offload data, if any:
1938  *      - RSS flag & hash
1939  *      - IP checksum flag
1940  *      - VLAN TCI, if any
1941  *      - error flags
1942  * @head HEAD of the packet cluster
1943  * @desc HW descriptor to get data from
1944  * @rxq Pointer to the Rx queue
1945  */
1946 static inline void
1947 ixgbe_fill_cluster_head_buf(
1948         struct rte_mbuf *head,
1949         union ixgbe_adv_rx_desc *desc,
1950         struct ixgbe_rx_queue *rxq,
1951         uint32_t staterr)
1952 {
1953         uint32_t pkt_info;
1954         uint64_t pkt_flags;
1955
1956         head->port = rxq->port_id;
1957
1958         /* The vlan_tci field is only valid when PKT_RX_VLAN is
1959          * set in the pkt_flags field.
1960          */
1961         head->vlan_tci = rte_le_to_cpu_16(desc->wb.upper.vlan);
1962         pkt_info = rte_le_to_cpu_32(desc->wb.lower.lo_dword.data);
1963         pkt_flags = rx_desc_status_to_pkt_flags(staterr, rxq->vlan_flags);
1964         pkt_flags |= rx_desc_error_to_pkt_flags(staterr);
1965         pkt_flags |= ixgbe_rxd_pkt_info_to_pkt_flags((uint16_t)pkt_info);
1966         head->ol_flags = pkt_flags;
1967         head->packet_type =
1968                 ixgbe_rxd_pkt_info_to_pkt_type(pkt_info, rxq->pkt_type_mask);
1969
1970         if (likely(pkt_flags & PKT_RX_RSS_HASH))
1971                 head->hash.rss = rte_le_to_cpu_32(desc->wb.lower.hi_dword.rss);
1972         else if (pkt_flags & PKT_RX_FDIR) {
1973                 head->hash.fdir.hash =
1974                         rte_le_to_cpu_16(desc->wb.lower.hi_dword.csum_ip.csum)
1975                                                           & IXGBE_ATR_HASH_MASK;
1976                 head->hash.fdir.id =
1977                         rte_le_to_cpu_16(desc->wb.lower.hi_dword.csum_ip.ip_id);
1978         }
1979 }
1980
1981 /**
1982  * ixgbe_recv_pkts_lro - receive handler for and LRO case.
1983  *
1984  * @rx_queue Rx queue handle
1985  * @rx_pkts table of received packets
1986  * @nb_pkts size of rx_pkts table
1987  * @bulk_alloc if TRUE bulk allocation is used for a HW ring refilling
1988  *
1989  * Handles the Rx HW ring completions when RSC feature is configured. Uses an
1990  * additional ring of ixgbe_rsc_entry's that will hold the relevant RSC info.
1991  *
1992  * We use the same logic as in Linux and in FreeBSD ixgbe drivers:
1993  * 1) When non-EOP RSC completion arrives:
1994  *    a) Update the HEAD of the current RSC aggregation cluster with the new
1995  *       segment's data length.
1996  *    b) Set the "next" pointer of the current segment to point to the segment
1997  *       at the NEXTP index.
1998  *    c) Pass the HEAD of RSC aggregation cluster on to the next NEXTP entry
1999  *       in the sw_rsc_ring.
2000  * 2) When EOP arrives we just update the cluster's total length and offload
2001  *    flags and deliver the cluster up to the upper layers. In our case - put it
2002  *    in the rx_pkts table.
2003  *
2004  * Returns the number of received packets/clusters (according to the "bulk
2005  * receive" interface).
2006  */
2007 static inline uint16_t
2008 ixgbe_recv_pkts_lro(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts,
2009                     bool bulk_alloc)
2010 {
2011         struct ixgbe_rx_queue *rxq = rx_queue;
2012         volatile union ixgbe_adv_rx_desc *rx_ring = rxq->rx_ring;
2013         struct ixgbe_rx_entry *sw_ring = rxq->sw_ring;
2014         struct ixgbe_scattered_rx_entry *sw_sc_ring = rxq->sw_sc_ring;
2015         uint16_t rx_id = rxq->rx_tail;
2016         uint16_t nb_rx = 0;
2017         uint16_t nb_hold = rxq->nb_rx_hold;
2018         uint16_t prev_id = rxq->rx_tail;
2019
2020         while (nb_rx < nb_pkts) {
2021                 bool eop;
2022                 struct ixgbe_rx_entry *rxe;
2023                 struct ixgbe_scattered_rx_entry *sc_entry;
2024                 struct ixgbe_scattered_rx_entry *next_sc_entry;
2025                 struct ixgbe_rx_entry *next_rxe = NULL;
2026                 struct rte_mbuf *first_seg;
2027                 struct rte_mbuf *rxm;
2028                 struct rte_mbuf *nmb;
2029                 union ixgbe_adv_rx_desc rxd;
2030                 uint16_t data_len;
2031                 uint16_t next_id;
2032                 volatile union ixgbe_adv_rx_desc *rxdp;
2033                 uint32_t staterr;
2034
2035 next_desc:
2036                 /*
2037                  * The code in this whole file uses the volatile pointer to
2038                  * ensure the read ordering of the status and the rest of the
2039                  * descriptor fields (on the compiler level only!!!). This is so
2040                  * UGLY - why not to just use the compiler barrier instead? DPDK
2041                  * even has the rte_compiler_barrier() for that.
2042                  *
2043                  * But most importantly this is just wrong because this doesn't
2044                  * ensure memory ordering in a general case at all. For
2045                  * instance, DPDK is supposed to work on Power CPUs where
2046                  * compiler barrier may just not be enough!
2047                  *
2048                  * I tried to write only this function properly to have a
2049                  * starting point (as a part of an LRO/RSC series) but the
2050                  * compiler cursed at me when I tried to cast away the
2051                  * "volatile" from rx_ring (yes, it's volatile too!!!). So, I'm
2052                  * keeping it the way it is for now.
2053                  *
2054                  * The code in this file is broken in so many other places and
2055                  * will just not work on a big endian CPU anyway therefore the
2056                  * lines below will have to be revisited together with the rest
2057                  * of the ixgbe PMD.
2058                  *
2059                  * TODO:
2060                  *    - Get rid of "volatile" crap and let the compiler do its
2061                  *      job.
2062                  *    - Use the proper memory barrier (rte_rmb()) to ensure the
2063                  *      memory ordering below.
2064                  */
2065                 rxdp = &rx_ring[rx_id];
2066                 staterr = rte_le_to_cpu_32(rxdp->wb.upper.status_error);
2067
2068                 if (!(staterr & IXGBE_RXDADV_STAT_DD))
2069                         break;
2070
2071                 rxd = *rxdp;
2072
2073                 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u "
2074                                   "staterr=0x%x data_len=%u",
2075                            rxq->port_id, rxq->queue_id, rx_id, staterr,
2076                            rte_le_to_cpu_16(rxd.wb.upper.length));
2077
2078                 if (!bulk_alloc) {
2079                         nmb = rte_mbuf_raw_alloc(rxq->mb_pool);
2080                         if (nmb == NULL) {
2081                                 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed "
2082                                                   "port_id=%u queue_id=%u",
2083                                            rxq->port_id, rxq->queue_id);
2084
2085                                 rte_eth_devices[rxq->port_id].data->
2086                                                         rx_mbuf_alloc_failed++;
2087                                 break;
2088                         }
2089                 } else if (nb_hold > rxq->rx_free_thresh) {
2090                         uint16_t next_rdt = rxq->rx_free_trigger;
2091
2092                         if (!ixgbe_rx_alloc_bufs(rxq, false)) {
2093                                 rte_wmb();
2094                                 IXGBE_PCI_REG_WRITE_RELAXED(rxq->rdt_reg_addr,
2095                                                             next_rdt);
2096                                 nb_hold -= rxq->rx_free_thresh;
2097                         } else {
2098                                 PMD_RX_LOG(DEBUG, "RX bulk alloc failed "
2099                                                   "port_id=%u queue_id=%u",
2100                                            rxq->port_id, rxq->queue_id);
2101
2102                                 rte_eth_devices[rxq->port_id].data->
2103                                                         rx_mbuf_alloc_failed++;
2104                                 break;
2105                         }
2106                 }
2107
2108                 nb_hold++;
2109                 rxe = &sw_ring[rx_id];
2110                 eop = staterr & IXGBE_RXDADV_STAT_EOP;
2111
2112                 next_id = rx_id + 1;
2113                 if (next_id == rxq->nb_rx_desc)
2114                         next_id = 0;
2115
2116                 /* Prefetch next mbuf while processing current one. */
2117                 rte_ixgbe_prefetch(sw_ring[next_id].mbuf);
2118
2119                 /*
2120                  * When next RX descriptor is on a cache-line boundary,
2121                  * prefetch the next 4 RX descriptors and the next 4 pointers
2122                  * to mbufs.
2123                  */
2124                 if ((next_id & 0x3) == 0) {
2125                         rte_ixgbe_prefetch(&rx_ring[next_id]);
2126                         rte_ixgbe_prefetch(&sw_ring[next_id]);
2127                 }
2128
2129                 rxm = rxe->mbuf;
2130
2131                 if (!bulk_alloc) {
2132                         __le64 dma =
2133                           rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
2134                         /*
2135                          * Update RX descriptor with the physical address of the
2136                          * new data buffer of the new allocated mbuf.
2137                          */
2138                         rxe->mbuf = nmb;
2139
2140                         rxm->data_off = RTE_PKTMBUF_HEADROOM;
2141                         rxdp->read.hdr_addr = 0;
2142                         rxdp->read.pkt_addr = dma;
2143                 } else
2144                         rxe->mbuf = NULL;
2145
2146                 /*
2147                  * Set data length & data buffer address of mbuf.
2148                  */
2149                 data_len = rte_le_to_cpu_16(rxd.wb.upper.length);
2150                 rxm->data_len = data_len;
2151
2152                 if (!eop) {
2153                         uint16_t nextp_id;
2154                         /*
2155                          * Get next descriptor index:
2156                          *  - For RSC it's in the NEXTP field.
2157                          *  - For a scattered packet - it's just a following
2158                          *    descriptor.
2159                          */
2160                         if (ixgbe_rsc_count(&rxd))
2161                                 nextp_id =
2162                                         (staterr & IXGBE_RXDADV_NEXTP_MASK) >>
2163                                                        IXGBE_RXDADV_NEXTP_SHIFT;
2164                         else
2165                                 nextp_id = next_id;
2166
2167                         next_sc_entry = &sw_sc_ring[nextp_id];
2168                         next_rxe = &sw_ring[nextp_id];
2169                         rte_ixgbe_prefetch(next_rxe);
2170                 }
2171
2172                 sc_entry = &sw_sc_ring[rx_id];
2173                 first_seg = sc_entry->fbuf;
2174                 sc_entry->fbuf = NULL;
2175
2176                 /*
2177                  * If this is the first buffer of the received packet,
2178                  * set the pointer to the first mbuf of the packet and
2179                  * initialize its context.
2180                  * Otherwise, update the total length and the number of segments
2181                  * of the current scattered packet, and update the pointer to
2182                  * the last mbuf of the current packet.
2183                  */
2184                 if (first_seg == NULL) {
2185                         first_seg = rxm;
2186                         first_seg->pkt_len = data_len;
2187                         first_seg->nb_segs = 1;
2188                 } else {
2189                         first_seg->pkt_len += data_len;
2190                         first_seg->nb_segs++;
2191                 }
2192
2193                 prev_id = rx_id;
2194                 rx_id = next_id;
2195
2196                 /*
2197                  * If this is not the last buffer of the received packet, update
2198                  * the pointer to the first mbuf at the NEXTP entry in the
2199                  * sw_sc_ring and continue to parse the RX ring.
2200                  */
2201                 if (!eop && next_rxe) {
2202                         rxm->next = next_rxe->mbuf;
2203                         next_sc_entry->fbuf = first_seg;
2204                         goto next_desc;
2205                 }
2206
2207                 /* Initialize the first mbuf of the returned packet */
2208                 ixgbe_fill_cluster_head_buf(first_seg, &rxd, rxq, staterr);
2209
2210                 /*
2211                  * Deal with the case, when HW CRC srip is disabled.
2212                  * That can't happen when LRO is enabled, but still could
2213                  * happen for scattered RX mode.
2214                  */
2215                 first_seg->pkt_len -= rxq->crc_len;
2216                 if (unlikely(rxm->data_len <= rxq->crc_len)) {
2217                         struct rte_mbuf *lp;
2218
2219                         for (lp = first_seg; lp->next != rxm; lp = lp->next)
2220                                 ;
2221
2222                         first_seg->nb_segs--;
2223                         lp->data_len -= rxq->crc_len - rxm->data_len;
2224                         lp->next = NULL;
2225                         rte_pktmbuf_free_seg(rxm);
2226                 } else
2227                         rxm->data_len -= rxq->crc_len;
2228
2229                 /* Prefetch data of first segment, if configured to do so. */
2230                 rte_packet_prefetch((char *)first_seg->buf_addr +
2231                         first_seg->data_off);
2232
2233                 /*
2234                  * Store the mbuf address into the next entry of the array
2235                  * of returned packets.
2236                  */
2237                 rx_pkts[nb_rx++] = first_seg;
2238         }
2239
2240         /*
2241          * Record index of the next RX descriptor to probe.
2242          */
2243         rxq->rx_tail = rx_id;
2244
2245         /*
2246          * If the number of free RX descriptors is greater than the RX free
2247          * threshold of the queue, advance the Receive Descriptor Tail (RDT)
2248          * register.
2249          * Update the RDT with the value of the last processed RX descriptor
2250          * minus 1, to guarantee that the RDT register is never equal to the
2251          * RDH register, which creates a "full" ring situtation from the
2252          * hardware point of view...
2253          */
2254         if (!bulk_alloc && nb_hold > rxq->rx_free_thresh) {
2255                 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u "
2256                            "nb_hold=%u nb_rx=%u",
2257                            rxq->port_id, rxq->queue_id, rx_id, nb_hold, nb_rx);
2258
2259                 rte_wmb();
2260                 IXGBE_PCI_REG_WRITE_RELAXED(rxq->rdt_reg_addr, prev_id);
2261                 nb_hold = 0;
2262         }
2263
2264         rxq->nb_rx_hold = nb_hold;
2265         return nb_rx;
2266 }
2267
2268 uint16_t
2269 ixgbe_recv_pkts_lro_single_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
2270                                  uint16_t nb_pkts)
2271 {
2272         return ixgbe_recv_pkts_lro(rx_queue, rx_pkts, nb_pkts, false);
2273 }
2274
2275 uint16_t
2276 ixgbe_recv_pkts_lro_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
2277                                uint16_t nb_pkts)
2278 {
2279         return ixgbe_recv_pkts_lro(rx_queue, rx_pkts, nb_pkts, true);
2280 }
2281
2282 /*********************************************************************
2283  *
2284  *  Queue management functions
2285  *
2286  **********************************************************************/
2287
2288 static void __attribute__((cold))
2289 ixgbe_tx_queue_release_mbufs(struct ixgbe_tx_queue *txq)
2290 {
2291         unsigned i;
2292
2293         if (txq->sw_ring != NULL) {
2294                 for (i = 0; i < txq->nb_tx_desc; i++) {
2295                         if (txq->sw_ring[i].mbuf != NULL) {
2296                                 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
2297                                 txq->sw_ring[i].mbuf = NULL;
2298                         }
2299                 }
2300         }
2301 }
2302
2303 static void __attribute__((cold))
2304 ixgbe_tx_free_swring(struct ixgbe_tx_queue *txq)
2305 {
2306         if (txq != NULL &&
2307             txq->sw_ring != NULL)
2308                 rte_free(txq->sw_ring);
2309 }
2310
2311 static void __attribute__((cold))
2312 ixgbe_tx_queue_release(struct ixgbe_tx_queue *txq)
2313 {
2314         if (txq != NULL && txq->ops != NULL) {
2315                 txq->ops->release_mbufs(txq);
2316                 txq->ops->free_swring(txq);
2317                 rte_free(txq);
2318         }
2319 }
2320
2321 void __attribute__((cold))
2322 ixgbe_dev_tx_queue_release(void *txq)
2323 {
2324         ixgbe_tx_queue_release(txq);
2325 }
2326
2327 /* (Re)set dynamic ixgbe_tx_queue fields to defaults */
2328 static void __attribute__((cold))
2329 ixgbe_reset_tx_queue(struct ixgbe_tx_queue *txq)
2330 {
2331         static const union ixgbe_adv_tx_desc zeroed_desc = {{0}};
2332         struct ixgbe_tx_entry *txe = txq->sw_ring;
2333         uint16_t prev, i;
2334
2335         /* Zero out HW ring memory */
2336         for (i = 0; i < txq->nb_tx_desc; i++) {
2337                 txq->tx_ring[i] = zeroed_desc;
2338         }
2339
2340         /* Initialize SW ring entries */
2341         prev = (uint16_t) (txq->nb_tx_desc - 1);
2342         for (i = 0; i < txq->nb_tx_desc; i++) {
2343                 volatile union ixgbe_adv_tx_desc *txd = &txq->tx_ring[i];
2344
2345                 txd->wb.status = rte_cpu_to_le_32(IXGBE_TXD_STAT_DD);
2346                 txe[i].mbuf = NULL;
2347                 txe[i].last_id = i;
2348                 txe[prev].next_id = i;
2349                 prev = i;
2350         }
2351
2352         txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
2353         txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
2354
2355         txq->tx_tail = 0;
2356         txq->nb_tx_used = 0;
2357         /*
2358          * Always allow 1 descriptor to be un-allocated to avoid
2359          * a H/W race condition
2360          */
2361         txq->last_desc_cleaned = (uint16_t)(txq->nb_tx_desc - 1);
2362         txq->nb_tx_free = (uint16_t)(txq->nb_tx_desc - 1);
2363         txq->ctx_curr = 0;
2364         memset((void *)&txq->ctx_cache, 0,
2365                 IXGBE_CTX_NUM * sizeof(struct ixgbe_advctx_info));
2366 }
2367
2368 static const struct ixgbe_txq_ops def_txq_ops = {
2369         .release_mbufs = ixgbe_tx_queue_release_mbufs,
2370         .free_swring = ixgbe_tx_free_swring,
2371         .reset = ixgbe_reset_tx_queue,
2372 };
2373
2374 /* Takes an ethdev and a queue and sets up the tx function to be used based on
2375  * the queue parameters. Used in tx_queue_setup by primary process and then
2376  * in dev_init by secondary process when attaching to an existing ethdev.
2377  */
2378 void __attribute__((cold))
2379 ixgbe_set_tx_function(struct rte_eth_dev *dev, struct ixgbe_tx_queue *txq)
2380 {
2381         /* Use a simple Tx queue (no offloads, no multi segs) if possible */
2382         if ((txq->offloads == 0) &&
2383 #ifdef RTE_LIBRTE_SECURITY
2384                         !(txq->using_ipsec) &&
2385 #endif
2386                         (txq->tx_rs_thresh >= RTE_PMD_IXGBE_TX_MAX_BURST)) {
2387                 PMD_INIT_LOG(DEBUG, "Using simple tx code path");
2388                 dev->tx_pkt_prepare = NULL;
2389 #ifdef RTE_IXGBE_INC_VECTOR
2390                 if (txq->tx_rs_thresh <= RTE_IXGBE_TX_MAX_FREE_BUF_SZ &&
2391                                 (rte_eal_process_type() != RTE_PROC_PRIMARY ||
2392                                         ixgbe_txq_vec_setup(txq) == 0)) {
2393                         PMD_INIT_LOG(DEBUG, "Vector tx enabled.");
2394                         dev->tx_pkt_burst = ixgbe_xmit_pkts_vec;
2395                 } else
2396 #endif
2397                 dev->tx_pkt_burst = ixgbe_xmit_pkts_simple;
2398         } else {
2399                 PMD_INIT_LOG(DEBUG, "Using full-featured tx code path");
2400                 PMD_INIT_LOG(DEBUG,
2401                                 " - offloads = 0x%" PRIx64,
2402                                 txq->offloads);
2403                 PMD_INIT_LOG(DEBUG,
2404                                 " - tx_rs_thresh = %lu " "[RTE_PMD_IXGBE_TX_MAX_BURST=%lu]",
2405                                 (unsigned long)txq->tx_rs_thresh,
2406                                 (unsigned long)RTE_PMD_IXGBE_TX_MAX_BURST);
2407                 dev->tx_pkt_burst = ixgbe_xmit_pkts;
2408                 dev->tx_pkt_prepare = ixgbe_prep_pkts;
2409         }
2410 }
2411
2412 uint64_t
2413 ixgbe_get_tx_queue_offloads(struct rte_eth_dev *dev)
2414 {
2415         RTE_SET_USED(dev);
2416
2417         return 0;
2418 }
2419
2420 uint64_t
2421 ixgbe_get_tx_port_offloads(struct rte_eth_dev *dev)
2422 {
2423         uint64_t tx_offload_capa;
2424         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2425
2426         tx_offload_capa =
2427                 DEV_TX_OFFLOAD_VLAN_INSERT |
2428                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
2429                 DEV_TX_OFFLOAD_UDP_CKSUM   |
2430                 DEV_TX_OFFLOAD_TCP_CKSUM   |
2431                 DEV_TX_OFFLOAD_SCTP_CKSUM  |
2432                 DEV_TX_OFFLOAD_TCP_TSO     |
2433                 DEV_TX_OFFLOAD_MULTI_SEGS;
2434
2435         if (hw->mac.type == ixgbe_mac_82599EB ||
2436             hw->mac.type == ixgbe_mac_X540)
2437                 tx_offload_capa |= DEV_TX_OFFLOAD_MACSEC_INSERT;
2438
2439         if (hw->mac.type == ixgbe_mac_X550 ||
2440             hw->mac.type == ixgbe_mac_X550EM_x ||
2441             hw->mac.type == ixgbe_mac_X550EM_a)
2442                 tx_offload_capa |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM;
2443
2444 #ifdef RTE_LIBRTE_SECURITY
2445         if (dev->security_ctx)
2446                 tx_offload_capa |= DEV_TX_OFFLOAD_SECURITY;
2447 #endif
2448         return tx_offload_capa;
2449 }
2450
2451 static int
2452 ixgbe_check_tx_queue_offloads(struct rte_eth_dev *dev, uint64_t requested)
2453 {
2454         uint64_t port_offloads = dev->data->dev_conf.txmode.offloads;
2455         uint64_t queue_supported = ixgbe_get_tx_queue_offloads(dev);
2456         uint64_t port_supported = ixgbe_get_tx_port_offloads(dev);
2457
2458         if ((requested & (queue_supported | port_supported)) != requested)
2459                 return 0;
2460
2461         if ((port_offloads ^ requested) & port_supported)
2462                 return 0;
2463
2464         return 1;
2465 }
2466
2467 int __attribute__((cold))
2468 ixgbe_dev_tx_queue_setup(struct rte_eth_dev *dev,
2469                          uint16_t queue_idx,
2470                          uint16_t nb_desc,
2471                          unsigned int socket_id,
2472                          const struct rte_eth_txconf *tx_conf)
2473 {
2474         const struct rte_memzone *tz;
2475         struct ixgbe_tx_queue *txq;
2476         struct ixgbe_hw     *hw;
2477         uint16_t tx_rs_thresh, tx_free_thresh;
2478
2479         PMD_INIT_FUNC_TRACE();
2480         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2481
2482         /*
2483          * Don't verify port offloads for application which
2484          * use the old API.
2485          */
2486         if (!ixgbe_check_tx_queue_offloads(dev, tx_conf->offloads)) {
2487                 PMD_INIT_LOG(ERR, "%p: Tx queue offloads 0x%" PRIx64
2488                         " don't match port offloads 0x%" PRIx64
2489                         " or supported queue offloads 0x%" PRIx64
2490                         " or supported port offloads 0x%" PRIx64,
2491                         (void *)dev, tx_conf->offloads,
2492                         dev->data->dev_conf.txmode.offloads,
2493                         ixgbe_get_tx_queue_offloads(dev),
2494                         ixgbe_get_tx_port_offloads(dev));
2495                 return -ENOTSUP;
2496         }
2497
2498         /*
2499          * Validate number of transmit descriptors.
2500          * It must not exceed hardware maximum, and must be multiple
2501          * of IXGBE_ALIGN.
2502          */
2503         if (nb_desc % IXGBE_TXD_ALIGN != 0 ||
2504                         (nb_desc > IXGBE_MAX_RING_DESC) ||
2505                         (nb_desc < IXGBE_MIN_RING_DESC)) {
2506                 return -EINVAL;
2507         }
2508
2509         /*
2510          * The following two parameters control the setting of the RS bit on
2511          * transmit descriptors.
2512          * TX descriptors will have their RS bit set after txq->tx_rs_thresh
2513          * descriptors have been used.
2514          * The TX descriptor ring will be cleaned after txq->tx_free_thresh
2515          * descriptors are used or if the number of descriptors required
2516          * to transmit a packet is greater than the number of free TX
2517          * descriptors.
2518          * The following constraints must be satisfied:
2519          *  tx_rs_thresh must be greater than 0.
2520          *  tx_rs_thresh must be less than the size of the ring minus 2.
2521          *  tx_rs_thresh must be less than or equal to tx_free_thresh.
2522          *  tx_rs_thresh must be a divisor of the ring size.
2523          *  tx_free_thresh must be greater than 0.
2524          *  tx_free_thresh must be less than the size of the ring minus 3.
2525          * One descriptor in the TX ring is used as a sentinel to avoid a
2526          * H/W race condition, hence the maximum threshold constraints.
2527          * When set to zero use default values.
2528          */
2529         tx_rs_thresh = (uint16_t)((tx_conf->tx_rs_thresh) ?
2530                         tx_conf->tx_rs_thresh : DEFAULT_TX_RS_THRESH);
2531         tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ?
2532                         tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH);
2533         if (tx_rs_thresh >= (nb_desc - 2)) {
2534                 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than the number "
2535                         "of TX descriptors minus 2. (tx_rs_thresh=%u "
2536                         "port=%d queue=%d)", (unsigned int)tx_rs_thresh,
2537                         (int)dev->data->port_id, (int)queue_idx);
2538                 return -(EINVAL);
2539         }
2540         if (tx_rs_thresh > DEFAULT_TX_RS_THRESH) {
2541                 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less or equal than %u. "
2542                         "(tx_rs_thresh=%u port=%d queue=%d)",
2543                         DEFAULT_TX_RS_THRESH, (unsigned int)tx_rs_thresh,
2544                         (int)dev->data->port_id, (int)queue_idx);
2545                 return -(EINVAL);
2546         }
2547         if (tx_free_thresh >= (nb_desc - 3)) {
2548                 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than the "
2549                              "tx_free_thresh must be less than the number of "
2550                              "TX descriptors minus 3. (tx_free_thresh=%u "
2551                              "port=%d queue=%d)",
2552                              (unsigned int)tx_free_thresh,
2553                              (int)dev->data->port_id, (int)queue_idx);
2554                 return -(EINVAL);
2555         }
2556         if (tx_rs_thresh > tx_free_thresh) {
2557                 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than or equal to "
2558                              "tx_free_thresh. (tx_free_thresh=%u "
2559                              "tx_rs_thresh=%u port=%d queue=%d)",
2560                              (unsigned int)tx_free_thresh,
2561                              (unsigned int)tx_rs_thresh,
2562                              (int)dev->data->port_id,
2563                              (int)queue_idx);
2564                 return -(EINVAL);
2565         }
2566         if ((nb_desc % tx_rs_thresh) != 0) {
2567                 PMD_INIT_LOG(ERR, "tx_rs_thresh must be a divisor of the "
2568                              "number of TX descriptors. (tx_rs_thresh=%u "
2569                              "port=%d queue=%d)", (unsigned int)tx_rs_thresh,
2570                              (int)dev->data->port_id, (int)queue_idx);
2571                 return -(EINVAL);
2572         }
2573
2574         /*
2575          * If rs_bit_thresh is greater than 1, then TX WTHRESH should be
2576          * set to 0. If WTHRESH is greater than zero, the RS bit is ignored
2577          * by the NIC and all descriptors are written back after the NIC
2578          * accumulates WTHRESH descriptors.
2579          */
2580         if ((tx_rs_thresh > 1) && (tx_conf->tx_thresh.wthresh != 0)) {
2581                 PMD_INIT_LOG(ERR, "TX WTHRESH must be set to 0 if "
2582                              "tx_rs_thresh is greater than 1. (tx_rs_thresh=%u "
2583                              "port=%d queue=%d)", (unsigned int)tx_rs_thresh,
2584                              (int)dev->data->port_id, (int)queue_idx);
2585                 return -(EINVAL);
2586         }
2587
2588         /* Free memory prior to re-allocation if needed... */
2589         if (dev->data->tx_queues[queue_idx] != NULL) {
2590                 ixgbe_tx_queue_release(dev->data->tx_queues[queue_idx]);
2591                 dev->data->tx_queues[queue_idx] = NULL;
2592         }
2593
2594         /* First allocate the tx queue data structure */
2595         txq = rte_zmalloc_socket("ethdev TX queue", sizeof(struct ixgbe_tx_queue),
2596                                  RTE_CACHE_LINE_SIZE, socket_id);
2597         if (txq == NULL)
2598                 return -ENOMEM;
2599
2600         /*
2601          * Allocate TX ring hardware descriptors. A memzone large enough to
2602          * handle the maximum ring size is allocated in order to allow for
2603          * resizing in later calls to the queue setup function.
2604          */
2605         tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx,
2606                         sizeof(union ixgbe_adv_tx_desc) * IXGBE_MAX_RING_DESC,
2607                         IXGBE_ALIGN, socket_id);
2608         if (tz == NULL) {
2609                 ixgbe_tx_queue_release(txq);
2610                 return -ENOMEM;
2611         }
2612
2613         txq->nb_tx_desc = nb_desc;
2614         txq->tx_rs_thresh = tx_rs_thresh;
2615         txq->tx_free_thresh = tx_free_thresh;
2616         txq->pthresh = tx_conf->tx_thresh.pthresh;
2617         txq->hthresh = tx_conf->tx_thresh.hthresh;
2618         txq->wthresh = tx_conf->tx_thresh.wthresh;
2619         txq->queue_id = queue_idx;
2620         txq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ?
2621                 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx);
2622         txq->port_id = dev->data->port_id;
2623         txq->offloads = tx_conf->offloads;
2624         txq->ops = &def_txq_ops;
2625         txq->tx_deferred_start = tx_conf->tx_deferred_start;
2626 #ifdef RTE_LIBRTE_SECURITY
2627         txq->using_ipsec = !!(dev->data->dev_conf.txmode.offloads &
2628                         DEV_TX_OFFLOAD_SECURITY);
2629 #endif
2630
2631         /*
2632          * Modification to set VFTDT for virtual function if vf is detected
2633          */
2634         if (hw->mac.type == ixgbe_mac_82599_vf ||
2635             hw->mac.type == ixgbe_mac_X540_vf ||
2636             hw->mac.type == ixgbe_mac_X550_vf ||
2637             hw->mac.type == ixgbe_mac_X550EM_x_vf ||
2638             hw->mac.type == ixgbe_mac_X550EM_a_vf)
2639                 txq->tdt_reg_addr = IXGBE_PCI_REG_ADDR(hw, IXGBE_VFTDT(queue_idx));
2640         else
2641                 txq->tdt_reg_addr = IXGBE_PCI_REG_ADDR(hw, IXGBE_TDT(txq->reg_idx));
2642
2643         txq->tx_ring_phys_addr = tz->iova;
2644         txq->tx_ring = (union ixgbe_adv_tx_desc *) tz->addr;
2645
2646         /* Allocate software ring */
2647         txq->sw_ring = rte_zmalloc_socket("txq->sw_ring",
2648                                 sizeof(struct ixgbe_tx_entry) * nb_desc,
2649                                 RTE_CACHE_LINE_SIZE, socket_id);
2650         if (txq->sw_ring == NULL) {
2651                 ixgbe_tx_queue_release(txq);
2652                 return -ENOMEM;
2653         }
2654         PMD_INIT_LOG(DEBUG, "sw_ring=%p hw_ring=%p dma_addr=0x%"PRIx64,
2655                      txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr);
2656
2657         /* set up vector or scalar TX function as appropriate */
2658         ixgbe_set_tx_function(dev, txq);
2659
2660         txq->ops->reset(txq);
2661
2662         dev->data->tx_queues[queue_idx] = txq;
2663
2664
2665         return 0;
2666 }
2667
2668 /**
2669  * ixgbe_free_sc_cluster - free the not-yet-completed scattered cluster
2670  *
2671  * The "next" pointer of the last segment of (not-yet-completed) RSC clusters
2672  * in the sw_rsc_ring is not set to NULL but rather points to the next
2673  * mbuf of this RSC aggregation (that has not been completed yet and still
2674  * resides on the HW ring). So, instead of calling for rte_pktmbuf_free() we
2675  * will just free first "nb_segs" segments of the cluster explicitly by calling
2676  * an rte_pktmbuf_free_seg().
2677  *
2678  * @m scattered cluster head
2679  */
2680 static void __attribute__((cold))
2681 ixgbe_free_sc_cluster(struct rte_mbuf *m)
2682 {
2683         uint16_t i, nb_segs = m->nb_segs;
2684         struct rte_mbuf *next_seg;
2685
2686         for (i = 0; i < nb_segs; i++) {
2687                 next_seg = m->next;
2688                 rte_pktmbuf_free_seg(m);
2689                 m = next_seg;
2690         }
2691 }
2692
2693 static void __attribute__((cold))
2694 ixgbe_rx_queue_release_mbufs(struct ixgbe_rx_queue *rxq)
2695 {
2696         unsigned i;
2697
2698 #ifdef RTE_IXGBE_INC_VECTOR
2699         /* SSE Vector driver has a different way of releasing mbufs. */
2700         if (rxq->rx_using_sse) {
2701                 ixgbe_rx_queue_release_mbufs_vec(rxq);
2702                 return;
2703         }
2704 #endif
2705
2706         if (rxq->sw_ring != NULL) {
2707                 for (i = 0; i < rxq->nb_rx_desc; i++) {
2708                         if (rxq->sw_ring[i].mbuf != NULL) {
2709                                 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
2710                                 rxq->sw_ring[i].mbuf = NULL;
2711                         }
2712                 }
2713                 if (rxq->rx_nb_avail) {
2714                         for (i = 0; i < rxq->rx_nb_avail; ++i) {
2715                                 struct rte_mbuf *mb;
2716
2717                                 mb = rxq->rx_stage[rxq->rx_next_avail + i];
2718                                 rte_pktmbuf_free_seg(mb);
2719                         }
2720                         rxq->rx_nb_avail = 0;
2721                 }
2722         }
2723
2724         if (rxq->sw_sc_ring)
2725                 for (i = 0; i < rxq->nb_rx_desc; i++)
2726                         if (rxq->sw_sc_ring[i].fbuf) {
2727                                 ixgbe_free_sc_cluster(rxq->sw_sc_ring[i].fbuf);
2728                                 rxq->sw_sc_ring[i].fbuf = NULL;
2729                         }
2730 }
2731
2732 static void __attribute__((cold))
2733 ixgbe_rx_queue_release(struct ixgbe_rx_queue *rxq)
2734 {
2735         if (rxq != NULL) {
2736                 ixgbe_rx_queue_release_mbufs(rxq);
2737                 rte_free(rxq->sw_ring);
2738                 rte_free(rxq->sw_sc_ring);
2739                 rte_free(rxq);
2740         }
2741 }
2742
2743 void __attribute__((cold))
2744 ixgbe_dev_rx_queue_release(void *rxq)
2745 {
2746         ixgbe_rx_queue_release(rxq);
2747 }
2748
2749 /*
2750  * Check if Rx Burst Bulk Alloc function can be used.
2751  * Return
2752  *        0: the preconditions are satisfied and the bulk allocation function
2753  *           can be used.
2754  *  -EINVAL: the preconditions are NOT satisfied and the default Rx burst
2755  *           function must be used.
2756  */
2757 static inline int __attribute__((cold))
2758 check_rx_burst_bulk_alloc_preconditions(struct ixgbe_rx_queue *rxq)
2759 {
2760         int ret = 0;
2761
2762         /*
2763          * Make sure the following pre-conditions are satisfied:
2764          *   rxq->rx_free_thresh >= RTE_PMD_IXGBE_RX_MAX_BURST
2765          *   rxq->rx_free_thresh < rxq->nb_rx_desc
2766          *   (rxq->nb_rx_desc % rxq->rx_free_thresh) == 0
2767          * Scattered packets are not supported.  This should be checked
2768          * outside of this function.
2769          */
2770         if (!(rxq->rx_free_thresh >= RTE_PMD_IXGBE_RX_MAX_BURST)) {
2771                 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
2772                              "rxq->rx_free_thresh=%d, "
2773                              "RTE_PMD_IXGBE_RX_MAX_BURST=%d",
2774                              rxq->rx_free_thresh, RTE_PMD_IXGBE_RX_MAX_BURST);
2775                 ret = -EINVAL;
2776         } else if (!(rxq->rx_free_thresh < rxq->nb_rx_desc)) {
2777                 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
2778                              "rxq->rx_free_thresh=%d, "
2779                              "rxq->nb_rx_desc=%d",
2780                              rxq->rx_free_thresh, rxq->nb_rx_desc);
2781                 ret = -EINVAL;
2782         } else if (!((rxq->nb_rx_desc % rxq->rx_free_thresh) == 0)) {
2783                 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
2784                              "rxq->nb_rx_desc=%d, "
2785                              "rxq->rx_free_thresh=%d",
2786                              rxq->nb_rx_desc, rxq->rx_free_thresh);
2787                 ret = -EINVAL;
2788         }
2789
2790         return ret;
2791 }
2792
2793 /* Reset dynamic ixgbe_rx_queue fields back to defaults */
2794 static void __attribute__((cold))
2795 ixgbe_reset_rx_queue(struct ixgbe_adapter *adapter, struct ixgbe_rx_queue *rxq)
2796 {
2797         static const union ixgbe_adv_rx_desc zeroed_desc = {{0}};
2798         unsigned i;
2799         uint16_t len = rxq->nb_rx_desc;
2800
2801         /*
2802          * By default, the Rx queue setup function allocates enough memory for
2803          * IXGBE_MAX_RING_DESC.  The Rx Burst bulk allocation function requires
2804          * extra memory at the end of the descriptor ring to be zero'd out.
2805          */
2806         if (adapter->rx_bulk_alloc_allowed)
2807                 /* zero out extra memory */
2808                 len += RTE_PMD_IXGBE_RX_MAX_BURST;
2809
2810         /*
2811          * Zero out HW ring memory. Zero out extra memory at the end of
2812          * the H/W ring so look-ahead logic in Rx Burst bulk alloc function
2813          * reads extra memory as zeros.
2814          */
2815         for (i = 0; i < len; i++) {
2816                 rxq->rx_ring[i] = zeroed_desc;
2817         }
2818
2819         /*
2820          * initialize extra software ring entries. Space for these extra
2821          * entries is always allocated
2822          */
2823         memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf));
2824         for (i = rxq->nb_rx_desc; i < len; ++i) {
2825                 rxq->sw_ring[i].mbuf = &rxq->fake_mbuf;
2826         }
2827
2828         rxq->rx_nb_avail = 0;
2829         rxq->rx_next_avail = 0;
2830         rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1);
2831         rxq->rx_tail = 0;
2832         rxq->nb_rx_hold = 0;
2833         rxq->pkt_first_seg = NULL;
2834         rxq->pkt_last_seg = NULL;
2835
2836 #ifdef RTE_IXGBE_INC_VECTOR
2837         rxq->rxrearm_start = 0;
2838         rxq->rxrearm_nb = 0;
2839 #endif
2840 }
2841
2842 static int
2843 ixgbe_is_vf(struct rte_eth_dev *dev)
2844 {
2845         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2846
2847         switch (hw->mac.type) {
2848         case ixgbe_mac_82599_vf:
2849         case ixgbe_mac_X540_vf:
2850         case ixgbe_mac_X550_vf:
2851         case ixgbe_mac_X550EM_x_vf:
2852         case ixgbe_mac_X550EM_a_vf:
2853                 return 1;
2854         default:
2855                 return 0;
2856         }
2857 }
2858
2859 uint64_t
2860 ixgbe_get_rx_queue_offloads(struct rte_eth_dev *dev)
2861 {
2862         uint64_t offloads = 0;
2863         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2864
2865         if (hw->mac.type != ixgbe_mac_82598EB)
2866                 offloads |= DEV_RX_OFFLOAD_VLAN_STRIP;
2867
2868         return offloads;
2869 }
2870
2871 uint64_t
2872 ixgbe_get_rx_port_offloads(struct rte_eth_dev *dev)
2873 {
2874         uint64_t offloads;
2875         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2876
2877         offloads = DEV_RX_OFFLOAD_IPV4_CKSUM  |
2878                    DEV_RX_OFFLOAD_UDP_CKSUM   |
2879                    DEV_RX_OFFLOAD_TCP_CKSUM   |
2880                    DEV_RX_OFFLOAD_CRC_STRIP   |
2881                    DEV_RX_OFFLOAD_JUMBO_FRAME |
2882                    DEV_RX_OFFLOAD_SCATTER;
2883
2884         if (hw->mac.type == ixgbe_mac_82598EB)
2885                 offloads |= DEV_RX_OFFLOAD_VLAN_STRIP;
2886
2887         if (ixgbe_is_vf(dev) == 0)
2888                 offloads |= (DEV_RX_OFFLOAD_VLAN_FILTER |
2889                              DEV_RX_OFFLOAD_VLAN_EXTEND);
2890
2891         /*
2892          * RSC is only supported by 82599 and x540 PF devices in a non-SR-IOV
2893          * mode.
2894          */
2895         if ((hw->mac.type == ixgbe_mac_82599EB ||
2896              hw->mac.type == ixgbe_mac_X540) &&
2897             !RTE_ETH_DEV_SRIOV(dev).active)
2898                 offloads |= DEV_RX_OFFLOAD_TCP_LRO;
2899
2900         if (hw->mac.type == ixgbe_mac_82599EB ||
2901             hw->mac.type == ixgbe_mac_X540)
2902                 offloads |= DEV_RX_OFFLOAD_MACSEC_STRIP;
2903
2904         if (hw->mac.type == ixgbe_mac_X550 ||
2905             hw->mac.type == ixgbe_mac_X550EM_x ||
2906             hw->mac.type == ixgbe_mac_X550EM_a)
2907                 offloads |= DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM;
2908
2909 #ifdef RTE_LIBRTE_SECURITY
2910         if (dev->security_ctx)
2911                 offloads |= DEV_RX_OFFLOAD_SECURITY;
2912 #endif
2913
2914         return offloads;
2915 }
2916
2917 static int
2918 ixgbe_check_rx_queue_offloads(struct rte_eth_dev *dev, uint64_t requested)
2919 {
2920         uint64_t port_offloads = dev->data->dev_conf.rxmode.offloads;
2921         uint64_t queue_supported = ixgbe_get_rx_queue_offloads(dev);
2922         uint64_t port_supported = ixgbe_get_rx_port_offloads(dev);
2923
2924         if ((requested & (queue_supported | port_supported)) != requested)
2925                 return 0;
2926
2927         if ((port_offloads ^ requested) & port_supported)
2928                 return 0;
2929
2930         return 1;
2931 }
2932
2933 int __attribute__((cold))
2934 ixgbe_dev_rx_queue_setup(struct rte_eth_dev *dev,
2935                          uint16_t queue_idx,
2936                          uint16_t nb_desc,
2937                          unsigned int socket_id,
2938                          const struct rte_eth_rxconf *rx_conf,
2939                          struct rte_mempool *mp)
2940 {
2941         const struct rte_memzone *rz;
2942         struct ixgbe_rx_queue *rxq;
2943         struct ixgbe_hw     *hw;
2944         uint16_t len;
2945         struct ixgbe_adapter *adapter =
2946                 (struct ixgbe_adapter *)dev->data->dev_private;
2947
2948         PMD_INIT_FUNC_TRACE();
2949         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2950
2951         if (!ixgbe_check_rx_queue_offloads(dev, rx_conf->offloads)) {
2952                 PMD_INIT_LOG(ERR, "%p: Rx queue offloads 0x%" PRIx64
2953                         " don't match port offloads 0x%" PRIx64
2954                         " or supported port offloads 0x%" PRIx64
2955                         " or supported queue offloads 0x%" PRIx64,
2956                         (void *)dev, rx_conf->offloads,
2957                         dev->data->dev_conf.rxmode.offloads,
2958                         ixgbe_get_rx_port_offloads(dev),
2959                         ixgbe_get_rx_queue_offloads(dev));
2960                 return -ENOTSUP;
2961         }
2962
2963         /*
2964          * Validate number of receive descriptors.
2965          * It must not exceed hardware maximum, and must be multiple
2966          * of IXGBE_ALIGN.
2967          */
2968         if (nb_desc % IXGBE_RXD_ALIGN != 0 ||
2969                         (nb_desc > IXGBE_MAX_RING_DESC) ||
2970                         (nb_desc < IXGBE_MIN_RING_DESC)) {
2971                 return -EINVAL;
2972         }
2973
2974         /* Free memory prior to re-allocation if needed... */
2975         if (dev->data->rx_queues[queue_idx] != NULL) {
2976                 ixgbe_rx_queue_release(dev->data->rx_queues[queue_idx]);
2977                 dev->data->rx_queues[queue_idx] = NULL;
2978         }
2979
2980         /* First allocate the rx queue data structure */
2981         rxq = rte_zmalloc_socket("ethdev RX queue", sizeof(struct ixgbe_rx_queue),
2982                                  RTE_CACHE_LINE_SIZE, socket_id);
2983         if (rxq == NULL)
2984                 return -ENOMEM;
2985         rxq->mb_pool = mp;
2986         rxq->nb_rx_desc = nb_desc;
2987         rxq->rx_free_thresh = rx_conf->rx_free_thresh;
2988         rxq->queue_id = queue_idx;
2989         rxq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ?
2990                 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx);
2991         rxq->port_id = dev->data->port_id;
2992         rxq->crc_len = (uint8_t)((dev->data->dev_conf.rxmode.offloads &
2993                 DEV_RX_OFFLOAD_CRC_STRIP) ? 0 : ETHER_CRC_LEN);
2994         rxq->drop_en = rx_conf->rx_drop_en;
2995         rxq->rx_deferred_start = rx_conf->rx_deferred_start;
2996         rxq->offloads = rx_conf->offloads;
2997
2998         /*
2999          * The packet type in RX descriptor is different for different NICs.
3000          * Some bits are used for x550 but reserved for other NICS.
3001          * So set different masks for different NICs.
3002          */
3003         if (hw->mac.type == ixgbe_mac_X550 ||
3004             hw->mac.type == ixgbe_mac_X550EM_x ||
3005             hw->mac.type == ixgbe_mac_X550EM_a ||
3006             hw->mac.type == ixgbe_mac_X550_vf ||
3007             hw->mac.type == ixgbe_mac_X550EM_x_vf ||
3008             hw->mac.type == ixgbe_mac_X550EM_a_vf)
3009                 rxq->pkt_type_mask = IXGBE_PACKET_TYPE_MASK_X550;
3010         else
3011                 rxq->pkt_type_mask = IXGBE_PACKET_TYPE_MASK_82599;
3012
3013         /*
3014          * Allocate RX ring hardware descriptors. A memzone large enough to
3015          * handle the maximum ring size is allocated in order to allow for
3016          * resizing in later calls to the queue setup function.
3017          */
3018         rz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx,
3019                                       RX_RING_SZ, IXGBE_ALIGN, socket_id);
3020         if (rz == NULL) {
3021                 ixgbe_rx_queue_release(rxq);
3022                 return -ENOMEM;
3023         }
3024
3025         /*
3026          * Zero init all the descriptors in the ring.
3027          */
3028         memset(rz->addr, 0, RX_RING_SZ);
3029
3030         /*
3031          * Modified to setup VFRDT for Virtual Function
3032          */
3033         if (hw->mac.type == ixgbe_mac_82599_vf ||
3034             hw->mac.type == ixgbe_mac_X540_vf ||
3035             hw->mac.type == ixgbe_mac_X550_vf ||
3036             hw->mac.type == ixgbe_mac_X550EM_x_vf ||
3037             hw->mac.type == ixgbe_mac_X550EM_a_vf) {
3038                 rxq->rdt_reg_addr =
3039                         IXGBE_PCI_REG_ADDR(hw, IXGBE_VFRDT(queue_idx));
3040                 rxq->rdh_reg_addr =
3041                         IXGBE_PCI_REG_ADDR(hw, IXGBE_VFRDH(queue_idx));
3042         } else {
3043                 rxq->rdt_reg_addr =
3044                         IXGBE_PCI_REG_ADDR(hw, IXGBE_RDT(rxq->reg_idx));
3045                 rxq->rdh_reg_addr =
3046                         IXGBE_PCI_REG_ADDR(hw, IXGBE_RDH(rxq->reg_idx));
3047         }
3048
3049         rxq->rx_ring_phys_addr = rz->iova;
3050         rxq->rx_ring = (union ixgbe_adv_rx_desc *) rz->addr;
3051
3052         /*
3053          * Certain constraints must be met in order to use the bulk buffer
3054          * allocation Rx burst function. If any of Rx queues doesn't meet them
3055          * the feature should be disabled for the whole port.
3056          */
3057         if (check_rx_burst_bulk_alloc_preconditions(rxq)) {
3058                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Rx Bulk Alloc "
3059                                     "preconditions - canceling the feature for "
3060                                     "the whole port[%d]",
3061                              rxq->queue_id, rxq->port_id);
3062                 adapter->rx_bulk_alloc_allowed = false;
3063         }
3064
3065         /*
3066          * Allocate software ring. Allow for space at the end of the
3067          * S/W ring to make sure look-ahead logic in bulk alloc Rx burst
3068          * function does not access an invalid memory region.
3069          */
3070         len = nb_desc;
3071         if (adapter->rx_bulk_alloc_allowed)
3072                 len += RTE_PMD_IXGBE_RX_MAX_BURST;
3073
3074         rxq->sw_ring = rte_zmalloc_socket("rxq->sw_ring",
3075                                           sizeof(struct ixgbe_rx_entry) * len,
3076                                           RTE_CACHE_LINE_SIZE, socket_id);
3077         if (!rxq->sw_ring) {
3078                 ixgbe_rx_queue_release(rxq);
3079                 return -ENOMEM;
3080         }
3081
3082         /*
3083          * Always allocate even if it's not going to be needed in order to
3084          * simplify the code.
3085          *
3086          * This ring is used in LRO and Scattered Rx cases and Scattered Rx may
3087          * be requested in ixgbe_dev_rx_init(), which is called later from
3088          * dev_start() flow.
3089          */
3090         rxq->sw_sc_ring =
3091                 rte_zmalloc_socket("rxq->sw_sc_ring",
3092                                    sizeof(struct ixgbe_scattered_rx_entry) * len,
3093                                    RTE_CACHE_LINE_SIZE, socket_id);
3094         if (!rxq->sw_sc_ring) {
3095                 ixgbe_rx_queue_release(rxq);
3096                 return -ENOMEM;
3097         }
3098
3099         PMD_INIT_LOG(DEBUG, "sw_ring=%p sw_sc_ring=%p hw_ring=%p "
3100                             "dma_addr=0x%"PRIx64,
3101                      rxq->sw_ring, rxq->sw_sc_ring, rxq->rx_ring,
3102                      rxq->rx_ring_phys_addr);
3103
3104         if (!rte_is_power_of_2(nb_desc)) {
3105                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
3106                                     "preconditions - canceling the feature for "
3107                                     "the whole port[%d]",
3108                              rxq->queue_id, rxq->port_id);
3109                 adapter->rx_vec_allowed = false;
3110         } else
3111                 ixgbe_rxq_vec_setup(rxq);
3112
3113         dev->data->rx_queues[queue_idx] = rxq;
3114
3115         ixgbe_reset_rx_queue(adapter, rxq);
3116
3117         return 0;
3118 }
3119
3120 uint32_t
3121 ixgbe_dev_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id)
3122 {
3123 #define IXGBE_RXQ_SCAN_INTERVAL 4
3124         volatile union ixgbe_adv_rx_desc *rxdp;
3125         struct ixgbe_rx_queue *rxq;
3126         uint32_t desc = 0;
3127
3128         rxq = dev->data->rx_queues[rx_queue_id];
3129         rxdp = &(rxq->rx_ring[rxq->rx_tail]);
3130
3131         while ((desc < rxq->nb_rx_desc) &&
3132                 (rxdp->wb.upper.status_error &
3133                         rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD))) {
3134                 desc += IXGBE_RXQ_SCAN_INTERVAL;
3135                 rxdp += IXGBE_RXQ_SCAN_INTERVAL;
3136                 if (rxq->rx_tail + desc >= rxq->nb_rx_desc)
3137                         rxdp = &(rxq->rx_ring[rxq->rx_tail +
3138                                 desc - rxq->nb_rx_desc]);
3139         }
3140
3141         return desc;
3142 }
3143
3144 int
3145 ixgbe_dev_rx_descriptor_done(void *rx_queue, uint16_t offset)
3146 {
3147         volatile union ixgbe_adv_rx_desc *rxdp;
3148         struct ixgbe_rx_queue *rxq = rx_queue;
3149         uint32_t desc;
3150
3151         if (unlikely(offset >= rxq->nb_rx_desc))
3152                 return 0;
3153         desc = rxq->rx_tail + offset;
3154         if (desc >= rxq->nb_rx_desc)
3155                 desc -= rxq->nb_rx_desc;
3156
3157         rxdp = &rxq->rx_ring[desc];
3158         return !!(rxdp->wb.upper.status_error &
3159                         rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD));
3160 }
3161
3162 int
3163 ixgbe_dev_rx_descriptor_status(void *rx_queue, uint16_t offset)
3164 {
3165         struct ixgbe_rx_queue *rxq = rx_queue;
3166         volatile uint32_t *status;
3167         uint32_t nb_hold, desc;
3168
3169         if (unlikely(offset >= rxq->nb_rx_desc))
3170                 return -EINVAL;
3171
3172 #ifdef RTE_IXGBE_INC_VECTOR
3173         if (rxq->rx_using_sse)
3174                 nb_hold = rxq->rxrearm_nb;
3175         else
3176 #endif
3177                 nb_hold = rxq->nb_rx_hold;
3178         if (offset >= rxq->nb_rx_desc - nb_hold)
3179                 return RTE_ETH_RX_DESC_UNAVAIL;
3180
3181         desc = rxq->rx_tail + offset;
3182         if (desc >= rxq->nb_rx_desc)
3183                 desc -= rxq->nb_rx_desc;
3184
3185         status = &rxq->rx_ring[desc].wb.upper.status_error;
3186         if (*status & rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD))
3187                 return RTE_ETH_RX_DESC_DONE;
3188
3189         return RTE_ETH_RX_DESC_AVAIL;
3190 }
3191
3192 int
3193 ixgbe_dev_tx_descriptor_status(void *tx_queue, uint16_t offset)
3194 {
3195         struct ixgbe_tx_queue *txq = tx_queue;
3196         volatile uint32_t *status;
3197         uint32_t desc;
3198
3199         if (unlikely(offset >= txq->nb_tx_desc))
3200                 return -EINVAL;
3201
3202         desc = txq->tx_tail + offset;
3203         /* go to next desc that has the RS bit */
3204         desc = ((desc + txq->tx_rs_thresh - 1) / txq->tx_rs_thresh) *
3205                 txq->tx_rs_thresh;
3206         if (desc >= txq->nb_tx_desc) {
3207                 desc -= txq->nb_tx_desc;
3208                 if (desc >= txq->nb_tx_desc)
3209                         desc -= txq->nb_tx_desc;
3210         }
3211
3212         status = &txq->tx_ring[desc].wb.status;
3213         if (*status & rte_cpu_to_le_32(IXGBE_ADVTXD_STAT_DD))
3214                 return RTE_ETH_TX_DESC_DONE;
3215
3216         return RTE_ETH_TX_DESC_FULL;
3217 }
3218
3219 void __attribute__((cold))
3220 ixgbe_dev_clear_queues(struct rte_eth_dev *dev)
3221 {
3222         unsigned i;
3223         struct ixgbe_adapter *adapter =
3224                 (struct ixgbe_adapter *)dev->data->dev_private;
3225
3226         PMD_INIT_FUNC_TRACE();
3227
3228         for (i = 0; i < dev->data->nb_tx_queues; i++) {
3229                 struct ixgbe_tx_queue *txq = dev->data->tx_queues[i];
3230
3231                 if (txq != NULL) {
3232                         txq->ops->release_mbufs(txq);
3233                         txq->ops->reset(txq);
3234                 }
3235         }
3236
3237         for (i = 0; i < dev->data->nb_rx_queues; i++) {
3238                 struct ixgbe_rx_queue *rxq = dev->data->rx_queues[i];
3239
3240                 if (rxq != NULL) {
3241                         ixgbe_rx_queue_release_mbufs(rxq);
3242                         ixgbe_reset_rx_queue(adapter, rxq);
3243                 }
3244         }
3245 }
3246
3247 void
3248 ixgbe_dev_free_queues(struct rte_eth_dev *dev)
3249 {
3250         unsigned i;
3251
3252         PMD_INIT_FUNC_TRACE();
3253
3254         for (i = 0; i < dev->data->nb_rx_queues; i++) {
3255                 ixgbe_dev_rx_queue_release(dev->data->rx_queues[i]);
3256                 dev->data->rx_queues[i] = NULL;
3257         }
3258         dev->data->nb_rx_queues = 0;
3259
3260         for (i = 0; i < dev->data->nb_tx_queues; i++) {
3261                 ixgbe_dev_tx_queue_release(dev->data->tx_queues[i]);
3262                 dev->data->tx_queues[i] = NULL;
3263         }
3264         dev->data->nb_tx_queues = 0;
3265 }
3266
3267 /*********************************************************************
3268  *
3269  *  Device RX/TX init functions
3270  *
3271  **********************************************************************/
3272
3273 /**
3274  * Receive Side Scaling (RSS)
3275  * See section 7.1.2.8 in the following document:
3276  *     "Intel 82599 10 GbE Controller Datasheet" - Revision 2.1 October 2009
3277  *
3278  * Principles:
3279  * The source and destination IP addresses of the IP header and the source
3280  * and destination ports of TCP/UDP headers, if any, of received packets are
3281  * hashed against a configurable random key to compute a 32-bit RSS hash result.
3282  * The seven (7) LSBs of the 32-bit hash result are used as an index into a
3283  * 128-entry redirection table (RETA).  Each entry of the RETA provides a 3-bit
3284  * RSS output index which is used as the RX queue index where to store the
3285  * received packets.
3286  * The following output is supplied in the RX write-back descriptor:
3287  *     - 32-bit result of the Microsoft RSS hash function,
3288  *     - 4-bit RSS type field.
3289  */
3290
3291 /*
3292  * RSS random key supplied in section 7.1.2.8.3 of the Intel 82599 datasheet.
3293  * Used as the default key.
3294  */
3295 static uint8_t rss_intel_key[40] = {
3296         0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
3297         0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
3298         0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
3299         0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
3300         0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
3301 };
3302
3303 static void
3304 ixgbe_rss_disable(struct rte_eth_dev *dev)
3305 {
3306         struct ixgbe_hw *hw;
3307         uint32_t mrqc;
3308         uint32_t mrqc_reg;
3309
3310         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3311         mrqc_reg = ixgbe_mrqc_reg_get(hw->mac.type);
3312         mrqc = IXGBE_READ_REG(hw, mrqc_reg);
3313         mrqc &= ~IXGBE_MRQC_RSSEN;
3314         IXGBE_WRITE_REG(hw, mrqc_reg, mrqc);
3315 }
3316
3317 static void
3318 ixgbe_hw_rss_hash_set(struct ixgbe_hw *hw, struct rte_eth_rss_conf *rss_conf)
3319 {
3320         uint8_t  *hash_key;
3321         uint32_t mrqc;
3322         uint32_t rss_key;
3323         uint64_t rss_hf;
3324         uint16_t i;
3325         uint32_t mrqc_reg;
3326         uint32_t rssrk_reg;
3327
3328         mrqc_reg = ixgbe_mrqc_reg_get(hw->mac.type);
3329         rssrk_reg = ixgbe_rssrk_reg_get(hw->mac.type, 0);
3330
3331         hash_key = rss_conf->rss_key;
3332         if (hash_key != NULL) {
3333                 /* Fill in RSS hash key */
3334                 for (i = 0; i < 10; i++) {
3335                         rss_key  = hash_key[(i * 4)];
3336                         rss_key |= hash_key[(i * 4) + 1] << 8;
3337                         rss_key |= hash_key[(i * 4) + 2] << 16;
3338                         rss_key |= hash_key[(i * 4) + 3] << 24;
3339                         IXGBE_WRITE_REG_ARRAY(hw, rssrk_reg, i, rss_key);
3340                 }
3341         }
3342
3343         /* Set configured hashing protocols in MRQC register */
3344         rss_hf = rss_conf->rss_hf;
3345         mrqc = IXGBE_MRQC_RSSEN; /* Enable RSS */
3346         if (rss_hf & ETH_RSS_IPV4)
3347                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4;
3348         if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
3349                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP;
3350         if (rss_hf & ETH_RSS_IPV6)
3351                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6;
3352         if (rss_hf & ETH_RSS_IPV6_EX)
3353                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX;
3354         if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP)
3355                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP;
3356         if (rss_hf & ETH_RSS_IPV6_TCP_EX)
3357                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP;
3358         if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP)
3359                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP;
3360         if (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP)
3361                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP;
3362         if (rss_hf & ETH_RSS_IPV6_UDP_EX)
3363                 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
3364         IXGBE_WRITE_REG(hw, mrqc_reg, mrqc);
3365 }
3366
3367 int
3368 ixgbe_dev_rss_hash_update(struct rte_eth_dev *dev,
3369                           struct rte_eth_rss_conf *rss_conf)
3370 {
3371         struct ixgbe_hw *hw;
3372         uint32_t mrqc;
3373         uint64_t rss_hf;
3374         uint32_t mrqc_reg;
3375
3376         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3377
3378         if (!ixgbe_rss_update_sp(hw->mac.type)) {
3379                 PMD_DRV_LOG(ERR, "RSS hash update is not supported on this "
3380                         "NIC.");
3381                 return -ENOTSUP;
3382         }
3383         mrqc_reg = ixgbe_mrqc_reg_get(hw->mac.type);
3384
3385         /*
3386          * Excerpt from section 7.1.2.8 Receive-Side Scaling (RSS):
3387          *     "RSS enabling cannot be done dynamically while it must be
3388          *      preceded by a software reset"
3389          * Before changing anything, first check that the update RSS operation
3390          * does not attempt to disable RSS, if RSS was enabled at
3391          * initialization time, or does not attempt to enable RSS, if RSS was
3392          * disabled at initialization time.
3393          */
3394         rss_hf = rss_conf->rss_hf & IXGBE_RSS_OFFLOAD_ALL;
3395         mrqc = IXGBE_READ_REG(hw, mrqc_reg);
3396         if (!(mrqc & IXGBE_MRQC_RSSEN)) { /* RSS disabled */
3397                 if (rss_hf != 0) /* Enable RSS */
3398                         return -(EINVAL);
3399                 return 0; /* Nothing to do */
3400         }
3401         /* RSS enabled */
3402         if (rss_hf == 0) /* Disable RSS */
3403                 return -(EINVAL);
3404         ixgbe_hw_rss_hash_set(hw, rss_conf);
3405         return 0;
3406 }
3407
3408 int
3409 ixgbe_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
3410                             struct rte_eth_rss_conf *rss_conf)
3411 {
3412         struct ixgbe_hw *hw;
3413         uint8_t *hash_key;
3414         uint32_t mrqc;
3415         uint32_t rss_key;
3416         uint64_t rss_hf;
3417         uint16_t i;
3418         uint32_t mrqc_reg;
3419         uint32_t rssrk_reg;
3420
3421         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3422         mrqc_reg = ixgbe_mrqc_reg_get(hw->mac.type);
3423         rssrk_reg = ixgbe_rssrk_reg_get(hw->mac.type, 0);
3424         hash_key = rss_conf->rss_key;
3425         if (hash_key != NULL) {
3426                 /* Return RSS hash key */
3427                 for (i = 0; i < 10; i++) {
3428                         rss_key = IXGBE_READ_REG_ARRAY(hw, rssrk_reg, i);
3429                         hash_key[(i * 4)] = rss_key & 0x000000FF;
3430                         hash_key[(i * 4) + 1] = (rss_key >> 8) & 0x000000FF;
3431                         hash_key[(i * 4) + 2] = (rss_key >> 16) & 0x000000FF;
3432                         hash_key[(i * 4) + 3] = (rss_key >> 24) & 0x000000FF;
3433                 }
3434         }
3435
3436         /* Get RSS functions configured in MRQC register */
3437         mrqc = IXGBE_READ_REG(hw, mrqc_reg);
3438         if ((mrqc & IXGBE_MRQC_RSSEN) == 0) { /* RSS is disabled */
3439                 rss_conf->rss_hf = 0;
3440                 return 0;
3441         }
3442         rss_hf = 0;
3443         if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV4)
3444                 rss_hf |= ETH_RSS_IPV4;
3445         if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV4_TCP)
3446                 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP;
3447         if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6)
3448                 rss_hf |= ETH_RSS_IPV6;
3449         if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_EX)
3450                 rss_hf |= ETH_RSS_IPV6_EX;
3451         if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_TCP)
3452                 rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP;
3453         if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP)
3454                 rss_hf |= ETH_RSS_IPV6_TCP_EX;
3455         if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV4_UDP)
3456                 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP;
3457         if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_UDP)
3458                 rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP;
3459         if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP)
3460                 rss_hf |= ETH_RSS_IPV6_UDP_EX;
3461         rss_conf->rss_hf = rss_hf;
3462         return 0;
3463 }
3464
3465 static void
3466 ixgbe_rss_configure(struct rte_eth_dev *dev)
3467 {
3468         struct rte_eth_rss_conf rss_conf;
3469         struct ixgbe_hw *hw;
3470         uint32_t reta;
3471         uint16_t i;
3472         uint16_t j;
3473         uint16_t sp_reta_size;
3474         uint32_t reta_reg;
3475
3476         PMD_INIT_FUNC_TRACE();
3477         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3478
3479         sp_reta_size = ixgbe_reta_size_get(hw->mac.type);
3480
3481         /*
3482          * Fill in redirection table
3483          * The byte-swap is needed because NIC registers are in
3484          * little-endian order.
3485          */
3486         reta = 0;
3487         for (i = 0, j = 0; i < sp_reta_size; i++, j++) {
3488                 reta_reg = ixgbe_reta_reg_get(hw->mac.type, i);
3489
3490                 if (j == dev->data->nb_rx_queues)
3491                         j = 0;
3492                 reta = (reta << 8) | j;
3493                 if ((i & 3) == 3)
3494                         IXGBE_WRITE_REG(hw, reta_reg,
3495                                         rte_bswap32(reta));
3496         }
3497
3498         /*
3499          * Configure the RSS key and the RSS protocols used to compute
3500          * the RSS hash of input packets.
3501          */
3502         rss_conf = dev->data->dev_conf.rx_adv_conf.rss_conf;
3503         if ((rss_conf.rss_hf & IXGBE_RSS_OFFLOAD_ALL) == 0) {
3504                 ixgbe_rss_disable(dev);
3505                 return;
3506         }
3507         if (rss_conf.rss_key == NULL)
3508                 rss_conf.rss_key = rss_intel_key; /* Default hash key */
3509         ixgbe_hw_rss_hash_set(hw, &rss_conf);
3510 }
3511
3512 #define NUM_VFTA_REGISTERS 128
3513 #define NIC_RX_BUFFER_SIZE 0x200
3514 #define X550_RX_BUFFER_SIZE 0x180
3515
3516 static void
3517 ixgbe_vmdq_dcb_configure(struct rte_eth_dev *dev)
3518 {
3519         struct rte_eth_vmdq_dcb_conf *cfg;
3520         struct ixgbe_hw *hw;
3521         enum rte_eth_nb_pools num_pools;
3522         uint32_t mrqc, vt_ctl, queue_mapping, vlanctrl;
3523         uint16_t pbsize;
3524         uint8_t nb_tcs; /* number of traffic classes */
3525         int i;
3526
3527         PMD_INIT_FUNC_TRACE();
3528         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3529         cfg = &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf;
3530         num_pools = cfg->nb_queue_pools;
3531         /* Check we have a valid number of pools */
3532         if (num_pools != ETH_16_POOLS && num_pools != ETH_32_POOLS) {
3533                 ixgbe_rss_disable(dev);
3534                 return;
3535         }
3536         /* 16 pools -> 8 traffic classes, 32 pools -> 4 traffic classes */
3537         nb_tcs = (uint8_t)(ETH_VMDQ_DCB_NUM_QUEUES / (int)num_pools);
3538
3539         /*
3540          * RXPBSIZE
3541          * split rx buffer up into sections, each for 1 traffic class
3542          */
3543         switch (hw->mac.type) {
3544         case ixgbe_mac_X550:
3545         case ixgbe_mac_X550EM_x:
3546         case ixgbe_mac_X550EM_a:
3547                 pbsize = (uint16_t)(X550_RX_BUFFER_SIZE / nb_tcs);
3548                 break;
3549         default:
3550                 pbsize = (uint16_t)(NIC_RX_BUFFER_SIZE / nb_tcs);
3551                 break;
3552         }
3553         for (i = 0; i < nb_tcs; i++) {
3554                 uint32_t rxpbsize = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(i));
3555
3556                 rxpbsize &= (~(0x3FF << IXGBE_RXPBSIZE_SHIFT));
3557                 /* clear 10 bits. */
3558                 rxpbsize |= (pbsize << IXGBE_RXPBSIZE_SHIFT); /* set value */
3559                 IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), rxpbsize);
3560         }
3561         /* zero alloc all unused TCs */
3562         for (i = nb_tcs; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3563                 uint32_t rxpbsize = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(i));
3564
3565                 rxpbsize &= (~(0x3FF << IXGBE_RXPBSIZE_SHIFT));
3566                 /* clear 10 bits. */
3567                 IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), rxpbsize);
3568         }
3569
3570         /* MRQC: enable vmdq and dcb */
3571         mrqc = (num_pools == ETH_16_POOLS) ?
3572                 IXGBE_MRQC_VMDQRT8TCEN : IXGBE_MRQC_VMDQRT4TCEN;
3573         IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
3574
3575         /* PFVTCTL: turn on virtualisation and set the default pool */
3576         vt_ctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
3577         if (cfg->enable_default_pool) {
3578                 vt_ctl |= (cfg->default_pool << IXGBE_VT_CTL_POOL_SHIFT);
3579         } else {
3580                 vt_ctl |= IXGBE_VT_CTL_DIS_DEFPL;
3581         }
3582
3583         IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vt_ctl);
3584
3585         /* RTRUP2TC: mapping user priorities to traffic classes (TCs) */
3586         queue_mapping = 0;
3587         for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++)
3588                 /*
3589                  * mapping is done with 3 bits per priority,
3590                  * so shift by i*3 each time
3591                  */
3592                 queue_mapping |= ((cfg->dcb_tc[i] & 0x07) << (i * 3));
3593
3594         IXGBE_WRITE_REG(hw, IXGBE_RTRUP2TC, queue_mapping);
3595
3596         /* RTRPCS: DCB related */
3597         IXGBE_WRITE_REG(hw, IXGBE_RTRPCS, IXGBE_RMCS_RRM);
3598
3599         /* VLNCTRL: enable vlan filtering and allow all vlan tags through */
3600         vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
3601         vlanctrl |= IXGBE_VLNCTRL_VFE; /* enable vlan filters */
3602         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl);
3603
3604         /* VFTA - enable all vlan filters */
3605         for (i = 0; i < NUM_VFTA_REGISTERS; i++) {
3606                 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), 0xFFFFFFFF);
3607         }
3608
3609         /* VFRE: pool enabling for receive - 16 or 32 */
3610         IXGBE_WRITE_REG(hw, IXGBE_VFRE(0),
3611                         num_pools == ETH_16_POOLS ? 0xFFFF : 0xFFFFFFFF);
3612
3613         /*
3614          * MPSAR - allow pools to read specific mac addresses
3615          * In this case, all pools should be able to read from mac addr 0
3616          */
3617         IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(0), 0xFFFFFFFF);
3618         IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(0), 0xFFFFFFFF);
3619
3620         /* PFVLVF, PFVLVFB: set up filters for vlan tags as configured */
3621         for (i = 0; i < cfg->nb_pool_maps; i++) {
3622                 /* set vlan id in VF register and set the valid bit */
3623                 IXGBE_WRITE_REG(hw, IXGBE_VLVF(i), (IXGBE_VLVF_VIEN |
3624                                 (cfg->pool_map[i].vlan_id & 0xFFF)));
3625                 /*
3626                  * Put the allowed pools in VFB reg. As we only have 16 or 32
3627                  * pools, we only need to use the first half of the register
3628                  * i.e. bits 0-31
3629                  */
3630                 IXGBE_WRITE_REG(hw, IXGBE_VLVFB(i*2), cfg->pool_map[i].pools);
3631         }
3632 }
3633
3634 /**
3635  * ixgbe_dcb_config_tx_hw_config - Configure general DCB TX parameters
3636  * @dev: pointer to eth_dev structure
3637  * @dcb_config: pointer to ixgbe_dcb_config structure
3638  */
3639 static void
3640 ixgbe_dcb_tx_hw_config(struct rte_eth_dev *dev,
3641                        struct ixgbe_dcb_config *dcb_config)
3642 {
3643         uint32_t reg;
3644         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3645
3646         PMD_INIT_FUNC_TRACE();
3647         if (hw->mac.type != ixgbe_mac_82598EB) {
3648                 /* Disable the Tx desc arbiter so that MTQC can be changed */
3649                 reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3650                 reg |= IXGBE_RTTDCS_ARBDIS;
3651                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg);
3652
3653                 /* Enable DCB for Tx with 8 TCs */
3654                 if (dcb_config->num_tcs.pg_tcs == 8) {
3655                         reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;
3656                 } else {
3657                         reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
3658                 }
3659                 if (dcb_config->vt_mode)
3660                         reg |= IXGBE_MTQC_VT_ENA;
3661                 IXGBE_WRITE_REG(hw, IXGBE_MTQC, reg);
3662
3663                 /* Enable the Tx desc arbiter */
3664                 reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3665                 reg &= ~IXGBE_RTTDCS_ARBDIS;
3666                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg);
3667
3668                 /* Enable Security TX Buffer IFG for DCB */
3669                 reg = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG);
3670                 reg |= IXGBE_SECTX_DCB;
3671                 IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, reg);
3672         }
3673 }
3674
3675 /**
3676  * ixgbe_vmdq_dcb_hw_tx_config - Configure general VMDQ+DCB TX parameters
3677  * @dev: pointer to rte_eth_dev structure
3678  * @dcb_config: pointer to ixgbe_dcb_config structure
3679  */
3680 static void
3681 ixgbe_vmdq_dcb_hw_tx_config(struct rte_eth_dev *dev,
3682                         struct ixgbe_dcb_config *dcb_config)
3683 {
3684         struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3685                         &dev->data->dev_conf.tx_adv_conf.vmdq_dcb_tx_conf;
3686         struct ixgbe_hw *hw =
3687                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3688
3689         PMD_INIT_FUNC_TRACE();
3690         if (hw->mac.type != ixgbe_mac_82598EB)
3691                 /*PF VF Transmit Enable*/
3692                 IXGBE_WRITE_REG(hw, IXGBE_VFTE(0),
3693                         vmdq_tx_conf->nb_queue_pools == ETH_16_POOLS ? 0xFFFF : 0xFFFFFFFF);
3694
3695         /*Configure general DCB TX parameters*/
3696         ixgbe_dcb_tx_hw_config(dev, dcb_config);
3697 }
3698
3699 static void
3700 ixgbe_vmdq_dcb_rx_config(struct rte_eth_dev *dev,
3701                         struct ixgbe_dcb_config *dcb_config)
3702 {
3703         struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3704                         &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf;
3705         struct ixgbe_dcb_tc_config *tc;
3706         uint8_t i, j;
3707
3708         /* convert rte_eth_conf.rx_adv_conf to struct ixgbe_dcb_config */
3709         if (vmdq_rx_conf->nb_queue_pools == ETH_16_POOLS) {
3710                 dcb_config->num_tcs.pg_tcs = ETH_8_TCS;
3711                 dcb_config->num_tcs.pfc_tcs = ETH_8_TCS;
3712         } else {
3713                 dcb_config->num_tcs.pg_tcs = ETH_4_TCS;
3714                 dcb_config->num_tcs.pfc_tcs = ETH_4_TCS;
3715         }
3716
3717         /* Initialize User Priority to Traffic Class mapping */
3718         for (j = 0; j < IXGBE_DCB_MAX_TRAFFIC_CLASS; j++) {
3719                 tc = &dcb_config->tc_config[j];
3720                 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0;
3721         }
3722
3723         /* User Priority to Traffic Class mapping */
3724         for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3725                 j = vmdq_rx_conf->dcb_tc[i];
3726                 tc = &dcb_config->tc_config[j];
3727                 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap |=
3728                                                 (uint8_t)(1 << i);
3729         }
3730 }
3731
3732 static void
3733 ixgbe_dcb_vt_tx_config(struct rte_eth_dev *dev,
3734                         struct ixgbe_dcb_config *dcb_config)
3735 {
3736         struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3737                         &dev->data->dev_conf.tx_adv_conf.vmdq_dcb_tx_conf;
3738         struct ixgbe_dcb_tc_config *tc;
3739         uint8_t i, j;
3740
3741         /* convert rte_eth_conf.rx_adv_conf to struct ixgbe_dcb_config */
3742         if (vmdq_tx_conf->nb_queue_pools == ETH_16_POOLS) {
3743                 dcb_config->num_tcs.pg_tcs = ETH_8_TCS;
3744                 dcb_config->num_tcs.pfc_tcs = ETH_8_TCS;
3745         } else {
3746                 dcb_config->num_tcs.pg_tcs = ETH_4_TCS;
3747                 dcb_config->num_tcs.pfc_tcs = ETH_4_TCS;
3748         }
3749
3750         /* Initialize User Priority to Traffic Class mapping */
3751         for (j = 0; j < IXGBE_DCB_MAX_TRAFFIC_CLASS; j++) {
3752                 tc = &dcb_config->tc_config[j];
3753                 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0;
3754         }
3755
3756         /* User Priority to Traffic Class mapping */
3757         for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3758                 j = vmdq_tx_conf->dcb_tc[i];
3759                 tc = &dcb_config->tc_config[j];
3760                 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap |=
3761                                                 (uint8_t)(1 << i);
3762         }
3763 }
3764
3765 static void
3766 ixgbe_dcb_rx_config(struct rte_eth_dev *dev,
3767                 struct ixgbe_dcb_config *dcb_config)
3768 {
3769         struct rte_eth_dcb_rx_conf *rx_conf =
3770                         &dev->data->dev_conf.rx_adv_conf.dcb_rx_conf;
3771         struct ixgbe_dcb_tc_config *tc;
3772         uint8_t i, j;
3773
3774         dcb_config->num_tcs.pg_tcs = (uint8_t)rx_conf->nb_tcs;
3775         dcb_config->num_tcs.pfc_tcs = (uint8_t)rx_conf->nb_tcs;
3776
3777         /* Initialize User Priority to Traffic Class mapping */
3778         for (j = 0; j < IXGBE_DCB_MAX_TRAFFIC_CLASS; j++) {
3779                 tc = &dcb_config->tc_config[j];
3780                 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0;
3781         }
3782
3783         /* User Priority to Traffic Class mapping */
3784         for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3785                 j = rx_conf->dcb_tc[i];
3786                 tc = &dcb_config->tc_config[j];
3787                 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap |=
3788                                                 (uint8_t)(1 << i);
3789         }
3790 }
3791
3792 static void
3793 ixgbe_dcb_tx_config(struct rte_eth_dev *dev,
3794                 struct ixgbe_dcb_config *dcb_config)
3795 {
3796         struct rte_eth_dcb_tx_conf *tx_conf =
3797                         &dev->data->dev_conf.tx_adv_conf.dcb_tx_conf;
3798         struct ixgbe_dcb_tc_config *tc;
3799         uint8_t i, j;
3800
3801         dcb_config->num_tcs.pg_tcs = (uint8_t)tx_conf->nb_tcs;
3802         dcb_config->num_tcs.pfc_tcs = (uint8_t)tx_conf->nb_tcs;
3803
3804         /* Initialize User Priority to Traffic Class mapping */
3805         for (j = 0; j < IXGBE_DCB_MAX_TRAFFIC_CLASS; j++) {
3806                 tc = &dcb_config->tc_config[j];
3807                 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0;
3808         }
3809
3810         /* User Priority to Traffic Class mapping */
3811         for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3812                 j = tx_conf->dcb_tc[i];
3813                 tc = &dcb_config->tc_config[j];
3814                 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap |=
3815                                                 (uint8_t)(1 << i);
3816         }
3817 }
3818
3819 /**
3820  * ixgbe_dcb_rx_hw_config - Configure general DCB RX HW parameters
3821  * @dev: pointer to eth_dev structure
3822  * @dcb_config: pointer to ixgbe_dcb_config structure
3823  */
3824 static void
3825 ixgbe_dcb_rx_hw_config(struct rte_eth_dev *dev,
3826                        struct ixgbe_dcb_config *dcb_config)
3827 {
3828         uint32_t reg;
3829         uint32_t vlanctrl;
3830         uint8_t i;
3831         uint32_t q;
3832         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3833
3834         PMD_INIT_FUNC_TRACE();
3835         /*
3836          * Disable the arbiter before changing parameters
3837          * (always enable recycle mode; WSP)
3838          */
3839         reg = IXGBE_RTRPCS_RRM | IXGBE_RTRPCS_RAC | IXGBE_RTRPCS_ARBDIS;
3840         IXGBE_WRITE_REG(hw, IXGBE_RTRPCS, reg);
3841
3842         if (hw->mac.type != ixgbe_mac_82598EB) {
3843                 reg = IXGBE_READ_REG(hw, IXGBE_MRQC);
3844                 if (dcb_config->num_tcs.pg_tcs == 4) {
3845                         if (dcb_config->vt_mode)
3846                                 reg = (reg & ~IXGBE_MRQC_MRQE_MASK) |
3847                                         IXGBE_MRQC_VMDQRT4TCEN;
3848                         else {
3849                                 /* no matter the mode is DCB or DCB_RSS, just
3850                                  * set the MRQE to RSSXTCEN. RSS is controlled
3851                                  * by RSS_FIELD
3852                                  */
3853                                 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, 0);
3854                                 reg = (reg & ~IXGBE_MRQC_MRQE_MASK) |
3855                                         IXGBE_MRQC_RTRSS4TCEN;
3856                         }
3857                 }
3858                 if (dcb_config->num_tcs.pg_tcs == 8) {
3859                         if (dcb_config->vt_mode)
3860                                 reg = (reg & ~IXGBE_MRQC_MRQE_MASK) |
3861                                         IXGBE_MRQC_VMDQRT8TCEN;
3862                         else {
3863                                 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, 0);
3864                                 reg = (reg & ~IXGBE_MRQC_MRQE_MASK) |
3865                                         IXGBE_MRQC_RTRSS8TCEN;
3866                         }
3867                 }
3868
3869                 IXGBE_WRITE_REG(hw, IXGBE_MRQC, reg);
3870
3871                 if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
3872                         /* Disable drop for all queues in VMDQ mode*/
3873                         for (q = 0; q < IXGBE_MAX_RX_QUEUE_NUM; q++)
3874                                 IXGBE_WRITE_REG(hw, IXGBE_QDE,
3875                                                 (IXGBE_QDE_WRITE |
3876                                                  (q << IXGBE_QDE_IDX_SHIFT)));
3877                 } else {
3878                         /* Enable drop for all queues in SRIOV mode */
3879                         for (q = 0; q < IXGBE_MAX_RX_QUEUE_NUM; q++)
3880                                 IXGBE_WRITE_REG(hw, IXGBE_QDE,
3881                                                 (IXGBE_QDE_WRITE |
3882                                                  (q << IXGBE_QDE_IDX_SHIFT) |
3883                                                  IXGBE_QDE_ENABLE));
3884                 }
3885         }
3886
3887         /* VLNCTRL: enable vlan filtering and allow all vlan tags through */
3888         vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
3889         vlanctrl |= IXGBE_VLNCTRL_VFE; /* enable vlan filters */
3890         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl);
3891
3892         /* VFTA - enable all vlan filters */
3893         for (i = 0; i < NUM_VFTA_REGISTERS; i++) {
3894                 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), 0xFFFFFFFF);
3895         }
3896
3897         /*
3898          * Configure Rx packet plane (recycle mode; WSP) and
3899          * enable arbiter
3900          */
3901         reg = IXGBE_RTRPCS_RRM | IXGBE_RTRPCS_RAC;
3902         IXGBE_WRITE_REG(hw, IXGBE_RTRPCS, reg);
3903 }
3904
3905 static void
3906 ixgbe_dcb_hw_arbite_rx_config(struct ixgbe_hw *hw, uint16_t *refill,
3907                         uint16_t *max, uint8_t *bwg_id, uint8_t *tsa, uint8_t *map)
3908 {
3909         switch (hw->mac.type) {
3910         case ixgbe_mac_82598EB:
3911                 ixgbe_dcb_config_rx_arbiter_82598(hw, refill, max, tsa);
3912                 break;
3913         case ixgbe_mac_82599EB:
3914         case ixgbe_mac_X540:
3915         case ixgbe_mac_X550:
3916         case ixgbe_mac_X550EM_x:
3917         case ixgbe_mac_X550EM_a:
3918                 ixgbe_dcb_config_rx_arbiter_82599(hw, refill, max, bwg_id,
3919                                                   tsa, map);
3920                 break;
3921         default:
3922                 break;
3923         }
3924 }
3925
3926 static void
3927 ixgbe_dcb_hw_arbite_tx_config(struct ixgbe_hw *hw, uint16_t *refill, uint16_t *max,
3928                             uint8_t *bwg_id, uint8_t *tsa, uint8_t *map)
3929 {
3930         switch (hw->mac.type) {
3931         case ixgbe_mac_82598EB:
3932                 ixgbe_dcb_config_tx_desc_arbiter_82598(hw, refill, max, bwg_id, tsa);
3933                 ixgbe_dcb_config_tx_data_arbiter_82598(hw, refill, max, bwg_id, tsa);
3934                 break;
3935         case ixgbe_mac_82599EB:
3936         case ixgbe_mac_X540:
3937         case ixgbe_mac_X550:
3938         case ixgbe_mac_X550EM_x:
3939         case ixgbe_mac_X550EM_a:
3940                 ixgbe_dcb_config_tx_desc_arbiter_82599(hw, refill, max, bwg_id, tsa);
3941                 ixgbe_dcb_config_tx_data_arbiter_82599(hw, refill, max, bwg_id, tsa, map);
3942                 break;
3943         default:
3944                 break;
3945         }
3946 }
3947
3948 #define DCB_RX_CONFIG  1
3949 #define DCB_TX_CONFIG  1
3950 #define DCB_TX_PB      1024
3951 /**
3952  * ixgbe_dcb_hw_configure - Enable DCB and configure
3953  * general DCB in VT mode and non-VT mode parameters
3954  * @dev: pointer to rte_eth_dev structure
3955  * @dcb_config: pointer to ixgbe_dcb_config structure
3956  */
3957 static int
3958 ixgbe_dcb_hw_configure(struct rte_eth_dev *dev,
3959                         struct ixgbe_dcb_config *dcb_config)
3960 {
3961         int     ret = 0;
3962         uint8_t i, pfc_en, nb_tcs;
3963         uint16_t pbsize, rx_buffer_size;
3964         uint8_t config_dcb_rx = 0;
3965         uint8_t config_dcb_tx = 0;
3966         uint8_t tsa[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0};
3967         uint8_t bwgid[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0};
3968         uint16_t refill[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0};
3969         uint16_t max[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0};
3970         uint8_t map[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0};
3971         struct ixgbe_dcb_tc_config *tc;
3972         uint32_t max_frame = dev->data->mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
3973         struct ixgbe_hw *hw =
3974                         IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3975         struct ixgbe_bw_conf *bw_conf =
3976                 IXGBE_DEV_PRIVATE_TO_BW_CONF(dev->data->dev_private);
3977
3978         switch (dev->data->dev_conf.rxmode.mq_mode) {
3979         case ETH_MQ_RX_VMDQ_DCB:
3980                 dcb_config->vt_mode = true;
3981                 if (hw->mac.type != ixgbe_mac_82598EB) {
3982                         config_dcb_rx = DCB_RX_CONFIG;
3983                         /*
3984                          *get dcb and VT rx configuration parameters
3985                          *from rte_eth_conf
3986                          */
3987                         ixgbe_vmdq_dcb_rx_config(dev, dcb_config);
3988                         /*Configure general VMDQ and DCB RX parameters*/
3989                         ixgbe_vmdq_dcb_configure(dev);
3990                 }
3991                 break;
3992         case ETH_MQ_RX_DCB:
3993         case ETH_MQ_RX_DCB_RSS:
3994                 dcb_config->vt_mode = false;
3995                 config_dcb_rx = DCB_RX_CONFIG;
3996                 /* Get dcb TX configuration parameters from rte_eth_conf */
3997                 ixgbe_dcb_rx_config(dev, dcb_config);
3998                 /*Configure general DCB RX parameters*/
3999                 ixgbe_dcb_rx_hw_config(dev, dcb_config);
4000                 break;
4001         default:
4002                 PMD_INIT_LOG(ERR, "Incorrect DCB RX mode configuration");
4003                 break;
4004         }
4005         switch (dev->data->dev_conf.txmode.mq_mode) {
4006         case ETH_MQ_TX_VMDQ_DCB:
4007                 dcb_config->vt_mode = true;
4008                 config_dcb_tx = DCB_TX_CONFIG;
4009                 /* get DCB and VT TX configuration parameters
4010                  * from rte_eth_conf
4011                  */
4012                 ixgbe_dcb_vt_tx_config(dev, dcb_config);
4013                 /*Configure general VMDQ and DCB TX parameters*/
4014                 ixgbe_vmdq_dcb_hw_tx_config(dev, dcb_config);
4015                 break;
4016
4017         case ETH_MQ_TX_DCB:
4018                 dcb_config->vt_mode = false;
4019                 config_dcb_tx = DCB_TX_CONFIG;
4020                 /*get DCB TX configuration parameters from rte_eth_conf*/
4021                 ixgbe_dcb_tx_config(dev, dcb_config);
4022                 /*Configure general DCB TX parameters*/
4023                 ixgbe_dcb_tx_hw_config(dev, dcb_config);
4024                 break;
4025         default:
4026                 PMD_INIT_LOG(ERR, "Incorrect DCB TX mode configuration");
4027                 break;
4028         }
4029
4030         nb_tcs = dcb_config->num_tcs.pfc_tcs;
4031         /* Unpack map */
4032         ixgbe_dcb_unpack_map_cee(dcb_config, IXGBE_DCB_RX_CONFIG, map);
4033         if (nb_tcs == ETH_4_TCS) {
4034                 /* Avoid un-configured priority mapping to TC0 */
4035                 uint8_t j = 4;
4036                 uint8_t mask = 0xFF;
4037
4038                 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES - 4; i++)
4039                         mask = (uint8_t)(mask & (~(1 << map[i])));
4040                 for (i = 0; mask && (i < IXGBE_DCB_MAX_TRAFFIC_CLASS); i++) {
4041                         if ((mask & 0x1) && (j < ETH_DCB_NUM_USER_PRIORITIES))
4042                                 map[j++] = i;
4043                         mask >>= 1;
4044                 }
4045                 /* Re-configure 4 TCs BW */
4046                 for (i = 0; i < nb_tcs; i++) {
4047                         tc = &dcb_config->tc_config[i];
4048                         if (bw_conf->tc_num != nb_tcs)
4049                                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent =
4050                                         (uint8_t)(100 / nb_tcs);
4051                         tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent =
4052                                                 (uint8_t)(100 / nb_tcs);
4053                 }
4054                 for (; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
4055                         tc = &dcb_config->tc_config[i];
4056                         tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent = 0;
4057                         tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent = 0;
4058                 }
4059         } else {
4060                 /* Re-configure 8 TCs BW */
4061                 for (i = 0; i < nb_tcs; i++) {
4062                         tc = &dcb_config->tc_config[i];
4063                         if (bw_conf->tc_num != nb_tcs)
4064                                 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent =
4065                                         (uint8_t)(100 / nb_tcs + (i & 1));
4066                         tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent =
4067                                 (uint8_t)(100 / nb_tcs + (i & 1));
4068                 }
4069         }
4070
4071         switch (hw->mac.type) {
4072         case ixgbe_mac_X550:
4073         case ixgbe_mac_X550EM_x:
4074         case ixgbe_mac_X550EM_a:
4075                 rx_buffer_size = X550_RX_BUFFER_SIZE;
4076                 break;
4077         default:
4078                 rx_buffer_size = NIC_RX_BUFFER_SIZE;
4079                 break;
4080         }
4081
4082         if (config_dcb_rx) {
4083                 /* Set RX buffer size */
4084                 pbsize = (uint16_t)(rx_buffer_size / nb_tcs);
4085                 uint32_t rxpbsize = pbsize << IXGBE_RXPBSIZE_SHIFT;
4086
4087                 for (i = 0; i < nb_tcs; i++) {
4088                         IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), rxpbsize);
4089                 }
4090                 /* zero alloc all unused TCs */
4091                 for (; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
4092                         IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), 0);
4093                 }
4094         }
4095         if (config_dcb_tx) {
4096                 /* Only support an equally distributed
4097                  *  Tx packet buffer strategy.
4098                  */
4099                 uint32_t txpktsize = IXGBE_TXPBSIZE_MAX / nb_tcs;
4100                 uint32_t txpbthresh = (txpktsize / DCB_TX_PB) - IXGBE_TXPKT_SIZE_MAX;
4101
4102                 for (i = 0; i < nb_tcs; i++) {
4103                         IXGBE_WRITE_REG(hw, IXGBE_TXPBSIZE(i), txpktsize);
4104                         IXGBE_WRITE_REG(hw, IXGBE_TXPBTHRESH(i), txpbthresh);
4105                 }
4106                 /* Clear unused TCs, if any, to zero buffer size*/
4107                 for (; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
4108                         IXGBE_WRITE_REG(hw, IXGBE_TXPBSIZE(i), 0);
4109                         IXGBE_WRITE_REG(hw, IXGBE_TXPBTHRESH(i), 0);
4110                 }
4111         }
4112
4113         /*Calculates traffic class credits*/
4114         ixgbe_dcb_calculate_tc_credits_cee(hw, dcb_config, max_frame,
4115                                 IXGBE_DCB_TX_CONFIG);
4116         ixgbe_dcb_calculate_tc_credits_cee(hw, dcb_config, max_frame,
4117                                 IXGBE_DCB_RX_CONFIG);
4118
4119         if (config_dcb_rx) {
4120                 /* Unpack CEE standard containers */
4121                 ixgbe_dcb_unpack_refill_cee(dcb_config, IXGBE_DCB_RX_CONFIG, refill);
4122                 ixgbe_dcb_unpack_max_cee(dcb_config, max);
4123                 ixgbe_dcb_unpack_bwgid_cee(dcb_config, IXGBE_DCB_RX_CONFIG, bwgid);
4124                 ixgbe_dcb_unpack_tsa_cee(dcb_config, IXGBE_DCB_RX_CONFIG, tsa);
4125                 /* Configure PG(ETS) RX */
4126                 ixgbe_dcb_hw_arbite_rx_config(hw, refill, max, bwgid, tsa, map);
4127         }
4128
4129         if (config_dcb_tx) {
4130                 /* Unpack CEE standard containers */
4131                 ixgbe_dcb_unpack_refill_cee(dcb_config, IXGBE_DCB_TX_CONFIG, refill);
4132                 ixgbe_dcb_unpack_max_cee(dcb_config, max);
4133                 ixgbe_dcb_unpack_bwgid_cee(dcb_config, IXGBE_DCB_TX_CONFIG, bwgid);
4134                 ixgbe_dcb_unpack_tsa_cee(dcb_config, IXGBE_DCB_TX_CONFIG, tsa);
4135                 /* Configure PG(ETS) TX */
4136                 ixgbe_dcb_hw_arbite_tx_config(hw, refill, max, bwgid, tsa, map);
4137         }
4138
4139         /*Configure queue statistics registers*/
4140         ixgbe_dcb_config_tc_stats_82599(hw, dcb_config);
4141
4142         /* Check if the PFC is supported */
4143         if (dev->data->dev_conf.dcb_capability_en & ETH_DCB_PFC_SUPPORT) {
4144                 pbsize = (uint16_t)(rx_buffer_size / nb_tcs);
4145                 for (i = 0; i < nb_tcs; i++) {
4146                         /*
4147                         * If the TC count is 8,and the default high_water is 48,
4148                         * the low_water is 16 as default.
4149                         */
4150                         hw->fc.high_water[i] = (pbsize * 3) / 4;
4151                         hw->fc.low_water[i] = pbsize / 4;
4152                         /* Enable pfc for this TC */
4153                         tc = &dcb_config->tc_config[i];
4154                         tc->pfc = ixgbe_dcb_pfc_enabled;
4155                 }
4156                 ixgbe_dcb_unpack_pfc_cee(dcb_config, map, &pfc_en);
4157                 if (dcb_config->num_tcs.pfc_tcs == ETH_4_TCS)
4158                         pfc_en &= 0x0F;
4159                 ret = ixgbe_dcb_config_pfc(hw, pfc_en, map);
4160         }
4161
4162         return ret;
4163 }
4164
4165 /**
4166  * ixgbe_configure_dcb - Configure DCB  Hardware
4167  * @dev: pointer to rte_eth_dev
4168  */
4169 void ixgbe_configure_dcb(struct rte_eth_dev *dev)
4170 {
4171         struct ixgbe_dcb_config *dcb_cfg =
4172                         IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private);
4173         struct rte_eth_conf *dev_conf = &(dev->data->dev_conf);
4174
4175         PMD_INIT_FUNC_TRACE();
4176
4177         /* check support mq_mode for DCB */
4178         if ((dev_conf->rxmode.mq_mode != ETH_MQ_RX_VMDQ_DCB) &&
4179             (dev_conf->rxmode.mq_mode != ETH_MQ_RX_DCB) &&
4180             (dev_conf->rxmode.mq_mode != ETH_MQ_RX_DCB_RSS))
4181                 return;
4182
4183         if (dev->data->nb_rx_queues > ETH_DCB_NUM_QUEUES)
4184                 return;
4185
4186         /** Configure DCB hardware **/
4187         ixgbe_dcb_hw_configure(dev, dcb_cfg);
4188 }
4189
4190 /*
4191  * VMDq only support for 10 GbE NIC.
4192  */
4193 static void
4194 ixgbe_vmdq_rx_hw_configure(struct rte_eth_dev *dev)
4195 {
4196         struct rte_eth_vmdq_rx_conf *cfg;
4197         struct ixgbe_hw *hw;
4198         enum rte_eth_nb_pools num_pools;
4199         uint32_t mrqc, vt_ctl, vlanctrl;
4200         uint32_t vmolr = 0;
4201         int i;
4202
4203         PMD_INIT_FUNC_TRACE();
4204         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4205         cfg = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
4206         num_pools = cfg->nb_queue_pools;
4207
4208         ixgbe_rss_disable(dev);
4209
4210         /* MRQC: enable vmdq */
4211         mrqc = IXGBE_MRQC_VMDQEN;
4212         IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4213
4214         /* PFVTCTL: turn on virtualisation and set the default pool */
4215         vt_ctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
4216         if (cfg->enable_default_pool)
4217                 vt_ctl |= (cfg->default_pool << IXGBE_VT_CTL_POOL_SHIFT);
4218         else
4219                 vt_ctl |= IXGBE_VT_CTL_DIS_DEFPL;
4220
4221         IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vt_ctl);
4222
4223         for (i = 0; i < (int)num_pools; i++) {
4224                 vmolr = ixgbe_convert_vm_rx_mask_to_val(cfg->rx_mode, vmolr);
4225                 IXGBE_WRITE_REG(hw, IXGBE_VMOLR(i), vmolr);
4226         }
4227
4228         /* VLNCTRL: enable vlan filtering and allow all vlan tags through */
4229         vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4230         vlanctrl |= IXGBE_VLNCTRL_VFE; /* enable vlan filters */
4231         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl);
4232
4233         /* VFTA - enable all vlan filters */
4234         for (i = 0; i < NUM_VFTA_REGISTERS; i++)
4235                 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), UINT32_MAX);
4236
4237         /* VFRE: pool enabling for receive - 64 */
4238         IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), UINT32_MAX);
4239         if (num_pools == ETH_64_POOLS)
4240                 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), UINT32_MAX);
4241
4242         /*
4243          * MPSAR - allow pools to read specific mac addresses
4244          * In this case, all pools should be able to read from mac addr 0
4245          */
4246         IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(0), UINT32_MAX);
4247         IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(0), UINT32_MAX);
4248
4249         /* PFVLVF, PFVLVFB: set up filters for vlan tags as configured */
4250         for (i = 0; i < cfg->nb_pool_maps; i++) {
4251                 /* set vlan id in VF register and set the valid bit */
4252                 IXGBE_WRITE_REG(hw, IXGBE_VLVF(i), (IXGBE_VLVF_VIEN |
4253                                 (cfg->pool_map[i].vlan_id & IXGBE_RXD_VLAN_ID_MASK)));
4254                 /*
4255                  * Put the allowed pools in VFB reg. As we only have 16 or 64
4256                  * pools, we only need to use the first half of the register
4257                  * i.e. bits 0-31
4258                  */
4259                 if (((cfg->pool_map[i].pools >> 32) & UINT32_MAX) == 0)
4260                         IXGBE_WRITE_REG(hw, IXGBE_VLVFB(i * 2),
4261                                         (cfg->pool_map[i].pools & UINT32_MAX));
4262                 else
4263                         IXGBE_WRITE_REG(hw, IXGBE_VLVFB((i * 2 + 1)),
4264                                         ((cfg->pool_map[i].pools >> 32) & UINT32_MAX));
4265
4266         }
4267
4268         /* PFDMA Tx General Switch Control Enables VMDQ loopback */
4269         if (cfg->enable_loop_back) {
4270                 IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN);
4271                 for (i = 0; i < RTE_IXGBE_VMTXSW_REGISTER_COUNT; i++)
4272                         IXGBE_WRITE_REG(hw, IXGBE_VMTXSW(i), UINT32_MAX);
4273         }
4274
4275         IXGBE_WRITE_FLUSH(hw);
4276 }
4277
4278 /*
4279  * ixgbe_dcb_config_tx_hw_config - Configure general VMDq TX parameters
4280  * @hw: pointer to hardware structure
4281  */
4282 static void
4283 ixgbe_vmdq_tx_hw_configure(struct ixgbe_hw *hw)
4284 {
4285         uint32_t reg;
4286         uint32_t q;
4287
4288         PMD_INIT_FUNC_TRACE();
4289         /*PF VF Transmit Enable*/
4290         IXGBE_WRITE_REG(hw, IXGBE_VFTE(0), UINT32_MAX);
4291         IXGBE_WRITE_REG(hw, IXGBE_VFTE(1), UINT32_MAX);
4292
4293         /* Disable the Tx desc arbiter so that MTQC can be changed */
4294         reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
4295         reg |= IXGBE_RTTDCS_ARBDIS;
4296         IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg);
4297
4298         reg = IXGBE_MTQC_VT_ENA | IXGBE_MTQC_64VF;
4299         IXGBE_WRITE_REG(hw, IXGBE_MTQC, reg);
4300
4301         /* Disable drop for all queues */
4302         for (q = 0; q < IXGBE_MAX_RX_QUEUE_NUM; q++)
4303                 IXGBE_WRITE_REG(hw, IXGBE_QDE,
4304                   (IXGBE_QDE_WRITE | (q << IXGBE_QDE_IDX_SHIFT)));
4305
4306         /* Enable the Tx desc arbiter */
4307         reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
4308         reg &= ~IXGBE_RTTDCS_ARBDIS;
4309         IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg);
4310
4311         IXGBE_WRITE_FLUSH(hw);
4312 }
4313
4314 static int __attribute__((cold))
4315 ixgbe_alloc_rx_queue_mbufs(struct ixgbe_rx_queue *rxq)
4316 {
4317         struct ixgbe_rx_entry *rxe = rxq->sw_ring;
4318         uint64_t dma_addr;
4319         unsigned int i;
4320
4321         /* Initialize software ring entries */
4322         for (i = 0; i < rxq->nb_rx_desc; i++) {
4323                 volatile union ixgbe_adv_rx_desc *rxd;
4324                 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mb_pool);
4325
4326                 if (mbuf == NULL) {
4327                         PMD_INIT_LOG(ERR, "RX mbuf alloc failed queue_id=%u",
4328                                      (unsigned) rxq->queue_id);
4329                         return -ENOMEM;
4330                 }
4331
4332                 mbuf->data_off = RTE_PKTMBUF_HEADROOM;
4333                 mbuf->port = rxq->port_id;
4334
4335                 dma_addr =
4336                         rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf));
4337                 rxd = &rxq->rx_ring[i];
4338                 rxd->read.hdr_addr = 0;
4339                 rxd->read.pkt_addr = dma_addr;
4340                 rxe[i].mbuf = mbuf;
4341         }
4342
4343         return 0;
4344 }
4345
4346 static int
4347 ixgbe_config_vf_rss(struct rte_eth_dev *dev)
4348 {
4349         struct ixgbe_hw *hw;
4350         uint32_t mrqc;
4351
4352         ixgbe_rss_configure(dev);
4353
4354         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4355
4356         /* MRQC: enable VF RSS */
4357         mrqc = IXGBE_READ_REG(hw, IXGBE_MRQC);
4358         mrqc &= ~IXGBE_MRQC_MRQE_MASK;
4359         switch (RTE_ETH_DEV_SRIOV(dev).active) {
4360         case ETH_64_POOLS:
4361                 mrqc |= IXGBE_MRQC_VMDQRSS64EN;
4362                 break;
4363
4364         case ETH_32_POOLS:
4365                 mrqc |= IXGBE_MRQC_VMDQRSS32EN;
4366                 break;
4367
4368         default:
4369                 PMD_INIT_LOG(ERR, "Invalid pool number in IOV mode with VMDQ RSS");
4370                 return -EINVAL;
4371         }
4372
4373         IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4374
4375         return 0;
4376 }
4377
4378 static int
4379 ixgbe_config_vf_default(struct rte_eth_dev *dev)
4380 {
4381         struct ixgbe_hw *hw =
4382                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4383
4384         switch (RTE_ETH_DEV_SRIOV(dev).active) {
4385         case ETH_64_POOLS:
4386                 IXGBE_WRITE_REG(hw, IXGBE_MRQC,
4387                         IXGBE_MRQC_VMDQEN);
4388                 break;
4389
4390         case ETH_32_POOLS:
4391                 IXGBE_WRITE_REG(hw, IXGBE_MRQC,
4392                         IXGBE_MRQC_VMDQRT4TCEN);
4393                 break;
4394
4395         case ETH_16_POOLS:
4396                 IXGBE_WRITE_REG(hw, IXGBE_MRQC,
4397                         IXGBE_MRQC_VMDQRT8TCEN);
4398                 break;
4399         default:
4400                 PMD_INIT_LOG(ERR,
4401                         "invalid pool number in IOV mode");
4402                 break;
4403         }
4404         return 0;
4405 }
4406
4407 static int
4408 ixgbe_dev_mq_rx_configure(struct rte_eth_dev *dev)
4409 {
4410         struct ixgbe_hw *hw =
4411                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4412
4413         if (hw->mac.type == ixgbe_mac_82598EB)
4414                 return 0;
4415
4416         if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
4417                 /*
4418                  * SRIOV inactive scheme
4419                  * any DCB/RSS w/o VMDq multi-queue setting
4420                  */
4421                 switch (dev->data->dev_conf.rxmode.mq_mode) {
4422                 case ETH_MQ_RX_RSS:
4423                 case ETH_MQ_RX_DCB_RSS:
4424                 case ETH_MQ_RX_VMDQ_RSS:
4425                         ixgbe_rss_configure(dev);
4426                         break;
4427
4428                 case ETH_MQ_RX_VMDQ_DCB:
4429                         ixgbe_vmdq_dcb_configure(dev);
4430                         break;
4431
4432                 case ETH_MQ_RX_VMDQ_ONLY:
4433                         ixgbe_vmdq_rx_hw_configure(dev);
4434                         break;
4435
4436                 case ETH_MQ_RX_NONE:
4437                 default:
4438                         /* if mq_mode is none, disable rss mode.*/
4439                         ixgbe_rss_disable(dev);
4440                         break;
4441                 }
4442         } else {
4443                 /* SRIOV active scheme
4444                  * Support RSS together with SRIOV.
4445                  */
4446                 switch (dev->data->dev_conf.rxmode.mq_mode) {
4447                 case ETH_MQ_RX_RSS:
4448                 case ETH_MQ_RX_VMDQ_RSS:
4449                         ixgbe_config_vf_rss(dev);
4450                         break;
4451                 case ETH_MQ_RX_VMDQ_DCB:
4452                 case ETH_MQ_RX_DCB:
4453                 /* In SRIOV, the configuration is the same as VMDq case */
4454                         ixgbe_vmdq_dcb_configure(dev);
4455                         break;
4456                 /* DCB/RSS together with SRIOV is not supported */
4457                 case ETH_MQ_RX_VMDQ_DCB_RSS:
4458                 case ETH_MQ_RX_DCB_RSS:
4459                         PMD_INIT_LOG(ERR,
4460                                 "Could not support DCB/RSS with VMDq & SRIOV");
4461                         return -1;
4462                 default:
4463                         ixgbe_config_vf_default(dev);
4464                         break;
4465                 }
4466         }
4467
4468         return 0;
4469 }
4470
4471 static int
4472 ixgbe_dev_mq_tx_configure(struct rte_eth_dev *dev)
4473 {
4474         struct ixgbe_hw *hw =
4475                 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4476         uint32_t mtqc;
4477         uint32_t rttdcs;
4478
4479         if (hw->mac.type == ixgbe_mac_82598EB)
4480                 return 0;
4481
4482         /* disable arbiter before setting MTQC */
4483         rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
4484         rttdcs |= IXGBE_RTTDCS_ARBDIS;
4485         IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
4486
4487         if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
4488                 /*
4489                  * SRIOV inactive scheme
4490                  * any DCB w/o VMDq multi-queue setting
4491                  */
4492                 if (dev->data->dev_conf.txmode.mq_mode == ETH_MQ_TX_VMDQ_ONLY)
4493                         ixgbe_vmdq_tx_hw_configure(hw);
4494                 else {
4495                         mtqc = IXGBE_MTQC_64Q_1PB;
4496                         IXGBE_WRITE_REG(hw, IXGBE_MTQC, mtqc);
4497                 }
4498         } else {
4499                 switch (RTE_ETH_DEV_SRIOV(dev).active) {
4500
4501                 /*
4502                  * SRIOV active scheme
4503                  * FIXME if support DCB together with VMDq & SRIOV
4504                  */
4505                 case ETH_64_POOLS:
4506                         mtqc = IXGBE_MTQC_VT_ENA | IXGBE_MTQC_64VF;
4507                         break;
4508                 case ETH_32_POOLS:
4509                         mtqc = IXGBE_MTQC_VT_ENA | IXGBE_MTQC_32VF;
4510                         break;
4511                 case ETH_16_POOLS:
4512                         mtqc = IXGBE_MTQC_VT_ENA | IXGBE_MTQC_RT_ENA |
4513                                 IXGBE_MTQC_8TC_8TQ;
4514                         break;
4515                 default:
4516                         mtqc = IXGBE_MTQC_64Q_1PB;
4517                         PMD_INIT_LOG(ERR, "invalid pool number in IOV mode");
4518                 }
4519                 IXGBE_WRITE_REG(hw, IXGBE_MTQC, mtqc);
4520         }
4521
4522         /* re-enable arbiter */
4523         rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
4524         IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
4525
4526         return 0;
4527 }
4528
4529 /**
4530  * ixgbe_get_rscctl_maxdesc - Calculate the RSCCTL[n].MAXDESC for PF
4531  *
4532  * Return the RSCCTL[n].MAXDESC for 82599 and x540 PF devices according to the
4533  * spec rev. 3.0 chapter 8.2.3.8.13.
4534  *
4535  * @pool Memory pool of the Rx queue
4536  */
4537 static inline uint32_t
4538 ixgbe_get_rscctl_maxdesc(struct rte_mempool *pool)
4539 {
4540         struct rte_pktmbuf_pool_private *mp_priv = rte_mempool_get_priv(pool);
4541
4542         /* MAXDESC * SRRCTL.BSIZEPKT must not exceed 64 KB minus one */
4543         uint16_t maxdesc =
4544                 IPV4_MAX_PKT_LEN /
4545                         (mp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM);
4546
4547         if (maxdesc >= 16)
4548                 return IXGBE_RSCCTL_MAXDESC_16;
4549         else if (maxdesc >= 8)
4550                 return IXGBE_RSCCTL_MAXDESC_8;
4551         else if (maxdesc >= 4)
4552                 return IXGBE_RSCCTL_MAXDESC_4;
4553         else
4554                 return IXGBE_RSCCTL_MAXDESC_1;
4555 }
4556
4557 /**
4558  * ixgbe_set_ivar - Setup the correct IVAR register for a particular MSIX
4559  * interrupt
4560  *
4561  * (Taken from FreeBSD tree)
4562  * (yes this is all very magic and confusing :)
4563  *
4564  * @dev port handle
4565  * @entry the register array entry
4566  * @vector the MSIX vector for this queue
4567  * @type RX/TX/MISC
4568  */
4569 static void
4570 ixgbe_set_ivar(struct rte_eth_dev *dev, u8 entry, u8 vector, s8 type)
4571 {
4572         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4573         u32 ivar, index;
4574
4575         vector |= IXGBE_IVAR_ALLOC_VAL;
4576
4577         switch (hw->mac.type) {
4578
4579         case ixgbe_mac_82598EB:
4580                 if (type == -1)
4581                         entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
4582                 else
4583                         entry += (type * 64);
4584                 index = (entry >> 2) & 0x1F;
4585                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4586                 ivar &= ~(0xFF << (8 * (entry & 0x3)));
4587                 ivar |= (vector << (8 * (entry & 0x3)));
4588                 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
4589                 break;
4590
4591         case ixgbe_mac_82599EB:
4592         case ixgbe_mac_X540:
4593                 if (type == -1) { /* MISC IVAR */
4594                         index = (entry & 1) * 8;
4595                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4596                         ivar &= ~(0xFF << index);
4597                         ivar |= (vector << index);
4598                         IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4599                 } else {        /* RX/TX IVARS */
4600                         index = (16 * (entry & 1)) + (8 * type);
4601                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
4602                         ivar &= ~(0xFF << index);
4603                         ivar |= (vector << index);
4604                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
4605                 }
4606
4607                 break;
4608
4609         default:
4610                 break;
4611         }
4612 }
4613
4614 void __attribute__((cold))
4615 ixgbe_set_rx_function(struct rte_eth_dev *dev)
4616 {
4617         uint16_t i, rx_using_sse;
4618         struct ixgbe_adapter *adapter =
4619                 (struct ixgbe_adapter *)dev->data->dev_private;
4620
4621         /*
4622          * In order to allow Vector Rx there are a few configuration
4623          * conditions to be met and Rx Bulk Allocation should be allowed.
4624          */
4625         if (ixgbe_rx_vec_dev_conf_condition_check(dev) ||
4626             !adapter->rx_bulk_alloc_allowed) {
4627                 PMD_INIT_LOG(DEBUG, "Port[%d] doesn't meet Vector Rx "
4628                                     "preconditions or RTE_IXGBE_INC_VECTOR is "
4629                                     "not enabled",
4630                              dev->data->port_id);
4631
4632                 adapter->rx_vec_allowed = false;
4633         }
4634
4635         /*
4636          * Initialize the appropriate LRO callback.
4637          *
4638          * If all queues satisfy the bulk allocation preconditions
4639          * (hw->rx_bulk_alloc_allowed is TRUE) then we may use bulk allocation.
4640          * Otherwise use a single allocation version.
4641          */
4642         if (dev->data->lro) {
4643                 if (adapter->rx_bulk_alloc_allowed) {
4644                         PMD_INIT_LOG(DEBUG, "LRO is requested. Using a bulk "
4645                                            "allocation version");
4646                         dev->rx_pkt_burst = ixgbe_recv_pkts_lro_bulk_alloc;
4647                 } else {
4648                         PMD_INIT_LOG(DEBUG, "LRO is requested. Using a single "
4649                                            "allocation version");
4650                         dev->rx_pkt_burst = ixgbe_recv_pkts_lro_single_alloc;
4651                 }
4652         } else if (dev->data->scattered_rx) {
4653                 /*
4654                  * Set the non-LRO scattered callback: there are Vector and
4655                  * single allocation versions.
4656                  */
4657                 if (adapter->rx_vec_allowed) {
4658                         PMD_INIT_LOG(DEBUG, "Using Vector Scattered Rx "
4659                                             "callback (port=%d).",
4660                                      dev->data->port_id);
4661
4662                         dev->rx_pkt_burst = ixgbe_recv_scattered_pkts_vec;
4663                 } else if (adapter->rx_bulk_alloc_allowed) {
4664                         PMD_INIT_LOG(DEBUG, "Using a Scattered with bulk "
4665                                            "allocation callback (port=%d).",
4666                                      dev->data->port_id);
4667                         dev->rx_pkt_burst = ixgbe_recv_pkts_lro_bulk_alloc;
4668                 } else {
4669                         PMD_INIT_LOG(DEBUG, "Using Regualr (non-vector, "
4670                                             "single allocation) "
4671                                             "Scattered Rx callback "
4672                                             "(port=%d).",
4673                                      dev->data->port_id);
4674
4675                         dev->rx_pkt_burst = ixgbe_recv_pkts_lro_single_alloc;
4676                 }
4677         /*
4678          * Below we set "simple" callbacks according to port/queues parameters.
4679          * If parameters allow we are going to choose between the following
4680          * callbacks:
4681          *    - Vector
4682          *    - Bulk Allocation
4683          *    - Single buffer allocation (the simplest one)
4684          */
4685         } else if (adapter->rx_vec_allowed) {
4686                 PMD_INIT_LOG(DEBUG, "Vector rx enabled, please make sure RX "
4687                                     "burst size no less than %d (port=%d).",
4688                              RTE_IXGBE_DESCS_PER_LOOP,
4689                              dev->data->port_id);
4690
4691                 dev->rx_pkt_burst = ixgbe_recv_pkts_vec;
4692         } else if (adapter->rx_bulk_alloc_allowed) {
4693                 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are "
4694                                     "satisfied. Rx Burst Bulk Alloc function "
4695                                     "will be used on port=%d.",
4696                              dev->data->port_id);
4697
4698                 dev->rx_pkt_burst = ixgbe_recv_pkts_bulk_alloc;
4699         } else {
4700                 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are not "
4701                                     "satisfied, or Scattered Rx is requested "
4702                                     "(port=%d).",
4703                              dev->data->port_id);
4704
4705                 dev->rx_pkt_burst = ixgbe_recv_pkts;
4706         }
4707
4708         /* Propagate information about RX function choice through all queues. */
4709
4710         rx_using_sse =
4711                 (dev->rx_pkt_burst == ixgbe_recv_scattered_pkts_vec ||
4712                 dev->rx_pkt_burst == ixgbe_recv_pkts_vec);
4713
4714         for (i = 0; i < dev->data->nb_rx_queues; i++) {
4715                 struct ixgbe_rx_queue *rxq = dev->data->rx_queues[i];
4716
4717                 rxq->rx_using_sse = rx_using_sse;
4718 #ifdef RTE_LIBRTE_SECURITY
4719                 rxq->using_ipsec = !!(dev->data->dev_conf.rxmode.offloads &
4720                                 DEV_RX_OFFLOAD_SECURITY);
4721 #endif
4722         }
4723 }
4724
4725 /**
4726  * ixgbe_set_rsc - configure RSC related port HW registers
4727  *
4728  * Configures the port's RSC related registers according to the 4.6.7.2 chapter
4729  * of 82599 Spec (x540 configuration is virtually the same).
4730  *
4731  * @dev port handle
4732  *
4733  * Returns 0 in case of success or a non-zero error code
4734  */
4735 static int
4736 ixgbe_set_rsc(struct rte_eth_dev *dev)
4737 {
4738         struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode;
4739         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4740         struct rte_eth_dev_info dev_info = { 0 };
4741         bool rsc_capable = false;
4742         uint16_t i;
4743         uint32_t rdrxctl;
4744         uint32_t rfctl;
4745
4746         /* Sanity check */
4747         dev->dev_ops->dev_infos_get(dev, &dev_info);
4748         if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TCP_LRO)
4749                 rsc_capable = true;
4750
4751         if (!rsc_capable && (rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO)) {
4752                 PMD_INIT_LOG(CRIT, "LRO is requested on HW that doesn't "
4753                                    "support it");
4754                 return -EINVAL;
4755         }
4756
4757         /* RSC global configuration (chapter 4.6.7.2.1 of 82599 Spec) */
4758
4759         if (!(rx_conf->offloads & DEV_RX_OFFLOAD_CRC_STRIP) &&
4760              (rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO)) {
4761                 /*
4762                  * According to chapter of 4.6.7.2.1 of the Spec Rev.
4763                  * 3.0 RSC configuration requires HW CRC stripping being
4764                  * enabled. If user requested both HW CRC stripping off
4765                  * and RSC on - return an error.
4766                  */
4767                 PMD_INIT_LOG(CRIT, "LRO can't be enabled when HW CRC "
4768                                     "is disabled");
4769                 return -EINVAL;
4770         }
4771
4772         /* RFCTL configuration  */
4773         rfctl = IXGBE_READ_REG(hw, IXGBE_RFCTL);
4774         if ((rsc_capable) && (rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO))
4775                 /*
4776                  * Since NFS packets coalescing is not supported - clear
4777                  * RFCTL.NFSW_DIS and RFCTL.NFSR_DIS when RSC is
4778                  * enabled.
4779                  */
4780                 rfctl &= ~(IXGBE_RFCTL_RSC_DIS | IXGBE_RFCTL_NFSW_DIS |
4781                            IXGBE_RFCTL_NFSR_DIS);
4782         else
4783                 rfctl |= IXGBE_RFCTL_RSC_DIS;
4784         IXGBE_WRITE_REG(hw, IXGBE_RFCTL, rfctl);
4785
4786         /* If LRO hasn't been requested - we are done here. */
4787         if (!(rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO))
4788                 return 0;
4789
4790         /* Set RDRXCTL.RSCACKC bit */
4791         rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
4792         rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
4793         IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
4794
4795         /* Per-queue RSC configuration (chapter 4.6.7.2.2 of 82599 Spec) */
4796         for (i = 0; i < dev->data->nb_rx_queues; i++) {
4797                 struct ixgbe_rx_queue *rxq = dev->data->rx_queues[i];
4798                 uint32_t srrctl =
4799                         IXGBE_READ_REG(hw, IXGBE_SRRCTL(rxq->reg_idx));
4800                 uint32_t rscctl =
4801                         IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxq->reg_idx));
4802                 uint32_t psrtype =
4803                         IXGBE_READ_REG(hw, IXGBE_PSRTYPE(rxq->reg_idx));
4804                 uint32_t eitr =
4805                         IXGBE_READ_REG(hw, IXGBE_EITR(rxq->reg_idx));
4806
4807                 /*
4808                  * ixgbe PMD doesn't support header-split at the moment.
4809                  *
4810                  * Following the 4.6.7.2.1 chapter of the 82599/x540
4811                  * Spec if RSC is enabled the SRRCTL[n].BSIZEHEADER
4812                  * should be configured even if header split is not
4813                  * enabled. We will configure it 128 bytes following the
4814                  * recommendation in the spec.
4815                  */
4816                 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
4817                 srrctl |= (128 << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) &
4818                                             IXGBE_SRRCTL_BSIZEHDR_MASK;
4819
4820                 /*
4821                  * TODO: Consider setting the Receive Descriptor Minimum
4822                  * Threshold Size for an RSC case. This is not an obviously
4823                  * beneficiary option but the one worth considering...
4824                  */
4825
4826                 rscctl |= IXGBE_RSCCTL_RSCEN;
4827                 rscctl |= ixgbe_get_rscctl_maxdesc(rxq->mb_pool);
4828                 psrtype |= IXGBE_PSRTYPE_TCPHDR;
4829
4830                 /*
4831                  * RSC: Set ITR interval corresponding to 2K ints/s.
4832                  *
4833                  * Full-sized RSC aggregations for a 10Gb/s link will
4834                  * arrive at about 20K aggregation/s rate.
4835                  *
4836                  * 2K inst/s rate will make only 10% of the
4837                  * aggregations to be closed due to the interrupt timer
4838                  * expiration for a streaming at wire-speed case.
4839                  *
4840                  * For a sparse streaming case this setting will yield
4841                  * at most 500us latency for a single RSC aggregation.
4842                  */
4843                 eitr &= ~IXGBE_EITR_ITR_INT_MASK;
4844                 eitr |= IXGBE_EITR_INTERVAL_US(500) | IXGBE_EITR_CNT_WDIS;
4845
4846                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxq->reg_idx), srrctl);
4847                 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxq->reg_idx), rscctl);
4848                 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(rxq->reg_idx), psrtype);
4849                 IXGBE_WRITE_REG(hw, IXGBE_EITR(rxq->reg_idx), eitr);
4850
4851                 /*
4852                  * RSC requires the mapping of the queue to the
4853                  * interrupt vector.
4854                  */
4855                 ixgbe_set_ivar(dev, rxq->reg_idx, i, 0);
4856         }
4857
4858         dev->data->lro = 1;
4859
4860         PMD_INIT_LOG(DEBUG, "enabling LRO mode");
4861
4862         return 0;
4863 }
4864
4865 /*
4866  * Initializes Receive Unit.
4867  */
4868 int __attribute__((cold))
4869 ixgbe_dev_rx_init(struct rte_eth_dev *dev)
4870 {
4871         struct ixgbe_hw     *hw;
4872         struct ixgbe_rx_queue *rxq;
4873         uint64_t bus_addr;
4874         uint32_t rxctrl;
4875         uint32_t fctrl;
4876         uint32_t hlreg0;
4877         uint32_t maxfrs;
4878         uint32_t srrctl;
4879         uint32_t rdrxctl;
4880         uint32_t rxcsum;
4881         uint16_t buf_size;
4882         uint16_t i;
4883         struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode;
4884         int rc;
4885
4886         PMD_INIT_FUNC_TRACE();
4887         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4888
4889         /*
4890          * Make sure receives are disabled while setting
4891          * up the RX context (registers, descriptor rings, etc.).
4892          */
4893         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
4894         IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl & ~IXGBE_RXCTRL_RXEN);
4895
4896         /* Enable receipt of broadcasted frames */
4897         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
4898         fctrl |= IXGBE_FCTRL_BAM;
4899         fctrl |= IXGBE_FCTRL_DPF;
4900         fctrl |= IXGBE_FCTRL_PMCF;
4901         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
4902
4903         /*
4904          * Configure CRC stripping, if any.
4905          */
4906         hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4907         if (rx_conf->offloads & DEV_RX_OFFLOAD_CRC_STRIP)
4908                 hlreg0 |= IXGBE_HLREG0_RXCRCSTRP;
4909         else
4910                 hlreg0 &= ~IXGBE_HLREG0_RXCRCSTRP;
4911
4912         /*
4913          * Configure jumbo frame support, if any.
4914          */
4915         if (rx_conf->offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) {
4916                 hlreg0 |= IXGBE_HLREG0_JUMBOEN;
4917                 maxfrs = IXGBE_READ_REG(hw, IXGBE_MAXFRS);
4918                 maxfrs &= 0x0000FFFF;
4919                 maxfrs |= (rx_conf->max_rx_pkt_len << 16);
4920                 IXGBE_WRITE_REG(hw, IXGBE_MAXFRS, maxfrs);
4921         } else
4922                 hlreg0 &= ~IXGBE_HLREG0_JUMBOEN;
4923
4924         /*
4925          * If loopback mode is configured for 82599, set LPBK bit.
4926          */
4927         if (hw->mac.type == ixgbe_mac_82599EB &&
4928                         dev->data->dev_conf.lpbk_mode == IXGBE_LPBK_82599_TX_RX)
4929                 hlreg0 |= IXGBE_HLREG0_LPBK;
4930         else
4931                 hlreg0 &= ~IXGBE_HLREG0_LPBK;
4932
4933         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);
4934
4935         /*
4936          * Assume no header split and no VLAN strip support
4937          * on any Rx queue first .
4938          */
4939         rx_conf->offloads &= ~DEV_RX_OFFLOAD_VLAN_STRIP;
4940         /* Setup RX queues */
4941         for (i = 0; i < dev->data->nb_rx_queues; i++) {
4942                 rxq = dev->data->rx_queues[i];
4943
4944                 /*
4945                  * Reset crc_len in case it was changed after queue setup by a
4946                  * call to configure.
4947                  */
4948                 rxq->crc_len = (rx_conf->offloads & DEV_RX_OFFLOAD_CRC_STRIP) ?
4949                                 0 : ETHER_CRC_LEN;
4950
4951                 /* Setup the Base and Length of the Rx Descriptor Rings */
4952                 bus_addr = rxq->rx_ring_phys_addr;
4953                 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(rxq->reg_idx),
4954                                 (uint32_t)(bus_addr & 0x00000000ffffffffULL));
4955                 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(rxq->reg_idx),
4956                                 (uint32_t)(bus_addr >> 32));
4957                 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(rxq->reg_idx),
4958                                 rxq->nb_rx_desc * sizeof(union ixgbe_adv_rx_desc));
4959                 IXGBE_WRITE_REG(hw, IXGBE_RDH(rxq->reg_idx), 0);
4960                 IXGBE_WRITE_REG(hw, IXGBE_RDT(rxq->reg_idx), 0);
4961
4962                 /* Configure the SRRCTL register */
4963                 srrctl = IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
4964
4965                 /* Set if packets are dropped when no descriptors available */
4966                 if (rxq->drop_en)
4967                         srrctl |= IXGBE_SRRCTL_DROP_EN;
4968
4969                 /*
4970                  * Configure the RX buffer size in the BSIZEPACKET field of
4971                  * the SRRCTL register of the queue.
4972                  * The value is in 1 KB resolution. Valid values can be from
4973                  * 1 KB to 16 KB.
4974                  */
4975                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) -
4976                         RTE_PKTMBUF_HEADROOM);
4977                 srrctl |= ((buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) &
4978                            IXGBE_SRRCTL_BSIZEPKT_MASK);
4979
4980                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxq->reg_idx), srrctl);
4981
4982                 buf_size = (uint16_t) ((srrctl & IXGBE_SRRCTL_BSIZEPKT_MASK) <<
4983                                        IXGBE_SRRCTL_BSIZEPKT_SHIFT);
4984
4985                 /* It adds dual VLAN length for supporting dual VLAN */
4986                 if (dev->data->dev_conf.rxmode.max_rx_pkt_len +
4987                                             2 * IXGBE_VLAN_TAG_SIZE > buf_size)
4988                         dev->data->scattered_rx = 1;
4989                 if (rxq->offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
4990                         rx_conf->offloads |= DEV_RX_OFFLOAD_VLAN_STRIP;
4991         }
4992
4993         if (rx_conf->offloads & DEV_RX_OFFLOAD_SCATTER)
4994                 dev->data->scattered_rx = 1;
4995
4996         /*
4997          * Device configured with multiple RX queues.
4998          */
4999         ixgbe_dev_mq_rx_configure(dev);
5000
5001         /*
5002          * Setup the Checksum Register.
5003          * Disable Full-Packet Checksum which is mutually exclusive with RSS.
5004          * Enable IP/L4 checkum computation by hardware if requested to do so.
5005          */
5006         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
5007         rxcsum |= IXGBE_RXCSUM_PCSD;
5008         if (rx_conf->offloads & DEV_RX_OFFLOAD_CHECKSUM)
5009                 rxcsum |= IXGBE_RXCSUM_IPPCSE;
5010         else
5011                 rxcsum &= ~IXGBE_RXCSUM_IPPCSE;
5012
5013         IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
5014
5015         if (hw->mac.type == ixgbe_mac_82599EB ||
5016             hw->mac.type == ixgbe_mac_X540) {
5017                 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
5018                 if (rx_conf->offloads & DEV_RX_OFFLOAD_CRC_STRIP)
5019                         rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
5020                 else
5021                         rdrxctl &= ~IXGBE_RDRXCTL_CRCSTRIP;
5022                 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
5023                 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
5024         }
5025
5026         rc = ixgbe_set_rsc(dev);
5027         if (rc)
5028                 return rc;
5029
5030         ixgbe_set_rx_function(dev);
5031
5032         return 0;
5033 }
5034
5035 /*
5036  * Initializes Transmit Unit.
5037  */
5038 void __attribute__((cold))
5039 ixgbe_dev_tx_init(struct rte_eth_dev *dev)
5040 {
5041         struct ixgbe_hw     *hw;
5042         struct ixgbe_tx_queue *txq;
5043         uint64_t bus_addr;
5044         uint32_t hlreg0;
5045         uint32_t txctrl;
5046         uint16_t i;
5047
5048         PMD_INIT_FUNC_TRACE();
5049         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5050
5051         /* Enable TX CRC (checksum offload requirement) and hw padding
5052          * (TSO requirement)
5053          */
5054         hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
5055         hlreg0 |= (IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN);
5056         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);
5057
5058         /* Setup the Base and Length of the Tx Descriptor Rings */
5059         for (i = 0; i < dev->data->nb_tx_queues; i++) {
5060                 txq = dev->data->tx_queues[i];
5061
5062                 bus_addr = txq->tx_ring_phys_addr;
5063                 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(txq->reg_idx),
5064                                 (uint32_t)(bus_addr & 0x00000000ffffffffULL));
5065                 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(txq->reg_idx),
5066                                 (uint32_t)(bus_addr >> 32));
5067                 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(txq->reg_idx),
5068                                 txq->nb_tx_desc * sizeof(union ixgbe_adv_tx_desc));
5069                 /* Setup the HW Tx Head and TX Tail descriptor pointers */
5070                 IXGBE_WRITE_REG(hw, IXGBE_TDH(txq->reg_idx), 0);
5071                 IXGBE_WRITE_REG(hw, IXGBE_TDT(txq->reg_idx), 0);
5072
5073                 /*
5074                  * Disable Tx Head Writeback RO bit, since this hoses
5075                  * bookkeeping if things aren't delivered in order.
5076                  */
5077                 switch (hw->mac.type) {
5078                 case ixgbe_mac_82598EB:
5079                         txctrl = IXGBE_READ_REG(hw,
5080                                                 IXGBE_DCA_TXCTRL(txq->reg_idx));
5081                         txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
5082                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(txq->reg_idx),
5083                                         txctrl);
5084                         break;
5085
5086                 case ixgbe_mac_82599EB:
5087                 case ixgbe_mac_X540:
5088                 case ixgbe_mac_X550:
5089                 case ixgbe_mac_X550EM_x:
5090                 case ixgbe_mac_X550EM_a:
5091                 default:
5092                         txctrl = IXGBE_READ_REG(hw,
5093                                                 IXGBE_DCA_TXCTRL_82599(txq->reg_idx));
5094                         txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
5095                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(txq->reg_idx),
5096                                         txctrl);
5097                         break;
5098                 }
5099         }
5100
5101         /* Device configured with multiple TX queues. */
5102         ixgbe_dev_mq_tx_configure(dev);
5103 }
5104
5105 /*
5106  * Set up link for 82599 loopback mode Tx->Rx.
5107  */
5108 static inline void __attribute__((cold))
5109 ixgbe_setup_loopback_link_82599(struct ixgbe_hw *hw)
5110 {
5111         PMD_INIT_FUNC_TRACE();
5112
5113         if (ixgbe_verify_lesm_fw_enabled_82599(hw)) {
5114                 if (hw->mac.ops.acquire_swfw_sync(hw, IXGBE_GSSR_MAC_CSR_SM) !=
5115                                 IXGBE_SUCCESS) {
5116                         PMD_INIT_LOG(ERR, "Could not enable loopback mode");
5117                         /* ignore error */
5118                         return;
5119                 }
5120         }
5121
5122         /* Restart link */
5123         IXGBE_WRITE_REG(hw,
5124                         IXGBE_AUTOC,
5125                         IXGBE_AUTOC_LMS_10G_LINK_NO_AN | IXGBE_AUTOC_FLU);
5126         ixgbe_reset_pipeline_82599(hw);
5127
5128         hw->mac.ops.release_swfw_sync(hw, IXGBE_GSSR_MAC_CSR_SM);
5129         msec_delay(50);
5130 }
5131
5132
5133 /*
5134  * Start Transmit and Receive Units.
5135  */
5136 int __attribute__((cold))
5137 ixgbe_dev_rxtx_start(struct rte_eth_dev *dev)
5138 {
5139         struct ixgbe_hw     *hw;
5140         struct ixgbe_tx_queue *txq;
5141         struct ixgbe_rx_queue *rxq;
5142         uint32_t txdctl;
5143         uint32_t dmatxctl;
5144         uint32_t rxctrl;
5145         uint16_t i;
5146         int ret = 0;
5147
5148         PMD_INIT_FUNC_TRACE();
5149         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5150
5151         for (i = 0; i < dev->data->nb_tx_queues; i++) {
5152                 txq = dev->data->tx_queues[i];
5153                 /* Setup Transmit Threshold Registers */
5154                 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txq->reg_idx));
5155                 txdctl |= txq->pthresh & 0x7F;
5156                 txdctl |= ((txq->hthresh & 0x7F) << 8);
5157                 txdctl |= ((txq->wthresh & 0x7F) << 16);
5158                 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txq->reg_idx), txdctl);
5159         }
5160
5161         if (hw->mac.type != ixgbe_mac_82598EB) {
5162                 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
5163                 dmatxctl |= IXGBE_DMATXCTL_TE;
5164                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
5165         }
5166
5167         for (i = 0; i < dev->data->nb_tx_queues; i++) {
5168                 txq = dev->data->tx_queues[i];
5169                 if (!txq->tx_deferred_start) {
5170                         ret = ixgbe_dev_tx_queue_start(dev, i);
5171                         if (ret < 0)
5172                                 return ret;
5173                 }
5174         }
5175
5176         for (i = 0; i < dev->data->nb_rx_queues; i++) {
5177                 rxq = dev->data->rx_queues[i];
5178                 if (!rxq->rx_deferred_start) {
5179                         ret = ixgbe_dev_rx_queue_start(dev, i);
5180                         if (ret < 0)
5181                                 return ret;
5182                 }
5183         }
5184
5185         /* Enable Receive engine */
5186         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
5187         if (hw->mac.type == ixgbe_mac_82598EB)
5188                 rxctrl |= IXGBE_RXCTRL_DMBYPS;
5189         rxctrl |= IXGBE_RXCTRL_RXEN;
5190         hw->mac.ops.enable_rx_dma(hw, rxctrl);
5191
5192         /* If loopback mode is enabled for 82599, set up the link accordingly */
5193         if (hw->mac.type == ixgbe_mac_82599EB &&
5194                         dev->data->dev_conf.lpbk_mode == IXGBE_LPBK_82599_TX_RX)
5195                 ixgbe_setup_loopback_link_82599(hw);
5196
5197 #ifdef RTE_LIBRTE_SECURITY
5198         if ((dev->data->dev_conf.rxmode.offloads &
5199                         DEV_RX_OFFLOAD_SECURITY) ||
5200                 (dev->data->dev_conf.txmode.offloads &
5201                         DEV_TX_OFFLOAD_SECURITY)) {
5202                 ret = ixgbe_crypto_enable_ipsec(dev);
5203                 if (ret != 0) {
5204                         PMD_DRV_LOG(ERR,
5205                                     "ixgbe_crypto_enable_ipsec fails with %d.",
5206                                     ret);
5207                         return ret;
5208                 }
5209         }
5210 #endif
5211
5212         return 0;
5213 }
5214
5215 /*
5216  * Start Receive Units for specified queue.
5217  */
5218 int __attribute__((cold))
5219 ixgbe_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
5220 {
5221         struct ixgbe_hw     *hw;
5222         struct ixgbe_rx_queue *rxq;
5223         uint32_t rxdctl;
5224         int poll_ms;
5225
5226         PMD_INIT_FUNC_TRACE();
5227         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5228
5229         if (rx_queue_id < dev->data->nb_rx_queues) {
5230                 rxq = dev->data->rx_queues[rx_queue_id];
5231
5232                 /* Allocate buffers for descriptor rings */
5233                 if (ixgbe_alloc_rx_queue_mbufs(rxq) != 0) {
5234                         PMD_INIT_LOG(ERR, "Could not alloc mbuf for queue:%d",
5235                                      rx_queue_id);
5236                         return -1;
5237                 }
5238                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
5239                 rxdctl |= IXGBE_RXDCTL_ENABLE;
5240                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxq->reg_idx), rxdctl);
5241
5242                 /* Wait until RX Enable ready */
5243                 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
5244                 do {
5245                         rte_delay_ms(1);
5246                         rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
5247                 } while (--poll_ms && !(rxdctl & IXGBE_RXDCTL_ENABLE));
5248                 if (!poll_ms)
5249                         PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d",
5250                                      rx_queue_id);
5251                 rte_wmb();
5252                 IXGBE_WRITE_REG(hw, IXGBE_RDH(rxq->reg_idx), 0);
5253                 IXGBE_WRITE_REG(hw, IXGBE_RDT(rxq->reg_idx), rxq->nb_rx_desc - 1);
5254                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
5255         } else
5256                 return -1;
5257
5258         return 0;
5259 }
5260
5261 /*
5262  * Stop Receive Units for specified queue.
5263  */
5264 int __attribute__((cold))
5265 ixgbe_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
5266 {
5267         struct ixgbe_hw     *hw;
5268         struct ixgbe_adapter *adapter =
5269                 (struct ixgbe_adapter *)dev->data->dev_private;
5270         struct ixgbe_rx_queue *rxq;
5271         uint32_t rxdctl;
5272         int poll_ms;
5273
5274         PMD_INIT_FUNC_TRACE();
5275         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5276
5277         if (rx_queue_id < dev->data->nb_rx_queues) {
5278                 rxq = dev->data->rx_queues[rx_queue_id];
5279
5280                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
5281                 rxdctl &= ~IXGBE_RXDCTL_ENABLE;
5282                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxq->reg_idx), rxdctl);
5283
5284                 /* Wait until RX Enable bit clear */
5285                 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
5286                 do {
5287                         rte_delay_ms(1);
5288                         rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
5289                 } while (--poll_ms && (rxdctl & IXGBE_RXDCTL_ENABLE));
5290                 if (!poll_ms)
5291                         PMD_INIT_LOG(ERR, "Could not disable Rx Queue %d",
5292                                      rx_queue_id);
5293
5294                 rte_delay_us(RTE_IXGBE_WAIT_100_US);
5295
5296                 ixgbe_rx_queue_release_mbufs(rxq);
5297                 ixgbe_reset_rx_queue(adapter, rxq);
5298                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
5299         } else
5300                 return -1;
5301
5302         return 0;
5303 }
5304
5305
5306 /*
5307  * Start Transmit Units for specified queue.
5308  */
5309 int __attribute__((cold))
5310 ixgbe_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
5311 {
5312         struct ixgbe_hw     *hw;
5313         struct ixgbe_tx_queue *txq;
5314         uint32_t txdctl;
5315         int poll_ms;
5316
5317         PMD_INIT_FUNC_TRACE();
5318         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5319
5320         if (tx_queue_id < dev->data->nb_tx_queues) {
5321                 txq = dev->data->tx_queues[tx_queue_id];
5322                 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txq->reg_idx));
5323                 txdctl |= IXGBE_TXDCTL_ENABLE;
5324                 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txq->reg_idx), txdctl);
5325
5326                 /* Wait until TX Enable ready */
5327                 if (hw->mac.type == ixgbe_mac_82599EB) {
5328                         poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
5329                         do {
5330                                 rte_delay_ms(1);
5331                                 txdctl = IXGBE_READ_REG(hw,
5332                                         IXGBE_TXDCTL(txq->reg_idx));
5333                         } while (--poll_ms && !(txdctl & IXGBE_TXDCTL_ENABLE));
5334                         if (!poll_ms)
5335                                 PMD_INIT_LOG(ERR, "Could not enable "
5336                                              "Tx Queue %d", tx_queue_id);
5337                 }
5338                 rte_wmb();
5339                 IXGBE_WRITE_REG(hw, IXGBE_TDH(txq->reg_idx), 0);
5340                 IXGBE_WRITE_REG(hw, IXGBE_TDT(txq->reg_idx), 0);
5341                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
5342         } else
5343                 return -1;
5344
5345         return 0;
5346 }
5347
5348 /*
5349  * Stop Transmit Units for specified queue.
5350  */
5351 int __attribute__((cold))
5352 ixgbe_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
5353 {
5354         struct ixgbe_hw     *hw;
5355         struct ixgbe_tx_queue *txq;
5356         uint32_t txdctl;
5357         uint32_t txtdh, txtdt;
5358         int poll_ms;
5359
5360         PMD_INIT_FUNC_TRACE();
5361         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5362
5363         if (tx_queue_id >= dev->data->nb_tx_queues)
5364                 return -1;
5365
5366         txq = dev->data->tx_queues[tx_queue_id];
5367
5368         /* Wait until TX queue is empty */
5369         if (hw->mac.type == ixgbe_mac_82599EB) {
5370                 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
5371                 do {
5372                         rte_delay_us(RTE_IXGBE_WAIT_100_US);
5373                         txtdh = IXGBE_READ_REG(hw,
5374                                                IXGBE_TDH(txq->reg_idx));
5375                         txtdt = IXGBE_READ_REG(hw,
5376                                                IXGBE_TDT(txq->reg_idx));
5377                 } while (--poll_ms && (txtdh != txtdt));
5378                 if (!poll_ms)
5379                         PMD_INIT_LOG(ERR, "Tx Queue %d is not empty "
5380                                      "when stopping.", tx_queue_id);
5381         }
5382
5383         txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txq->reg_idx));
5384         txdctl &= ~IXGBE_TXDCTL_ENABLE;
5385         IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txq->reg_idx), txdctl);
5386
5387         /* Wait until TX Enable bit clear */
5388         if (hw->mac.type == ixgbe_mac_82599EB) {
5389                 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS;
5390                 do {
5391                         rte_delay_ms(1);
5392                         txdctl = IXGBE_READ_REG(hw,
5393                                                 IXGBE_TXDCTL(txq->reg_idx));
5394                 } while (--poll_ms && (txdctl & IXGBE_TXDCTL_ENABLE));
5395                 if (!poll_ms)
5396                         PMD_INIT_LOG(ERR, "Could not disable "
5397                                      "Tx Queue %d", tx_queue_id);
5398         }
5399
5400         if (txq->ops != NULL) {
5401                 txq->ops->release_mbufs(txq);
5402                 txq->ops->reset(txq);
5403         }
5404         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
5405
5406         return 0;
5407 }
5408
5409 void
5410 ixgbe_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
5411         struct rte_eth_rxq_info *qinfo)
5412 {
5413         struct ixgbe_rx_queue *rxq;
5414
5415         rxq = dev->data->rx_queues[queue_id];
5416
5417         qinfo->mp = rxq->mb_pool;
5418         qinfo->scattered_rx = dev->data->scattered_rx;
5419         qinfo->nb_desc = rxq->nb_rx_desc;
5420
5421         qinfo->conf.rx_free_thresh = rxq->rx_free_thresh;
5422         qinfo->conf.rx_drop_en = rxq->drop_en;
5423         qinfo->conf.rx_deferred_start = rxq->rx_deferred_start;
5424         qinfo->conf.offloads = rxq->offloads;
5425 }
5426
5427 void
5428 ixgbe_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
5429         struct rte_eth_txq_info *qinfo)
5430 {
5431         struct ixgbe_tx_queue *txq;
5432
5433         txq = dev->data->tx_queues[queue_id];
5434
5435         qinfo->nb_desc = txq->nb_tx_desc;
5436
5437         qinfo->conf.tx_thresh.pthresh = txq->pthresh;
5438         qinfo->conf.tx_thresh.hthresh = txq->hthresh;
5439         qinfo->conf.tx_thresh.wthresh = txq->wthresh;
5440
5441         qinfo->conf.tx_free_thresh = txq->tx_free_thresh;
5442         qinfo->conf.tx_rs_thresh = txq->tx_rs_thresh;
5443         qinfo->conf.offloads = txq->offloads;
5444         qinfo->conf.tx_deferred_start = txq->tx_deferred_start;
5445 }
5446
5447 /*
5448  * [VF] Initializes Receive Unit.
5449  */
5450 int __attribute__((cold))
5451 ixgbevf_dev_rx_init(struct rte_eth_dev *dev)
5452 {
5453         struct ixgbe_hw     *hw;
5454         struct ixgbe_rx_queue *rxq;
5455         struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
5456         uint64_t bus_addr;
5457         uint32_t srrctl, psrtype = 0;
5458         uint16_t buf_size;
5459         uint16_t i;
5460         int ret;
5461
5462         PMD_INIT_FUNC_TRACE();
5463         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5464
5465         if (rte_is_power_of_2(dev->data->nb_rx_queues) == 0) {
5466                 PMD_INIT_LOG(ERR, "The number of Rx queue invalid, "
5467                         "it should be power of 2");
5468                 return -1;
5469         }
5470
5471         if (dev->data->nb_rx_queues > hw->mac.max_rx_queues) {
5472                 PMD_INIT_LOG(ERR, "The number of Rx queue invalid, "
5473                         "it should be equal to or less than %d",
5474                         hw->mac.max_rx_queues);
5475                 return -1;
5476         }
5477
5478         /*
5479          * When the VF driver issues a IXGBE_VF_RESET request, the PF driver
5480          * disables the VF receipt of packets if the PF MTU is > 1500.
5481          * This is done to deal with 82599 limitations that imposes
5482          * the PF and all VFs to share the same MTU.
5483          * Then, the PF driver enables again the VF receipt of packet when
5484          * the VF driver issues a IXGBE_VF_SET_LPE request.
5485          * In the meantime, the VF device cannot be used, even if the VF driver
5486          * and the Guest VM network stack are ready to accept packets with a
5487          * size up to the PF MTU.
5488          * As a work-around to this PF behaviour, force the call to
5489          * ixgbevf_rlpml_set_vf even if jumbo frames are not used. This way,
5490          * VF packets received can work in all cases.
5491          */
5492         ixgbevf_rlpml_set_vf(hw,
5493                 (uint16_t)dev->data->dev_conf.rxmode.max_rx_pkt_len);
5494
5495         /*
5496          * Assume no header split and no VLAN strip support
5497          * on any Rx queue first .
5498          */
5499         rxmode->offloads &= ~DEV_RX_OFFLOAD_VLAN_STRIP;
5500         /* Setup RX queues */
5501         for (i = 0; i < dev->data->nb_rx_queues; i++) {
5502                 rxq = dev->data->rx_queues[i];
5503
5504                 /* Allocate buffers for descriptor rings */
5505                 ret = ixgbe_alloc_rx_queue_mbufs(rxq);
5506                 if (ret)
5507                         return ret;
5508
5509                 /* Setup the Base and Length of the Rx Descriptor Rings */
5510                 bus_addr = rxq->rx_ring_phys_addr;
5511
5512                 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(i),
5513                                 (uint32_t)(bus_addr & 0x00000000ffffffffULL));
5514                 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(i),
5515                                 (uint32_t)(bus_addr >> 32));
5516                 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(i),
5517                                 rxq->nb_rx_desc * sizeof(union ixgbe_adv_rx_desc));
5518                 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(i), 0);
5519                 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(i), 0);
5520
5521
5522                 /* Configure the SRRCTL register */
5523                 srrctl = IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
5524
5525                 /* Set if packets are dropped when no descriptors available */
5526                 if (rxq->drop_en)
5527                         srrctl |= IXGBE_SRRCTL_DROP_EN;
5528
5529                 /*
5530                  * Configure the RX buffer size in the BSIZEPACKET field of
5531                  * the SRRCTL register of the queue.
5532                  * The value is in 1 KB resolution. Valid values can be from
5533                  * 1 KB to 16 KB.
5534                  */
5535                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) -
5536                         RTE_PKTMBUF_HEADROOM);
5537                 srrctl |= ((buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) &
5538                            IXGBE_SRRCTL_BSIZEPKT_MASK);
5539
5540                 /*
5541                  * VF modification to write virtual function SRRCTL register
5542                  */
5543                 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(i), srrctl);
5544
5545                 buf_size = (uint16_t) ((srrctl & IXGBE_SRRCTL_BSIZEPKT_MASK) <<
5546                                        IXGBE_SRRCTL_BSIZEPKT_SHIFT);
5547
5548                 if (rxmode->offloads & DEV_RX_OFFLOAD_SCATTER ||
5549                     /* It adds dual VLAN length for supporting dual VLAN */
5550                     (rxmode->max_rx_pkt_len +
5551                                 2 * IXGBE_VLAN_TAG_SIZE) > buf_size) {
5552                         if (!dev->data->scattered_rx)
5553                                 PMD_INIT_LOG(DEBUG, "forcing scatter mode");
5554                         dev->data->scattered_rx = 1;
5555                 }
5556
5557                 if (rxq->offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
5558                         rxmode->offloads |= DEV_RX_OFFLOAD_VLAN_STRIP;
5559         }
5560
5561         /* Set RQPL for VF RSS according to max Rx queue */
5562         psrtype |= (dev->data->nb_rx_queues >> 1) <<
5563                 IXGBE_PSRTYPE_RQPL_SHIFT;
5564         IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype);
5565
5566         ixgbe_set_rx_function(dev);
5567
5568         return 0;
5569 }
5570
5571 /*
5572  * [VF] Initializes Transmit Unit.
5573  */
5574 void __attribute__((cold))
5575 ixgbevf_dev_tx_init(struct rte_eth_dev *dev)
5576 {
5577         struct ixgbe_hw     *hw;
5578         struct ixgbe_tx_queue *txq;
5579         uint64_t bus_addr;
5580         uint32_t txctrl;
5581         uint16_t i;
5582
5583         PMD_INIT_FUNC_TRACE();
5584         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5585
5586         /* Setup the Base and Length of the Tx Descriptor Rings */
5587         for (i = 0; i < dev->data->nb_tx_queues; i++) {
5588                 txq = dev->data->tx_queues[i];
5589                 bus_addr = txq->tx_ring_phys_addr;
5590                 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(i),
5591                                 (uint32_t)(bus_addr & 0x00000000ffffffffULL));
5592                 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(i),
5593                                 (uint32_t)(bus_addr >> 32));
5594                 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(i),
5595                                 txq->nb_tx_desc * sizeof(union ixgbe_adv_tx_desc));
5596                 /* Setup the HW Tx Head and TX Tail descriptor pointers */
5597                 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(i), 0);
5598                 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(i), 0);
5599
5600                 /*
5601                  * Disable Tx Head Writeback RO bit, since this hoses
5602                  * bookkeeping if things aren't delivered in order.
5603                  */
5604                 txctrl = IXGBE_READ_REG(hw,
5605                                 IXGBE_VFDCA_TXCTRL(i));
5606                 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
5607                 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(i),
5608                                 txctrl);
5609         }
5610 }
5611
5612 /*
5613  * [VF] Start Transmit and Receive Units.
5614  */
5615 void __attribute__((cold))
5616 ixgbevf_dev_rxtx_start(struct rte_eth_dev *dev)
5617 {
5618         struct ixgbe_hw     *hw;
5619         struct ixgbe_tx_queue *txq;
5620         struct ixgbe_rx_queue *rxq;
5621         uint32_t txdctl;
5622         uint32_t rxdctl;
5623         uint16_t i;
5624         int poll_ms;
5625
5626         PMD_INIT_FUNC_TRACE();
5627         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5628
5629         for (i = 0; i < dev->data->nb_tx_queues; i++) {
5630                 txq = dev->data->tx_queues[i];
5631                 /* Setup Transmit Threshold Registers */
5632                 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i));
5633                 txdctl |= txq->pthresh & 0x7F;
5634                 txdctl |= ((txq->hthresh & 0x7F) << 8);
5635                 txdctl |= ((txq->wthresh & 0x7F) << 16);
5636                 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl);
5637         }
5638
5639         for (i = 0; i < dev->data->nb_tx_queues; i++) {
5640
5641                 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i));
5642                 txdctl |= IXGBE_TXDCTL_ENABLE;
5643                 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl);
5644
5645                 poll_ms = 10;
5646                 /* Wait until TX Enable ready */
5647                 do {
5648                         rte_delay_ms(1);
5649                         txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i));
5650                 } while (--poll_ms && !(txdctl & IXGBE_TXDCTL_ENABLE));
5651                 if (!poll_ms)
5652                         PMD_INIT_LOG(ERR, "Could not enable Tx Queue %d", i);
5653         }
5654         for (i = 0; i < dev->data->nb_rx_queues; i++) {
5655
5656                 rxq = dev->data->rx_queues[i];
5657
5658                 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i));
5659                 rxdctl |= IXGBE_RXDCTL_ENABLE;
5660                 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), rxdctl);
5661
5662                 /* Wait until RX Enable ready */
5663                 poll_ms = 10;
5664                 do {
5665                         rte_delay_ms(1);
5666                         rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i));
5667                 } while (--poll_ms && !(rxdctl & IXGBE_RXDCTL_ENABLE));
5668                 if (!poll_ms)
5669                         PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", i);
5670                 rte_wmb();
5671                 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(i), rxq->nb_rx_desc - 1);
5672
5673         }
5674 }
5675
5676 int
5677 ixgbe_rss_conf_init(struct ixgbe_rte_flow_rss_conf *out,
5678                     const struct rte_flow_action_rss *in)
5679 {
5680         if (in->key_len > RTE_DIM(out->key) ||
5681             in->queue_num > RTE_DIM(out->queue))
5682                 return -EINVAL;
5683         out->conf = (struct rte_flow_action_rss){
5684                 .func = in->func,
5685                 .level = in->level,
5686                 .types = in->types,
5687                 .key_len = in->key_len,
5688                 .queue_num = in->queue_num,
5689                 .key = memcpy(out->key, in->key, in->key_len),
5690                 .queue = memcpy(out->queue, in->queue,
5691                                 sizeof(*in->queue) * in->queue_num),
5692         };
5693         return 0;
5694 }
5695
5696 int
5697 ixgbe_action_rss_same(const struct rte_flow_action_rss *comp,
5698                       const struct rte_flow_action_rss *with)
5699 {
5700         return (comp->func == with->func &&
5701                 comp->level == with->level &&
5702                 comp->types == with->types &&
5703                 comp->key_len == with->key_len &&
5704                 comp->queue_num == with->queue_num &&
5705                 !memcmp(comp->key, with->key, with->key_len) &&
5706                 !memcmp(comp->queue, with->queue,
5707                         sizeof(*with->queue) * with->queue_num));
5708 }
5709
5710 int
5711 ixgbe_config_rss_filter(struct rte_eth_dev *dev,
5712                 struct ixgbe_rte_flow_rss_conf *conf, bool add)
5713 {
5714         struct ixgbe_hw *hw;
5715         uint32_t reta;
5716         uint16_t i;
5717         uint16_t j;
5718         uint16_t sp_reta_size;
5719         uint32_t reta_reg;
5720         struct rte_eth_rss_conf rss_conf = {
5721                 .rss_key = conf->conf.key_len ?
5722                         (void *)(uintptr_t)conf->conf.key : NULL,
5723                 .rss_key_len = conf->conf.key_len,
5724                 .rss_hf = conf->conf.types,
5725         };
5726         struct ixgbe_filter_info *filter_info =
5727                 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
5728
5729         PMD_INIT_FUNC_TRACE();
5730         hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5731
5732         sp_reta_size = ixgbe_reta_size_get(hw->mac.type);
5733
5734         if (!add) {
5735                 if (ixgbe_action_rss_same(&filter_info->rss_info.conf,
5736                                           &conf->conf)) {
5737                         ixgbe_rss_disable(dev);
5738                         memset(&filter_info->rss_info, 0,
5739                                 sizeof(struct ixgbe_rte_flow_rss_conf));
5740                         return 0;
5741                 }
5742                 return -EINVAL;
5743         }
5744
5745         if (filter_info->rss_info.conf.queue_num)
5746                 return -EINVAL;
5747         /* Fill in redirection table
5748          * The byte-swap is needed because NIC registers are in
5749          * little-endian order.
5750          */
5751         reta = 0;
5752         for (i = 0, j = 0; i < sp_reta_size; i++, j++) {
5753                 reta_reg = ixgbe_reta_reg_get(hw->mac.type, i);
5754
5755                 if (j == conf->conf.queue_num)
5756                         j = 0;
5757                 reta = (reta << 8) | conf->conf.queue[j];
5758                 if ((i & 3) == 3)
5759                         IXGBE_WRITE_REG(hw, reta_reg,
5760                                         rte_bswap32(reta));
5761         }
5762
5763         /* Configure the RSS key and the RSS protocols used to compute
5764          * the RSS hash of input packets.
5765          */
5766         if ((rss_conf.rss_hf & IXGBE_RSS_OFFLOAD_ALL) == 0) {
5767                 ixgbe_rss_disable(dev);
5768                 return -EINVAL;
5769         }
5770         if (rss_conf.rss_key == NULL)
5771                 rss_conf.rss_key = rss_intel_key; /* Default hash key */
5772         ixgbe_hw_rss_hash_set(hw, &rss_conf);
5773
5774         if (ixgbe_rss_conf_init(&filter_info->rss_info, &conf->conf))
5775                 return -EINVAL;
5776
5777         return 0;
5778 }
5779
5780 /* Stubs needed for linkage when CONFIG_RTE_IXGBE_INC_VECTOR is set to 'n' */
5781 int __attribute__((weak))
5782 ixgbe_rx_vec_dev_conf_condition_check(struct rte_eth_dev __rte_unused *dev)
5783 {
5784         return -1;
5785 }
5786
5787 uint16_t __attribute__((weak))
5788 ixgbe_recv_pkts_vec(
5789         void __rte_unused *rx_queue,
5790         struct rte_mbuf __rte_unused **rx_pkts,
5791         uint16_t __rte_unused nb_pkts)
5792 {
5793         return 0;
5794 }
5795
5796 uint16_t __attribute__((weak))
5797 ixgbe_recv_scattered_pkts_vec(
5798         void __rte_unused *rx_queue,
5799         struct rte_mbuf __rte_unused **rx_pkts,
5800         uint16_t __rte_unused nb_pkts)
5801 {
5802         return 0;
5803 }
5804
5805 int __attribute__((weak))
5806 ixgbe_rxq_vec_setup(struct ixgbe_rx_queue __rte_unused *rxq)
5807 {
5808         return -1;
5809 }