drivers/net/i40e/i40e_rxtx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2016 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <stdlib.h>
   7 #include <string.h>
   8 #include <errno.h>
   9 #include <stdint.h>
  10 #include <stdarg.h>
  11 #include <unistd.h>
  12 #include <inttypes.h>
  13 #include <sys/queue.h>
  14
  15 #include <rte_string_fns.h>
  16 #include <rte_memzone.h>
  17 #include <rte_mbuf.h>
  18 #include <rte_malloc.h>
  19 #include <rte_ether.h>
  20 #include <rte_ethdev.h>
  21 #include <rte_tcp.h>
  22 #include <rte_sctp.h>
  23 #include <rte_udp.h>
  24 #include <rte_ip.h>
  25 #include <rte_net.h>
  26
  27 #include "i40e_logs.h"
  28 #include "base/i40e_prototype.h"
  29 #include "base/i40e_type.h"
  30 #include "i40e_ethdev.h"
  31 #include "i40e_rxtx.h"
  32
  33 #define DEFAULT_TX_RS_THRESH   32
  34 #define DEFAULT_TX_FREE_THRESH 32
  35
  36 #define I40E_TX_MAX_BURST  32
  37
  38 #define I40E_DMA_MEM_ALIGN 4096
  39
  40 /* Base address of the HW descriptor ring should be 128B aligned. */
  41 #define I40E_RING_BASE_ALIGN    128
  42
  43 #define I40E_SIMPLE_FLAGS ((uint32_t)ETH_TXQ_FLAGS_NOMULTSEGS | \
  44                                         ETH_TXQ_FLAGS_NOOFFLOADS)
  45
  46 #define I40E_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS)
  47
  48 #ifdef RTE_LIBRTE_IEEE1588
  49 #define I40E_TX_IEEE1588_TMST PKT_TX_IEEE1588_TMST
  50 #else
  51 #define I40E_TX_IEEE1588_TMST 0
  52 #endif
  53
  54 #define I40E_TX_CKSUM_OFFLOAD_MASK (             \
  55                 PKT_TX_IP_CKSUM |                \
  56                 PKT_TX_L4_MASK |                 \
  57                 PKT_TX_TCP_SEG |                 \
  58                 PKT_TX_OUTER_IP_CKSUM)
  59
  60 #define I40E_TX_OFFLOAD_MASK (  \
  61                 PKT_TX_IP_CKSUM |       \
  62                 PKT_TX_L4_MASK |        \
  63                 PKT_TX_OUTER_IP_CKSUM | \
  64                 PKT_TX_TCP_SEG |        \
  65                 PKT_TX_QINQ_PKT |       \
  66                 PKT_TX_VLAN_PKT |       \
  67                 PKT_TX_TUNNEL_MASK |    \
  68                 I40E_TX_IEEE1588_TMST)
  69
  70 #define I40E_TX_OFFLOAD_NOTSUP_MASK \
  71                 (PKT_TX_OFFLOAD_MASK ^ I40E_TX_OFFLOAD_MASK)
  72
  73 static uint16_t i40e_xmit_pkts_simple(void *tx_queue,
  74                                       struct rte_mbuf **tx_pkts,
  75                                       uint16_t nb_pkts);
  76
  77 static inline void
  78 i40e_rxd_to_vlan_tci(struct rte_mbuf *mb, volatile union i40e_rx_desc *rxdp)
  79 {
  80         if (rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len) &
  81                 (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)) {
  82                 mb->ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED;
  83                 mb->vlan_tci =
  84                         rte_le_to_cpu_16(rxdp->wb.qword0.lo_dword.l2tag1);
  85                 PMD_RX_LOG(DEBUG, "Descriptor l2tag1: %u",
  86                            rte_le_to_cpu_16(rxdp->wb.qword0.lo_dword.l2tag1));
  87         } else {
  88                 mb->vlan_tci = 0;
  89         }
  90 #ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC
  91         if (rte_le_to_cpu_16(rxdp->wb.qword2.ext_status) &
  92                 (1 << I40E_RX_DESC_EXT_STATUS_L2TAG2P_SHIFT)) {
  93                 mb->ol_flags |= PKT_RX_QINQ_STRIPPED;
  94                 mb->vlan_tci_outer = mb->vlan_tci;
  95                 mb->vlan_tci = rte_le_to_cpu_16(rxdp->wb.qword2.l2tag2_2);
  96                 PMD_RX_LOG(DEBUG, "Descriptor l2tag2_1: %u, l2tag2_2: %u",
  97                            rte_le_to_cpu_16(rxdp->wb.qword2.l2tag2_1),
  98                            rte_le_to_cpu_16(rxdp->wb.qword2.l2tag2_2));
  99         } else {
 100                 mb->vlan_tci_outer = 0;
 101         }
 102 #endif
 103         PMD_RX_LOG(DEBUG, "Mbuf vlan_tci: %u, vlan_tci_outer: %u",
 104                    mb->vlan_tci, mb->vlan_tci_outer);
 105 }
 106
 107 /* Translate the rx descriptor status to pkt flags */
 108 static inline uint64_t
 109 i40e_rxd_status_to_pkt_flags(uint64_t qword)
 110 {
 111         uint64_t flags;
 112
 113         /* Check if RSS_HASH */
 114         flags = (((qword >> I40E_RX_DESC_STATUS_FLTSTAT_SHIFT) &
 115                                         I40E_RX_DESC_FLTSTAT_RSS_HASH) ==
 116                         I40E_RX_DESC_FLTSTAT_RSS_HASH) ? PKT_RX_RSS_HASH : 0;
 117
 118         /* Check if FDIR Match */
 119         flags |= (qword & (1 << I40E_RX_DESC_STATUS_FLM_SHIFT) ?
 120                                                         PKT_RX_FDIR : 0);
 121
 122         return flags;
 123 }
 124
 125 static inline uint64_t
 126 i40e_rxd_error_to_pkt_flags(uint64_t qword)
 127 {
 128         uint64_t flags = 0;
 129         uint64_t error_bits = (qword >> I40E_RXD_QW1_ERROR_SHIFT);
 130
 131 #define I40E_RX_ERR_BITS 0x3f
 132         if (likely((error_bits & I40E_RX_ERR_BITS) == 0)) {
 133                 flags |= (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD);
 134                 return flags;
 135         }
 136
 137         if (unlikely(error_bits & (1 << I40E_RX_DESC_ERROR_IPE_SHIFT)))
 138                 flags |= PKT_RX_IP_CKSUM_BAD;
 139         else
 140                 flags |= PKT_RX_IP_CKSUM_GOOD;
 141
 142         if (unlikely(error_bits & (1 << I40E_RX_DESC_ERROR_L4E_SHIFT)))
 143                 flags |= PKT_RX_L4_CKSUM_BAD;
 144         else
 145                 flags |= PKT_RX_L4_CKSUM_GOOD;
 146
 147         if (unlikely(error_bits & (1 << I40E_RX_DESC_ERROR_EIPE_SHIFT)))
 148                 flags |= PKT_RX_EIP_CKSUM_BAD;
 149
 150         return flags;
 151 }
 152
 153 /* Function to check and set the ieee1588 timesync index and get the
 154  * appropriate flags.
 155  */
 156 #ifdef RTE_LIBRTE_IEEE1588
 157 static inline uint64_t
 158 i40e_get_iee15888_flags(struct rte_mbuf *mb, uint64_t qword)
 159 {
 160         uint64_t pkt_flags = 0;
 161         uint16_t tsyn = (qword & (I40E_RXD_QW1_STATUS_TSYNVALID_MASK
 162                                   | I40E_RXD_QW1_STATUS_TSYNINDX_MASK))
 163                                     >> I40E_RX_DESC_STATUS_TSYNINDX_SHIFT;
 164
 165         if ((mb->packet_type & RTE_PTYPE_L2_MASK)
 166                         == RTE_PTYPE_L2_ETHER_TIMESYNC)
 167                 pkt_flags = PKT_RX_IEEE1588_PTP;
 168         if (tsyn & 0x04) {
 169                 pkt_flags |= PKT_RX_IEEE1588_TMST;
 170                 mb->timesync = tsyn & 0x03;
 171         }
 172
 173         return pkt_flags;
 174 }
 175 #endif
 176
 177 #define I40E_RX_DESC_EXT_STATUS_FLEXBH_MASK   0x03
 178 #define I40E_RX_DESC_EXT_STATUS_FLEXBH_FD_ID  0x01
 179 #define I40E_RX_DESC_EXT_STATUS_FLEXBH_FLEX   0x02
 180 #define I40E_RX_DESC_EXT_STATUS_FLEXBL_MASK   0x03
 181 #define I40E_RX_DESC_EXT_STATUS_FLEXBL_FLEX   0x01
 182
 183 static inline uint64_t
 184 i40e_rxd_build_fdir(volatile union i40e_rx_desc *rxdp, struct rte_mbuf *mb)
 185 {
 186         uint64_t flags = 0;
 187 #ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC
 188         uint16_t flexbh, flexbl;
 189
 190         flexbh = (rte_le_to_cpu_32(rxdp->wb.qword2.ext_status) >>
 191                 I40E_RX_DESC_EXT_STATUS_FLEXBH_SHIFT) &
 192                 I40E_RX_DESC_EXT_STATUS_FLEXBH_MASK;
 193         flexbl = (rte_le_to_cpu_32(rxdp->wb.qword2.ext_status) >>
 194                 I40E_RX_DESC_EXT_STATUS_FLEXBL_SHIFT) &
 195                 I40E_RX_DESC_EXT_STATUS_FLEXBL_MASK;
 196
 197
 198         if (flexbh == I40E_RX_DESC_EXT_STATUS_FLEXBH_FD_ID) {
 199                 mb->hash.fdir.hi =
 200                         rte_le_to_cpu_32(rxdp->wb.qword3.hi_dword.fd_id);
 201                 flags |= PKT_RX_FDIR_ID;
 202         } else if (flexbh == I40E_RX_DESC_EXT_STATUS_FLEXBH_FLEX) {
 203                 mb->hash.fdir.hi =
 204                         rte_le_to_cpu_32(rxdp->wb.qword3.hi_dword.flex_bytes_hi);
 205                 flags |= PKT_RX_FDIR_FLX;
 206         }
 207         if (flexbl == I40E_RX_DESC_EXT_STATUS_FLEXBL_FLEX) {
 208                 mb->hash.fdir.lo =
 209                         rte_le_to_cpu_32(rxdp->wb.qword3.lo_dword.flex_bytes_lo);
 210                 flags |= PKT_RX_FDIR_FLX;
 211         }
 212 #else
 213         mb->hash.fdir.hi =
 214                 rte_le_to_cpu_32(rxdp->wb.qword0.hi_dword.fd_id);
 215         flags |= PKT_RX_FDIR_ID;
 216 #endif
 217         return flags;
 218 }
 219
 220 static inline void
 221 i40e_parse_tunneling_params(uint64_t ol_flags,
 222                             union i40e_tx_offload tx_offload,
 223                             uint32_t *cd_tunneling)
 224 {
 225         /* EIPT: External (outer) IP header type */
 226         if (ol_flags & PKT_TX_OUTER_IP_CKSUM)
 227                 *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV4;
 228         else if (ol_flags & PKT_TX_OUTER_IPV4)
 229                 *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM;
 230         else if (ol_flags & PKT_TX_OUTER_IPV6)
 231                 *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV6;
 232
 233         /* EIPLEN: External (outer) IP header length, in DWords */
 234         *cd_tunneling |= (tx_offload.outer_l3_len >> 2) <<
 235                 I40E_TXD_CTX_QW0_EXT_IPLEN_SHIFT;
 236
 237         /* L4TUNT: L4 Tunneling Type */
 238         switch (ol_flags & PKT_TX_TUNNEL_MASK) {
 239         case PKT_TX_TUNNEL_IPIP:
 240                 /* for non UDP / GRE tunneling, set to 00b */
 241                 break;
 242         case PKT_TX_TUNNEL_VXLAN:
 243         case PKT_TX_TUNNEL_GENEVE:
 244                 *cd_tunneling |= I40E_TXD_CTX_UDP_TUNNELING;
 245                 break;
 246         case PKT_TX_TUNNEL_GRE:
 247                 *cd_tunneling |= I40E_TXD_CTX_GRE_TUNNELING;
 248                 break;
 249         default:
 250                 PMD_TX_LOG(ERR, "Tunnel type not supported");
 251                 return;
 252         }
 253
 254         /* L4TUNLEN: L4 Tunneling Length, in Words
 255          *
 256          * We depend on app to set rte_mbuf.l2_len correctly.
 257          * For IP in GRE it should be set to the length of the GRE
 258          * header;
 259          * for MAC in GRE or MAC in UDP it should be set to the length
 260          * of the GRE or UDP headers plus the inner MAC up to including
 261          * its last Ethertype.
 262          */
 263         *cd_tunneling |= (tx_offload.l2_len >> 1) <<
 264                 I40E_TXD_CTX_QW0_NATLEN_SHIFT;
 265 }
 266
 267 static inline void
 268 i40e_txd_enable_checksum(uint64_t ol_flags,
 269                         uint32_t *td_cmd,
 270                         uint32_t *td_offset,
 271                         union i40e_tx_offload tx_offload)
 272 {
 273         /* Set MACLEN */
 274         if (ol_flags & PKT_TX_TUNNEL_MASK)
 275                 *td_offset |= (tx_offload.outer_l2_len >> 1)
 276                                 << I40E_TX_DESC_LENGTH_MACLEN_SHIFT;
 277         else
 278                 *td_offset |= (tx_offload.l2_len >> 1)
 279                         << I40E_TX_DESC_LENGTH_MACLEN_SHIFT;
 280
 281         /* Enable L3 checksum offloads */
 282         if (ol_flags & PKT_TX_IP_CKSUM) {
 283                 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4_CSUM;
 284                 *td_offset |= (tx_offload.l3_len >> 2)
 285                                 << I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
 286         } else if (ol_flags & PKT_TX_IPV4) {
 287                 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4;
 288                 *td_offset |= (tx_offload.l3_len >> 2)
 289                                 << I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
 290         } else if (ol_flags & PKT_TX_IPV6) {
 291                 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV6;
 292                 *td_offset |= (tx_offload.l3_len >> 2)
 293                                 << I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
 294         }
 295
 296         if (ol_flags & PKT_TX_TCP_SEG) {
 297                 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP;
 298                 *td_offset |= (tx_offload.l4_len >> 2)
 299                         << I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
 300                 return;
 301         }
 302
 303         /* Enable L4 checksum offloads */
 304         switch (ol_flags & PKT_TX_L4_MASK) {
 305         case PKT_TX_TCP_CKSUM:
 306                 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP;
 307                 *td_offset |= (sizeof(struct tcp_hdr) >> 2) <<
 308                                 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
 309                 break;
 310         case PKT_TX_SCTP_CKSUM:
 311                 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP;
 312                 *td_offset |= (sizeof(struct sctp_hdr) >> 2) <<
 313                                 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
 314                 break;
 315         case PKT_TX_UDP_CKSUM:
 316                 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP;
 317                 *td_offset |= (sizeof(struct udp_hdr) >> 2) <<
 318                                 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
 319                 break;
 320         default:
 321                 break;
 322         }
 323 }
 324
 325 /* Construct the tx flags */
 326 static inline uint64_t
 327 i40e_build_ctob(uint32_t td_cmd,
 328                 uint32_t td_offset,
 329                 unsigned int size,
 330                 uint32_t td_tag)
 331 {
 332         return rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DATA |
 333                         ((uint64_t)td_cmd  << I40E_TXD_QW1_CMD_SHIFT) |
 334                         ((uint64_t)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) |
 335                         ((uint64_t)size  << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) |
 336                         ((uint64_t)td_tag  << I40E_TXD_QW1_L2TAG1_SHIFT));
 337 }
 338
 339 static inline int
 340 i40e_xmit_cleanup(struct i40e_tx_queue *txq)
 341 {
 342         struct i40e_tx_entry *sw_ring = txq->sw_ring;
 343         volatile struct i40e_tx_desc *txd = txq->tx_ring;
 344         uint16_t last_desc_cleaned = txq->last_desc_cleaned;
 345         uint16_t nb_tx_desc = txq->nb_tx_desc;
 346         uint16_t desc_to_clean_to;
 347         uint16_t nb_tx_to_clean;
 348
 349         desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->tx_rs_thresh);
 350         if (desc_to_clean_to >= nb_tx_desc)
 351                 desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc);
 352
 353         desc_to_clean_to = sw_ring[desc_to_clean_to].last_id;
 354         if ((txd[desc_to_clean_to].cmd_type_offset_bsz &
 355                         rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) !=
 356                         rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE)) {
 357                 PMD_TX_FREE_LOG(DEBUG, "TX descriptor %4u is not done "
 358                         "(port=%d queue=%d)", desc_to_clean_to,
 359                                 txq->port_id, txq->queue_id);
 360                 return -1;
 361         }
 362
 363         if (last_desc_cleaned > desc_to_clean_to)
 364                 nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) +
 365                                                         desc_to_clean_to);
 366         else
 367                 nb_tx_to_clean = (uint16_t)(desc_to_clean_to -
 368                                         last_desc_cleaned);
 369
 370         txd[desc_to_clean_to].cmd_type_offset_bsz = 0;
 371
 372         txq->last_desc_cleaned = desc_to_clean_to;
 373         txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + nb_tx_to_clean);
 374
 375         return 0;
 376 }
 377
 378 static inline int
 379 #ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC
 380 check_rx_burst_bulk_alloc_preconditions(struct i40e_rx_queue *rxq)
 381 #else
 382 check_rx_burst_bulk_alloc_preconditions(__rte_unused struct i40e_rx_queue *rxq)
 383 #endif
 384 {
 385         int ret = 0;
 386
 387 #ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC
 388         if (!(rxq->rx_free_thresh >= RTE_PMD_I40E_RX_MAX_BURST)) {
 389                 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
 390                              "rxq->rx_free_thresh=%d, "
 391                              "RTE_PMD_I40E_RX_MAX_BURST=%d",
 392                              rxq->rx_free_thresh, RTE_PMD_I40E_RX_MAX_BURST);
 393                 ret = -EINVAL;
 394         } else if (!(rxq->rx_free_thresh < rxq->nb_rx_desc)) {
 395                 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
 396                              "rxq->rx_free_thresh=%d, "
 397                              "rxq->nb_rx_desc=%d",
 398                              rxq->rx_free_thresh, rxq->nb_rx_desc);
 399                 ret = -EINVAL;
 400         } else if (rxq->nb_rx_desc % rxq->rx_free_thresh != 0) {
 401                 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
 402                              "rxq->nb_rx_desc=%d, "
 403                              "rxq->rx_free_thresh=%d",
 404                              rxq->nb_rx_desc, rxq->rx_free_thresh);
 405                 ret = -EINVAL;
 406         }
 407 #else
 408         ret = -EINVAL;
 409 #endif
 410
 411         return ret;
 412 }
 413
 414 #ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC
 415 #define I40E_LOOK_AHEAD 8
 416 #if (I40E_LOOK_AHEAD != 8)
 417 #error "PMD I40E: I40E_LOOK_AHEAD must be 8\n"
 418 #endif
 419 static inline int
 420 i40e_rx_scan_hw_ring(struct i40e_rx_queue *rxq)
 421 {
 422         volatile union i40e_rx_desc *rxdp;
 423         struct i40e_rx_entry *rxep;
 424         struct rte_mbuf *mb;
 425         uint16_t pkt_len;
 426         uint64_t qword1;
 427         uint32_t rx_status;
 428         int32_t s[I40E_LOOK_AHEAD], nb_dd;
 429         int32_t i, j, nb_rx = 0;
 430         uint64_t pkt_flags;
 431         uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl;
 432
 433         rxdp = &rxq->rx_ring[rxq->rx_tail];
 434         rxep = &rxq->sw_ring[rxq->rx_tail];
 435
 436         qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len);
 437         rx_status = (qword1 & I40E_RXD_QW1_STATUS_MASK) >>
 438                                 I40E_RXD_QW1_STATUS_SHIFT;
 439
 440         /* Make sure there is at least 1 packet to receive */
 441         if (!(rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)))
 442                 return 0;
 443
 444         /**
 445          * Scan LOOK_AHEAD descriptors at a time to determine which
 446          * descriptors reference packets that are ready to be received.
 447          */
 448         for (i = 0; i < RTE_PMD_I40E_RX_MAX_BURST; i+=I40E_LOOK_AHEAD,
 449                         rxdp += I40E_LOOK_AHEAD, rxep += I40E_LOOK_AHEAD) {
 450                 /* Read desc statuses backwards to avoid race condition */
 451                 for (j = I40E_LOOK_AHEAD - 1; j >= 0; j--) {
 452                         qword1 = rte_le_to_cpu_64(\
 453                                 rxdp[j].wb.qword1.status_error_len);
 454                         s[j] = (qword1 & I40E_RXD_QW1_STATUS_MASK) >>
 455                                         I40E_RXD_QW1_STATUS_SHIFT;
 456                 }
 457
 458                 rte_smp_rmb();
 459
 460                 /* Compute how many status bits were set */
 461                 for (j = 0, nb_dd = 0; j < I40E_LOOK_AHEAD; j++)
 462                         nb_dd += s[j] & (1 << I40E_RX_DESC_STATUS_DD_SHIFT);
 463
 464                 nb_rx += nb_dd;
 465
 466                 /* Translate descriptor info to mbuf parameters */
 467                 for (j = 0; j < nb_dd; j++) {
 468                         mb = rxep[j].mbuf;
 469                         qword1 = rte_le_to_cpu_64(\
 470                                 rxdp[j].wb.qword1.status_error_len);
 471                         pkt_len = ((qword1 & I40E_RXD_QW1_LENGTH_PBUF_MASK) >>
 472                                 I40E_RXD_QW1_LENGTH_PBUF_SHIFT) - rxq->crc_len;
 473                         mb->data_len = pkt_len;
 474                         mb->pkt_len = pkt_len;
 475                         mb->ol_flags = 0;
 476                         i40e_rxd_to_vlan_tci(mb, &rxdp[j]);
 477                         pkt_flags = i40e_rxd_status_to_pkt_flags(qword1);
 478                         pkt_flags |= i40e_rxd_error_to_pkt_flags(qword1);
 479                         mb->packet_type =
 480                                 ptype_tbl[(uint8_t)((qword1 &
 481                                 I40E_RXD_QW1_PTYPE_MASK) >>
 482                                 I40E_RXD_QW1_PTYPE_SHIFT)];
 483                         if (pkt_flags & PKT_RX_RSS_HASH)
 484                                 mb->hash.rss = rte_le_to_cpu_32(\
 485                                         rxdp[j].wb.qword0.hi_dword.rss);
 486                         if (pkt_flags & PKT_RX_FDIR)
 487                                 pkt_flags |= i40e_rxd_build_fdir(&rxdp[j], mb);
 488
 489 #ifdef RTE_LIBRTE_IEEE1588
 490                         pkt_flags |= i40e_get_iee15888_flags(mb, qword1);
 491 #endif
 492                         mb->ol_flags |= pkt_flags;
 493
 494                 }
 495
 496                 for (j = 0; j < I40E_LOOK_AHEAD; j++)
 497                         rxq->rx_stage[i + j] = rxep[j].mbuf;
 498
 499                 if (nb_dd != I40E_LOOK_AHEAD)
 500                         break;
 501         }
 502
 503         /* Clear software ring entries */
 504         for (i = 0; i < nb_rx; i++)
 505                 rxq->sw_ring[rxq->rx_tail + i].mbuf = NULL;
 506
 507         return nb_rx;
 508 }
 509
 510 static inline uint16_t
 511 i40e_rx_fill_from_stage(struct i40e_rx_queue *rxq,
 512                         struct rte_mbuf **rx_pkts,
 513                         uint16_t nb_pkts)
 514 {
 515         uint16_t i;
 516         struct rte_mbuf **stage = &rxq->rx_stage[rxq->rx_next_avail];
 517
 518         nb_pkts = (uint16_t)RTE_MIN(nb_pkts, rxq->rx_nb_avail);
 519
 520         for (i = 0; i < nb_pkts; i++)
 521                 rx_pkts[i] = stage[i];
 522
 523         rxq->rx_nb_avail = (uint16_t)(rxq->rx_nb_avail - nb_pkts);
 524         rxq->rx_next_avail = (uint16_t)(rxq->rx_next_avail + nb_pkts);
 525
 526         return nb_pkts;
 527 }
 528
 529 static inline int
 530 i40e_rx_alloc_bufs(struct i40e_rx_queue *rxq)
 531 {
 532         volatile union i40e_rx_desc *rxdp;
 533         struct i40e_rx_entry *rxep;
 534         struct rte_mbuf *mb;
 535         uint16_t alloc_idx, i;
 536         uint64_t dma_addr;
 537         int diag;
 538
 539         /* Allocate buffers in bulk */
 540         alloc_idx = (uint16_t)(rxq->rx_free_trigger -
 541                                 (rxq->rx_free_thresh - 1));
 542         rxep = &(rxq->sw_ring[alloc_idx]);
 543         diag = rte_mempool_get_bulk(rxq->mp, (void *)rxep,
 544                                         rxq->rx_free_thresh);
 545         if (unlikely(diag != 0)) {
 546                 PMD_DRV_LOG(ERR, "Failed to get mbufs in bulk");
 547                 return -ENOMEM;
 548         }
 549
 550         rxdp = &rxq->rx_ring[alloc_idx];
 551         for (i = 0; i < rxq->rx_free_thresh; i++) {
 552                 if (likely(i < (rxq->rx_free_thresh - 1)))
 553                         /* Prefetch next mbuf */
 554                         rte_prefetch0(rxep[i + 1].mbuf);
 555
 556                 mb = rxep[i].mbuf;
 557                 rte_mbuf_refcnt_set(mb, 1);
 558                 mb->next = NULL;
 559                 mb->data_off = RTE_PKTMBUF_HEADROOM;
 560                 mb->nb_segs = 1;
 561                 mb->port = rxq->port_id;
 562                 dma_addr = rte_cpu_to_le_64(\
 563                         rte_mbuf_data_iova_default(mb));
 564                 rxdp[i].read.hdr_addr = 0;
 565                 rxdp[i].read.pkt_addr = dma_addr;
 566         }
 567
 568         /* Update rx tail regsiter */
 569         rte_wmb();
 570         I40E_PCI_REG_WRITE_RELAXED(rxq->qrx_tail, rxq->rx_free_trigger);
 571
 572         rxq->rx_free_trigger =
 573                 (uint16_t)(rxq->rx_free_trigger + rxq->rx_free_thresh);
 574         if (rxq->rx_free_trigger >= rxq->nb_rx_desc)
 575                 rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1);
 576
 577         return 0;
 578 }
 579
 580 static inline uint16_t
 581 rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 582 {
 583         struct i40e_rx_queue *rxq = (struct i40e_rx_queue *)rx_queue;
 584         struct rte_eth_dev *dev;
 585         uint16_t nb_rx = 0;
 586
 587         if (!nb_pkts)
 588                 return 0;
 589
 590         if (rxq->rx_nb_avail)
 591                 return i40e_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
 592
 593         nb_rx = (uint16_t)i40e_rx_scan_hw_ring(rxq);
 594         rxq->rx_next_avail = 0;
 595         rxq->rx_nb_avail = nb_rx;
 596         rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_rx);
 597
 598         if (rxq->rx_tail > rxq->rx_free_trigger) {
 599                 if (i40e_rx_alloc_bufs(rxq) != 0) {
 600                         uint16_t i, j;
 601
 602                         dev = I40E_VSI_TO_ETH_DEV(rxq->vsi);
 603                         dev->data->rx_mbuf_alloc_failed +=
 604                                 rxq->rx_free_thresh;
 605
 606                         rxq->rx_nb_avail = 0;
 607                         rxq->rx_tail = (uint16_t)(rxq->rx_tail - nb_rx);
 608                         for (i = 0, j = rxq->rx_tail; i < nb_rx; i++, j++)
 609                                 rxq->sw_ring[j].mbuf = rxq->rx_stage[i];
 610
 611                         return 0;
 612                 }
 613         }
 614
 615         if (rxq->rx_tail >= rxq->nb_rx_desc)
 616                 rxq->rx_tail = 0;
 617
 618         if (rxq->rx_nb_avail)
 619                 return i40e_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
 620
 621         return 0;
 622 }
 623
 624 static uint16_t
 625 i40e_recv_pkts_bulk_alloc(void *rx_queue,
 626                           struct rte_mbuf **rx_pkts,
 627                           uint16_t nb_pkts)
 628 {
 629         uint16_t nb_rx = 0, n, count;
 630
 631         if (unlikely(nb_pkts == 0))
 632                 return 0;
 633
 634         if (likely(nb_pkts <= RTE_PMD_I40E_RX_MAX_BURST))
 635                 return rx_recv_pkts(rx_queue, rx_pkts, nb_pkts);
 636
 637         while (nb_pkts) {
 638                 n = RTE_MIN(nb_pkts, RTE_PMD_I40E_RX_MAX_BURST);
 639                 count = rx_recv_pkts(rx_queue, &rx_pkts[nb_rx], n);
 640                 nb_rx = (uint16_t)(nb_rx + count);
 641                 nb_pkts = (uint16_t)(nb_pkts - count);
 642                 if (count < n)
 643                         break;
 644         }
 645
 646         return nb_rx;
 647 }
 648 #else
 649 static uint16_t
 650 i40e_recv_pkts_bulk_alloc(void __rte_unused *rx_queue,
 651                           struct rte_mbuf __rte_unused **rx_pkts,
 652                           uint16_t __rte_unused nb_pkts)
 653 {
 654         return 0;
 655 }
 656 #endif /* RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC */
 657
 658 uint16_t
 659 i40e_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 660 {
 661         struct i40e_rx_queue *rxq;
 662         volatile union i40e_rx_desc *rx_ring;
 663         volatile union i40e_rx_desc *rxdp;
 664         union i40e_rx_desc rxd;
 665         struct i40e_rx_entry *sw_ring;
 666         struct i40e_rx_entry *rxe;
 667         struct rte_eth_dev *dev;
 668         struct rte_mbuf *rxm;
 669         struct rte_mbuf *nmb;
 670         uint16_t nb_rx;
 671         uint32_t rx_status;
 672         uint64_t qword1;
 673         uint16_t rx_packet_len;
 674         uint16_t rx_id, nb_hold;
 675         uint64_t dma_addr;
 676         uint64_t pkt_flags;
 677         uint32_t *ptype_tbl;
 678
 679         nb_rx = 0;
 680         nb_hold = 0;
 681         rxq = rx_queue;
 682         rx_id = rxq->rx_tail;
 683         rx_ring = rxq->rx_ring;
 684         sw_ring = rxq->sw_ring;
 685         ptype_tbl = rxq->vsi->adapter->ptype_tbl;
 686
 687         while (nb_rx < nb_pkts) {
 688                 rxdp = &rx_ring[rx_id];
 689                 qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len);
 690                 rx_status = (qword1 & I40E_RXD_QW1_STATUS_MASK)
 691                                 >> I40E_RXD_QW1_STATUS_SHIFT;
 692
 693                 /* Check the DD bit first */
 694                 if (!(rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)))
 695                         break;
 696
 697                 nmb = rte_mbuf_raw_alloc(rxq->mp);
 698                 if (unlikely(!nmb)) {
 699                         dev = I40E_VSI_TO_ETH_DEV(rxq->vsi);
 700                         dev->data->rx_mbuf_alloc_failed++;
 701                         break;
 702                 }
 703
 704                 rxd = *rxdp;
 705                 nb_hold++;
 706                 rxe = &sw_ring[rx_id];
 707                 rx_id++;
 708                 if (unlikely(rx_id == rxq->nb_rx_desc))
 709                         rx_id = 0;
 710
 711                 /* Prefetch next mbuf */
 712                 rte_prefetch0(sw_ring[rx_id].mbuf);
 713
 714                 /**
 715                  * When next RX descriptor is on a cache line boundary,
 716                  * prefetch the next 4 RX descriptors and next 8 pointers
 717                  * to mbufs.
 718                  */
 719                 if ((rx_id & 0x3) == 0) {
 720                         rte_prefetch0(&rx_ring[rx_id]);
 721                         rte_prefetch0(&sw_ring[rx_id]);
 722                 }
 723                 rxm = rxe->mbuf;
 724                 rxe->mbuf = nmb;
 725                 dma_addr =
 726                         rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
 727                 rxdp->read.hdr_addr = 0;
 728                 rxdp->read.pkt_addr = dma_addr;
 729
 730                 rx_packet_len = ((qword1 & I40E_RXD_QW1_LENGTH_PBUF_MASK) >>
 731                                 I40E_RXD_QW1_LENGTH_PBUF_SHIFT) - rxq->crc_len;
 732
 733                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
 734                 rte_prefetch0(RTE_PTR_ADD(rxm->buf_addr, RTE_PKTMBUF_HEADROOM));
 735                 rxm->nb_segs = 1;
 736                 rxm->next = NULL;
 737                 rxm->pkt_len = rx_packet_len;
 738                 rxm->data_len = rx_packet_len;
 739                 rxm->port = rxq->port_id;
 740                 rxm->ol_flags = 0;
 741                 i40e_rxd_to_vlan_tci(rxm, &rxd);
 742                 pkt_flags = i40e_rxd_status_to_pkt_flags(qword1);
 743                 pkt_flags |= i40e_rxd_error_to_pkt_flags(qword1);
 744                 rxm->packet_type =
 745                         ptype_tbl[(uint8_t)((qword1 &
 746                         I40E_RXD_QW1_PTYPE_MASK) >> I40E_RXD_QW1_PTYPE_SHIFT)];
 747                 if (pkt_flags & PKT_RX_RSS_HASH)
 748                         rxm->hash.rss =
 749                                 rte_le_to_cpu_32(rxd.wb.qword0.hi_dword.rss);
 750                 if (pkt_flags & PKT_RX_FDIR)
 751                         pkt_flags |= i40e_rxd_build_fdir(&rxd, rxm);
 752
 753 #ifdef RTE_LIBRTE_IEEE1588
 754                 pkt_flags |= i40e_get_iee15888_flags(rxm, qword1);
 755 #endif
 756                 rxm->ol_flags |= pkt_flags;
 757
 758                 rx_pkts[nb_rx++] = rxm;
 759         }
 760         rxq->rx_tail = rx_id;
 761
 762         /**
 763          * If the number of free RX descriptors is greater than the RX free
 764          * threshold of the queue, advance the receive tail register of queue.
 765          * Update that register with the value of the last processed RX
 766          * descriptor minus 1.
 767          */
 768         nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold);
 769         if (nb_hold > rxq->rx_free_thresh) {
 770                 rx_id = (uint16_t) ((rx_id == 0) ?
 771                         (rxq->nb_rx_desc - 1) : (rx_id - 1));
 772                 I40E_PCI_REG_WRITE(rxq->qrx_tail, rx_id);
 773                 nb_hold = 0;
 774         }
 775         rxq->nb_rx_hold = nb_hold;
 776
 777         return nb_rx;
 778 }
 779
 780 uint16_t
 781 i40e_recv_scattered_pkts(void *rx_queue,
 782                          struct rte_mbuf **rx_pkts,
 783                          uint16_t nb_pkts)
 784 {
 785         struct i40e_rx_queue *rxq = rx_queue;
 786         volatile union i40e_rx_desc *rx_ring = rxq->rx_ring;
 787         volatile union i40e_rx_desc *rxdp;
 788         union i40e_rx_desc rxd;
 789         struct i40e_rx_entry *sw_ring = rxq->sw_ring;
 790         struct i40e_rx_entry *rxe;
 791         struct rte_mbuf *first_seg = rxq->pkt_first_seg;
 792         struct rte_mbuf *last_seg = rxq->pkt_last_seg;
 793         struct rte_mbuf *nmb, *rxm;
 794         uint16_t rx_id = rxq->rx_tail;
 795         uint16_t nb_rx = 0, nb_hold = 0, rx_packet_len;
 796         struct rte_eth_dev *dev;
 797         uint32_t rx_status;
 798         uint64_t qword1;
 799         uint64_t dma_addr;
 800         uint64_t pkt_flags;
 801         uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl;
 802
 803         while (nb_rx < nb_pkts) {
 804                 rxdp = &rx_ring[rx_id];
 805                 qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len);
 806                 rx_status = (qword1 & I40E_RXD_QW1_STATUS_MASK) >>
 807                                         I40E_RXD_QW1_STATUS_SHIFT;
 808
 809                 /* Check the DD bit */
 810                 if (!(rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)))
 811                         break;
 812
 813                 nmb = rte_mbuf_raw_alloc(rxq->mp);
 814                 if (unlikely(!nmb)) {
 815                         dev = I40E_VSI_TO_ETH_DEV(rxq->vsi);
 816                         dev->data->rx_mbuf_alloc_failed++;
 817                         break;
 818                 }
 819
 820                 rxd = *rxdp;
 821                 nb_hold++;
 822                 rxe = &sw_ring[rx_id];
 823                 rx_id++;
 824                 if (rx_id == rxq->nb_rx_desc)
 825                         rx_id = 0;
 826
 827                 /* Prefetch next mbuf */
 828                 rte_prefetch0(sw_ring[rx_id].mbuf);
 829
 830                 /**
 831                  * When next RX descriptor is on a cache line boundary,
 832                  * prefetch the next 4 RX descriptors and next 8 pointers
 833                  * to mbufs.
 834                  */
 835                 if ((rx_id & 0x3) == 0) {
 836                         rte_prefetch0(&rx_ring[rx_id]);
 837                         rte_prefetch0(&sw_ring[rx_id]);
 838                 }
 839
 840                 rxm = rxe->mbuf;
 841                 rxe->mbuf = nmb;
 842                 dma_addr =
 843                         rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
 844
 845                 /* Set data buffer address and data length of the mbuf */
 846                 rxdp->read.hdr_addr = 0;
 847                 rxdp->read.pkt_addr = dma_addr;
 848                 rx_packet_len = (qword1 & I40E_RXD_QW1_LENGTH_PBUF_MASK) >>
 849                                         I40E_RXD_QW1_LENGTH_PBUF_SHIFT;
 850                 rxm->data_len = rx_packet_len;
 851                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
 852
 853                 /**
 854                  * If this is the first buffer of the received packet, set the
 855                  * pointer to the first mbuf of the packet and initialize its
 856                  * context. Otherwise, update the total length and the number
 857                  * of segments of the current scattered packet, and update the
 858                  * pointer to the last mbuf of the current packet.
 859                  */
 860                 if (!first_seg) {
 861                         first_seg = rxm;
 862                         first_seg->nb_segs = 1;
 863                         first_seg->pkt_len = rx_packet_len;
 864                 } else {
 865                         first_seg->pkt_len =
 866                                 (uint16_t)(first_seg->pkt_len +
 867                                                 rx_packet_len);
 868                         first_seg->nb_segs++;
 869                         last_seg->next = rxm;
 870                 }
 871
 872                 /**
 873                  * If this is not the last buffer of the received packet,
 874                  * update the pointer to the last mbuf of the current scattered
 875                  * packet and continue to parse the RX ring.
 876                  */
 877                 if (!(rx_status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT))) {
 878                         last_seg = rxm;
 879                         continue;
 880                 }
 881
 882                 /**
 883                  * This is the last buffer of the received packet. If the CRC
 884                  * is not stripped by the hardware:
 885                  *  - Subtract the CRC length from the total packet length.
 886                  *  - If the last buffer only contains the whole CRC or a part
 887                  *  of it, free the mbuf associated to the last buffer. If part
 888                  *  of the CRC is also contained in the previous mbuf, subtract
 889                  *  the length of that CRC part from the data length of the
 890                  *  previous mbuf.
 891                  */
 892                 rxm->next = NULL;
 893                 if (unlikely(rxq->crc_len > 0)) {
 894                         first_seg->pkt_len -= ETHER_CRC_LEN;
 895                         if (rx_packet_len <= ETHER_CRC_LEN) {
 896                                 rte_pktmbuf_free_seg(rxm);
 897                                 first_seg->nb_segs--;
 898                                 last_seg->data_len =
 899                                         (uint16_t)(last_seg->data_len -
 900                                         (ETHER_CRC_LEN - rx_packet_len));
 901                                 last_seg->next = NULL;
 902                         } else
 903                                 rxm->data_len = (uint16_t)(rx_packet_len -
 904                                                                 ETHER_CRC_LEN);
 905                 }
 906
 907                 first_seg->port = rxq->port_id;
 908                 first_seg->ol_flags = 0;
 909                 i40e_rxd_to_vlan_tci(first_seg, &rxd);
 910                 pkt_flags = i40e_rxd_status_to_pkt_flags(qword1);
 911                 pkt_flags |= i40e_rxd_error_to_pkt_flags(qword1);
 912                 first_seg->packet_type =
 913                         ptype_tbl[(uint8_t)((qword1 &
 914                         I40E_RXD_QW1_PTYPE_MASK) >> I40E_RXD_QW1_PTYPE_SHIFT)];
 915                 if (pkt_flags & PKT_RX_RSS_HASH)
 916                         first_seg->hash.rss =
 917                                 rte_le_to_cpu_32(rxd.wb.qword0.hi_dword.rss);
 918                 if (pkt_flags & PKT_RX_FDIR)
 919                         pkt_flags |= i40e_rxd_build_fdir(&rxd, first_seg);
 920
 921 #ifdef RTE_LIBRTE_IEEE1588
 922                 pkt_flags |= i40e_get_iee15888_flags(first_seg, qword1);
 923 #endif
 924                 first_seg->ol_flags |= pkt_flags;
 925
 926                 /* Prefetch data of first segment, if configured to do so. */
 927                 rte_prefetch0(RTE_PTR_ADD(first_seg->buf_addr,
 928                         first_seg->data_off));
 929                 rx_pkts[nb_rx++] = first_seg;
 930                 first_seg = NULL;
 931         }
 932
 933         /* Record index of the next RX descriptor to probe. */
 934         rxq->rx_tail = rx_id;
 935         rxq->pkt_first_seg = first_seg;
 936         rxq->pkt_last_seg = last_seg;
 937
 938         /**
 939          * If the number of free RX descriptors is greater than the RX free
 940          * threshold of the queue, advance the Receive Descriptor Tail (RDT)
 941          * register. Update the RDT with the value of the last processed RX
 942          * descriptor minus 1, to guarantee that the RDT register is never
 943          * equal to the RDH register, which creates a "full" ring situtation
 944          * from the hardware point of view.
 945          */
 946         nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold);
 947         if (nb_hold > rxq->rx_free_thresh) {
 948                 rx_id = (uint16_t)(rx_id == 0 ?
 949                         (rxq->nb_rx_desc - 1) : (rx_id - 1));
 950                 I40E_PCI_REG_WRITE(rxq->qrx_tail, rx_id);
 951                 nb_hold = 0;
 952         }
 953         rxq->nb_rx_hold = nb_hold;
 954
 955         return nb_rx;
 956 }
 957
 958 /* Check if the context descriptor is needed for TX offloading */
 959 static inline uint16_t
 960 i40e_calc_context_desc(uint64_t flags)
 961 {
 962         static uint64_t mask = PKT_TX_OUTER_IP_CKSUM |
 963                 PKT_TX_TCP_SEG |
 964                 PKT_TX_QINQ_PKT |
 965                 PKT_TX_TUNNEL_MASK;
 966
 967 #ifdef RTE_LIBRTE_IEEE1588
 968         mask |= PKT_TX_IEEE1588_TMST;
 969 #endif
 970
 971         return (flags & mask) ? 1 : 0;
 972 }
 973
 974 /* set i40e TSO context descriptor */
 975 static inline uint64_t
 976 i40e_set_tso_ctx(struct rte_mbuf *mbuf, union i40e_tx_offload tx_offload)
 977 {
 978         uint64_t ctx_desc = 0;
 979         uint32_t cd_cmd, hdr_len, cd_tso_len;
 980
 981         if (!tx_offload.l4_len) {
 982                 PMD_DRV_LOG(DEBUG, "L4 length set to 0");
 983                 return ctx_desc;
 984         }
 985
 986         /**
 987          * in case of non tunneling packet, the outer_l2_len and
 988          * outer_l3_len must be 0.
 989          */
 990         hdr_len = tx_offload.outer_l2_len +
 991                 tx_offload.outer_l3_len +
 992                 tx_offload.l2_len +
 993                 tx_offload.l3_len +
 994                 tx_offload.l4_len;
 995
 996         cd_cmd = I40E_TX_CTX_DESC_TSO;
 997         cd_tso_len = mbuf->pkt_len - hdr_len;
 998         ctx_desc |= ((uint64_t)cd_cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) |
 999                 ((uint64_t)cd_tso_len <<
1000                  I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) |
1001                 ((uint64_t)mbuf->tso_segsz <<
1002                  I40E_TXD_CTX_QW1_MSS_SHIFT);
1003
1004         return ctx_desc;
1005 }
1006
1007 uint16_t
1008 i40e_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
1009 {
1010         struct i40e_tx_queue *txq;
1011         struct i40e_tx_entry *sw_ring;
1012         struct i40e_tx_entry *txe, *txn;
1013         volatile struct i40e_tx_desc *txd;
1014         volatile struct i40e_tx_desc *txr;
1015         struct rte_mbuf *tx_pkt;
1016         struct rte_mbuf *m_seg;
1017         uint32_t cd_tunneling_params;
1018         uint16_t tx_id;
1019         uint16_t nb_tx;
1020         uint32_t td_cmd;
1021         uint32_t td_offset;
1022         uint32_t td_tag;
1023         uint64_t ol_flags;
1024         uint16_t nb_used;
1025         uint16_t nb_ctx;
1026         uint16_t tx_last;
1027         uint16_t slen;
1028         uint64_t buf_dma_addr;
1029         union i40e_tx_offload tx_offload = {0};
1030
1031         txq = tx_queue;
1032         sw_ring = txq->sw_ring;
1033         txr = txq->tx_ring;
1034         tx_id = txq->tx_tail;
1035         txe = &sw_ring[tx_id];
1036
1037         /* Check if the descriptor ring needs to be cleaned. */
1038         if (txq->nb_tx_free < txq->tx_free_thresh)
1039                 i40e_xmit_cleanup(txq);
1040
1041         for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
1042                 td_cmd = 0;
1043                 td_tag = 0;
1044                 td_offset = 0;
1045
1046                 tx_pkt = *tx_pkts++;
1047                 RTE_MBUF_PREFETCH_TO_FREE(txe->mbuf);
1048
1049                 ol_flags = tx_pkt->ol_flags;
1050                 tx_offload.l2_len = tx_pkt->l2_len;
1051                 tx_offload.l3_len = tx_pkt->l3_len;
1052                 tx_offload.outer_l2_len = tx_pkt->outer_l2_len;
1053                 tx_offload.outer_l3_len = tx_pkt->outer_l3_len;
1054                 tx_offload.l4_len = tx_pkt->l4_len;
1055                 tx_offload.tso_segsz = tx_pkt->tso_segsz;
1056
1057                 /* Calculate the number of context descriptors needed. */
1058                 nb_ctx = i40e_calc_context_desc(ol_flags);
1059
1060                 /**
1061                  * The number of descriptors that must be allocated for
1062                  * a packet equals to the number of the segments of that
1063                  * packet plus 1 context descriptor if needed.
1064                  */
1065                 nb_used = (uint16_t)(tx_pkt->nb_segs + nb_ctx);
1066                 tx_last = (uint16_t)(tx_id + nb_used - 1);
1067
1068                 /* Circular ring */
1069                 if (tx_last >= txq->nb_tx_desc)
1070                         tx_last = (uint16_t)(tx_last - txq->nb_tx_desc);
1071
1072                 if (nb_used > txq->nb_tx_free) {
1073                         if (i40e_xmit_cleanup(txq) != 0) {
1074                                 if (nb_tx == 0)
1075                                         return 0;
1076                                 goto end_of_tx;
1077                         }
1078                         if (unlikely(nb_used > txq->tx_rs_thresh)) {
1079                                 while (nb_used > txq->nb_tx_free) {
1080                                         if (i40e_xmit_cleanup(txq) != 0) {
1081                                                 if (nb_tx == 0)
1082                                                         return 0;
1083                                                 goto end_of_tx;
1084                                         }
1085                                 }
1086                         }
1087                 }
1088
1089                 /* Descriptor based VLAN insertion */
1090                 if (ol_flags & (PKT_TX_VLAN_PKT | PKT_TX_QINQ_PKT)) {
1091                         td_cmd |= I40E_TX_DESC_CMD_IL2TAG1;
1092                         td_tag = tx_pkt->vlan_tci;
1093                 }
1094
1095                 /* Always enable CRC offload insertion */
1096                 td_cmd |= I40E_TX_DESC_CMD_ICRC;
1097
1098                 /* Fill in tunneling parameters if necessary */
1099                 cd_tunneling_params = 0;
1100                 if (ol_flags & PKT_TX_TUNNEL_MASK)
1101                         i40e_parse_tunneling_params(ol_flags, tx_offload,
1102                                                     &cd_tunneling_params);
1103                 /* Enable checksum offloading */
1104                 if (ol_flags & I40E_TX_CKSUM_OFFLOAD_MASK)
1105                         i40e_txd_enable_checksum(ol_flags, &td_cmd,
1106                                                  &td_offset, tx_offload);
1107
1108                 if (nb_ctx) {
1109                         /* Setup TX context descriptor if required */
1110                         volatile struct i40e_tx_context_desc *ctx_txd =
1111                                 (volatile struct i40e_tx_context_desc *)\
1112                                                         &txr[tx_id];
1113                         uint16_t cd_l2tag2 = 0;
1114                         uint64_t cd_type_cmd_tso_mss =
1115                                 I40E_TX_DESC_DTYPE_CONTEXT;
1116
1117                         txn = &sw_ring[txe->next_id];
1118                         RTE_MBUF_PREFETCH_TO_FREE(txn->mbuf);
1119                         if (txe->mbuf != NULL) {
1120                                 rte_pktmbuf_free_seg(txe->mbuf);
1121                                 txe->mbuf = NULL;
1122                         }
1123
1124                         /* TSO enabled means no timestamp */
1125                         if (ol_flags & PKT_TX_TCP_SEG)
1126                                 cd_type_cmd_tso_mss |=
1127                                         i40e_set_tso_ctx(tx_pkt, tx_offload);
1128                         else {
1129 #ifdef RTE_LIBRTE_IEEE1588
1130                                 if (ol_flags & PKT_TX_IEEE1588_TMST)
1131                                         cd_type_cmd_tso_mss |=
1132                                                 ((uint64_t)I40E_TX_CTX_DESC_TSYN <<
1133                                                  I40E_TXD_CTX_QW1_CMD_SHIFT);
1134 #endif
1135                         }
1136
1137                         ctx_txd->tunneling_params =
1138                                 rte_cpu_to_le_32(cd_tunneling_params);
1139                         if (ol_flags & PKT_TX_QINQ_PKT) {
1140                                 cd_l2tag2 = tx_pkt->vlan_tci_outer;
1141                                 cd_type_cmd_tso_mss |=
1142                                         ((uint64_t)I40E_TX_CTX_DESC_IL2TAG2 <<
1143                                                 I40E_TXD_CTX_QW1_CMD_SHIFT);
1144                         }
1145                         ctx_txd->l2tag2 = rte_cpu_to_le_16(cd_l2tag2);
1146                         ctx_txd->type_cmd_tso_mss =
1147                                 rte_cpu_to_le_64(cd_type_cmd_tso_mss);
1148
1149                         PMD_TX_LOG(DEBUG, "mbuf: %p, TCD[%u]:\n"
1150                                 "tunneling_params: %#x;\n"
1151                                 "l2tag2: %#hx;\n"
1152                                 "rsvd: %#hx;\n"
1153                                 "type_cmd_tso_mss: %#"PRIx64";\n",
1154                                 tx_pkt, tx_id,
1155                                 ctx_txd->tunneling_params,
1156                                 ctx_txd->l2tag2,
1157                                 ctx_txd->rsvd,
1158                                 ctx_txd->type_cmd_tso_mss);
1159
1160                         txe->last_id = tx_last;
1161                         tx_id = txe->next_id;
1162                         txe = txn;
1163                 }
1164
1165                 m_seg = tx_pkt;
1166                 do {
1167                         txd = &txr[tx_id];
1168                         txn = &sw_ring[txe->next_id];
1169
1170                         if (txe->mbuf)
1171                                 rte_pktmbuf_free_seg(txe->mbuf);
1172                         txe->mbuf = m_seg;
1173
1174                         /* Setup TX Descriptor */
1175                         slen = m_seg->data_len;
1176                         buf_dma_addr = rte_mbuf_data_iova(m_seg);
1177
1178                         PMD_TX_LOG(DEBUG, "mbuf: %p, TDD[%u]:\n"
1179                                 "buf_dma_addr: %#"PRIx64";\n"
1180                                 "td_cmd: %#x;\n"
1181                                 "td_offset: %#x;\n"
1182                                 "td_len: %u;\n"
1183                                 "td_tag: %#x;\n",
1184                                 tx_pkt, tx_id, buf_dma_addr,
1185                                 td_cmd, td_offset, slen, td_tag);
1186
1187                         txd->buffer_addr = rte_cpu_to_le_64(buf_dma_addr);
1188                         txd->cmd_type_offset_bsz = i40e_build_ctob(td_cmd,
1189                                                 td_offset, slen, td_tag);
1190                         txe->last_id = tx_last;
1191                         tx_id = txe->next_id;
1192                         txe = txn;
1193                         m_seg = m_seg->next;
1194                 } while (m_seg != NULL);
1195
1196                 /* The last packet data descriptor needs End Of Packet (EOP) */
1197                 td_cmd |= I40E_TX_DESC_CMD_EOP;
1198                 txq->nb_tx_used = (uint16_t)(txq->nb_tx_used + nb_used);
1199                 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_used);
1200
1201                 if (txq->nb_tx_used >= txq->tx_rs_thresh) {
1202                         PMD_TX_FREE_LOG(DEBUG,
1203                                         "Setting RS bit on TXD id="
1204                                         "%4u (port=%d queue=%d)",
1205                                         tx_last, txq->port_id, txq->queue_id);
1206
1207                         td_cmd |= I40E_TX_DESC_CMD_RS;
1208
1209                         /* Update txq RS bit counters */
1210                         txq->nb_tx_used = 0;
1211                 }
1212
1213                 txd->cmd_type_offset_bsz |=
1214                         rte_cpu_to_le_64(((uint64_t)td_cmd) <<
1215                                         I40E_TXD_QW1_CMD_SHIFT);
1216         }
1217
1218 end_of_tx:
1219         rte_wmb();
1220
1221         PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u",
1222                    (unsigned) txq->port_id, (unsigned) txq->queue_id,
1223                    (unsigned) tx_id, (unsigned) nb_tx);
1224
1225         I40E_PCI_REG_WRITE_RELAXED(txq->qtx_tail, tx_id);
1226         txq->tx_tail = tx_id;
1227
1228         return nb_tx;
1229 }
1230
1231 static __rte_always_inline int
1232 i40e_tx_free_bufs(struct i40e_tx_queue *txq)
1233 {
1234         struct i40e_tx_entry *txep;
1235         uint16_t i;
1236
1237         if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &
1238                         rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) !=
1239                         rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE))
1240                 return 0;
1241
1242         txep = &(txq->sw_ring[txq->tx_next_dd - (txq->tx_rs_thresh - 1)]);
1243
1244         for (i = 0; i < txq->tx_rs_thresh; i++)
1245                 rte_prefetch0((txep + i)->mbuf);
1246
1247         if (txq->txq_flags & (uint32_t)ETH_TXQ_FLAGS_NOREFCOUNT) {
1248                 for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) {
1249                         rte_mempool_put(txep->mbuf->pool, txep->mbuf);
1250                         txep->mbuf = NULL;
1251                 }
1252         } else {
1253                 for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) {
1254                         rte_pktmbuf_free_seg(txep->mbuf);
1255                         txep->mbuf = NULL;
1256                 }
1257         }
1258
1259         txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh);
1260         txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh);
1261         if (txq->tx_next_dd >= txq->nb_tx_desc)
1262                 txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
1263
1264         return txq->tx_rs_thresh;
1265 }
1266
1267 /* Populate 4 descriptors with data from 4 mbufs */
1268 static inline void
1269 tx4(volatile struct i40e_tx_desc *txdp, struct rte_mbuf **pkts)
1270 {
1271         uint64_t dma_addr;
1272         uint32_t i;
1273
1274         for (i = 0; i < 4; i++, txdp++, pkts++) {
1275                 dma_addr = rte_mbuf_data_iova(*pkts);
1276                 txdp->buffer_addr = rte_cpu_to_le_64(dma_addr);
1277                 txdp->cmd_type_offset_bsz =
1278                         i40e_build_ctob((uint32_t)I40E_TD_CMD, 0,
1279                                         (*pkts)->data_len, 0);
1280         }
1281 }
1282
1283 /* Populate 1 descriptor with data from 1 mbuf */
1284 static inline void
1285 tx1(volatile struct i40e_tx_desc *txdp, struct rte_mbuf **pkts)
1286 {
1287         uint64_t dma_addr;
1288
1289         dma_addr = rte_mbuf_data_iova(*pkts);
1290         txdp->buffer_addr = rte_cpu_to_le_64(dma_addr);
1291         txdp->cmd_type_offset_bsz =
1292                 i40e_build_ctob((uint32_t)I40E_TD_CMD, 0,
1293                                 (*pkts)->data_len, 0);
1294 }
1295
1296 /* Fill hardware descriptor ring with mbuf data */
1297 static inline void
1298 i40e_tx_fill_hw_ring(struct i40e_tx_queue *txq,
1299                      struct rte_mbuf **pkts,
1300                      uint16_t nb_pkts)
1301 {
1302         volatile struct i40e_tx_desc *txdp = &(txq->tx_ring[txq->tx_tail]);
1303         struct i40e_tx_entry *txep = &(txq->sw_ring[txq->tx_tail]);
1304         const int N_PER_LOOP = 4;
1305         const int N_PER_LOOP_MASK = N_PER_LOOP - 1;
1306         int mainpart, leftover;
1307         int i, j;
1308
1309         mainpart = (nb_pkts & ((uint32_t) ~N_PER_LOOP_MASK));
1310         leftover = (nb_pkts & ((uint32_t)  N_PER_LOOP_MASK));
1311         for (i = 0; i < mainpart; i += N_PER_LOOP) {
1312                 for (j = 0; j < N_PER_LOOP; ++j) {
1313                         (txep + i + j)->mbuf = *(pkts + i + j);
1314                 }
1315                 tx4(txdp + i, pkts + i);
1316         }
1317         if (unlikely(leftover > 0)) {
1318                 for (i = 0; i < leftover; ++i) {
1319                         (txep + mainpart + i)->mbuf = *(pkts + mainpart + i);
1320                         tx1(txdp + mainpart + i, pkts + mainpart + i);
1321                 }
1322         }
1323 }
1324
1325 static inline uint16_t
1326 tx_xmit_pkts(struct i40e_tx_queue *txq,
1327              struct rte_mbuf **tx_pkts,
1328              uint16_t nb_pkts)
1329 {
1330         volatile struct i40e_tx_desc *txr = txq->tx_ring;
1331         uint16_t n = 0;
1332
1333         /**
1334          * Begin scanning the H/W ring for done descriptors when the number
1335          * of available descriptors drops below tx_free_thresh. For each done
1336          * descriptor, free the associated buffer.
1337          */
1338         if (txq->nb_tx_free < txq->tx_free_thresh)
1339                 i40e_tx_free_bufs(txq);
1340
1341         /* Use available descriptor only */
1342         nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts);
1343         if (unlikely(!nb_pkts))
1344                 return 0;
1345
1346         txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts);
1347         if ((txq->tx_tail + nb_pkts) > txq->nb_tx_desc) {
1348                 n = (uint16_t)(txq->nb_tx_desc - txq->tx_tail);
1349                 i40e_tx_fill_hw_ring(txq, tx_pkts, n);
1350                 txr[txq->tx_next_rs].cmd_type_offset_bsz |=
1351                         rte_cpu_to_le_64(((uint64_t)I40E_TX_DESC_CMD_RS) <<
1352                                                 I40E_TXD_QW1_CMD_SHIFT);
1353                 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
1354                 txq->tx_tail = 0;
1355         }
1356
1357         /* Fill hardware descriptor ring with mbuf data */
1358         i40e_tx_fill_hw_ring(txq, tx_pkts + n, (uint16_t)(nb_pkts - n));
1359         txq->tx_tail = (uint16_t)(txq->tx_tail + (nb_pkts - n));
1360
1361         /* Determin if RS bit needs to be set */
1362         if (txq->tx_tail > txq->tx_next_rs) {
1363                 txr[txq->tx_next_rs].cmd_type_offset_bsz |=
1364                         rte_cpu_to_le_64(((uint64_t)I40E_TX_DESC_CMD_RS) <<
1365                                                 I40E_TXD_QW1_CMD_SHIFT);
1366                 txq->tx_next_rs =
1367                         (uint16_t)(txq->tx_next_rs + txq->tx_rs_thresh);
1368                 if (txq->tx_next_rs >= txq->nb_tx_desc)
1369                         txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
1370         }
1371
1372         if (txq->tx_tail >= txq->nb_tx_desc)
1373                 txq->tx_tail = 0;
1374
1375         /* Update the tx tail register */
1376         rte_wmb();
1377         I40E_PCI_REG_WRITE_RELAXED(txq->qtx_tail, txq->tx_tail);
1378
1379         return nb_pkts;
1380 }
1381
1382 static uint16_t
1383 i40e_xmit_pkts_simple(void *tx_queue,
1384                       struct rte_mbuf **tx_pkts,
1385                       uint16_t nb_pkts)
1386 {
1387         uint16_t nb_tx = 0;
1388
1389         if (likely(nb_pkts <= I40E_TX_MAX_BURST))
1390                 return tx_xmit_pkts((struct i40e_tx_queue *)tx_queue,
1391                                                 tx_pkts, nb_pkts);
1392
1393         while (nb_pkts) {
1394                 uint16_t ret, num = (uint16_t)RTE_MIN(nb_pkts,
1395                                                 I40E_TX_MAX_BURST);
1396
1397                 ret = tx_xmit_pkts((struct i40e_tx_queue *)tx_queue,
1398                                                 &tx_pkts[nb_tx], num);
1399                 nb_tx = (uint16_t)(nb_tx + ret);
1400                 nb_pkts = (uint16_t)(nb_pkts - ret);
1401                 if (ret < num)
1402                         break;
1403         }
1404
1405         return nb_tx;
1406 }
1407
1408 static uint16_t
1409 i40e_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
1410                    uint16_t nb_pkts)
1411 {
1412         uint16_t nb_tx = 0;
1413         struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue;
1414
1415         while (nb_pkts) {
1416                 uint16_t ret, num;
1417
1418                 num = (uint16_t)RTE_MIN(nb_pkts, txq->tx_rs_thresh);
1419                 ret = i40e_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
1420                                                 num);
1421                 nb_tx += ret;
1422                 nb_pkts -= ret;
1423                 if (ret < num)
1424                         break;
1425         }
1426
1427         return nb_tx;
1428 }
1429
1430 /*********************************************************************
1431  *
1432  *  TX prep functions
1433  *
1434  **********************************************************************/
1435 uint16_t
1436 i40e_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts,
1437                 uint16_t nb_pkts)
1438 {
1439         int i, ret;
1440         uint64_t ol_flags;
1441         struct rte_mbuf *m;
1442
1443         for (i = 0; i < nb_pkts; i++) {
1444                 m = tx_pkts[i];
1445                 ol_flags = m->ol_flags;
1446
1447                 /* Check for m->nb_segs to not exceed the limits. */
1448                 if (!(ol_flags & PKT_TX_TCP_SEG)) {
1449                         if (m->nb_segs > I40E_TX_MAX_SEG ||
1450                             m->nb_segs > I40E_TX_MAX_MTU_SEG) {
1451                                 rte_errno = -EINVAL;
1452                                 return i;
1453                         }
1454                 } else if ((m->tso_segsz < I40E_MIN_TSO_MSS) ||
1455                                 (m->tso_segsz > I40E_MAX_TSO_MSS)) {
1456                         /* MSS outside the range (256B - 9674B) are considered
1457                          * malicious
1458                          */
1459                         rte_errno = -EINVAL;
1460                         return i;
1461                 }
1462
1463                 if (ol_flags & I40E_TX_OFFLOAD_NOTSUP_MASK) {
1464                         rte_errno = -ENOTSUP;
1465                         return i;
1466                 }
1467
1468 #ifdef RTE_LIBRTE_ETHDEV_DEBUG
1469                 ret = rte_validate_tx_offload(m);
1470                 if (ret != 0) {
1471                         rte_errno = ret;
1472                         return i;
1473                 }
1474 #endif
1475                 ret = rte_net_intel_cksum_prepare(m);
1476                 if (ret != 0) {
1477                         rte_errno = ret;
1478                         return i;
1479                 }
1480         }
1481         return i;
1482 }
1483
1484 /*
1485  * Find the VSI the queue belongs to. 'queue_idx' is the queue index
1486  * application used, which assume having sequential ones. But from driver's
1487  * perspective, it's different. For example, q0 belongs to FDIR VSI, q1-q64
1488  * to MAIN VSI, , q65-96 to SRIOV VSIs, q97-128 to VMDQ VSIs. For application
1489  * running on host, q1-64 and q97-128 can be used, total 96 queues. They can
1490  * use queue_idx from 0 to 95 to access queues, while real queue would be
1491  * different. This function will do a queue mapping to find VSI the queue
1492  * belongs to.
1493  */
1494 static struct i40e_vsi*
1495 i40e_pf_get_vsi_by_qindex(struct i40e_pf *pf, uint16_t queue_idx)
1496 {
1497         /* the queue in MAIN VSI range */
1498         if (queue_idx < pf->main_vsi->nb_qps)
1499                 return pf->main_vsi;
1500
1501         queue_idx -= pf->main_vsi->nb_qps;
1502
1503         /* queue_idx is greater than VMDQ VSIs range */
1504         if (queue_idx > pf->nb_cfg_vmdq_vsi * pf->vmdq_nb_qps - 1) {
1505                 PMD_INIT_LOG(ERR, "queue_idx out of range. VMDQ configured?");
1506                 return NULL;
1507         }
1508
1509         return pf->vmdq[queue_idx / pf->vmdq_nb_qps].vsi;
1510 }
1511
1512 static uint16_t
1513 i40e_get_queue_offset_by_qindex(struct i40e_pf *pf, uint16_t queue_idx)
1514 {
1515         /* the queue in MAIN VSI range */
1516         if (queue_idx < pf->main_vsi->nb_qps)
1517                 return queue_idx;
1518
1519         /* It's VMDQ queues */
1520         queue_idx -= pf->main_vsi->nb_qps;
1521
1522         if (pf->nb_cfg_vmdq_vsi)
1523                 return queue_idx % pf->vmdq_nb_qps;
1524         else {
1525                 PMD_INIT_LOG(ERR, "Fail to get queue offset");
1526                 return (uint16_t)(-1);
1527         }
1528 }
1529
1530 int
1531 i40e_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1532 {
1533         struct i40e_rx_queue *rxq;
1534         int err = -1;
1535         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1536
1537         PMD_INIT_FUNC_TRACE();
1538
1539         if (rx_queue_id < dev->data->nb_rx_queues) {
1540                 rxq = dev->data->rx_queues[rx_queue_id];
1541
1542                 err = i40e_alloc_rx_queue_mbufs(rxq);
1543                 if (err) {
1544                         PMD_DRV_LOG(ERR, "Failed to allocate RX queue mbuf");
1545                         return err;
1546                 }
1547
1548                 rte_wmb();
1549
1550                 /* Init the RX tail regieter. */
1551                 I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
1552
1553                 err = i40e_switch_rx_queue(hw, rxq->reg_idx, TRUE);
1554
1555                 if (err) {
1556                         PMD_DRV_LOG(ERR, "Failed to switch RX queue %u on",
1557                                     rx_queue_id);
1558
1559                         i40e_rx_queue_release_mbufs(rxq);
1560                         i40e_reset_rx_queue(rxq);
1561                 } else
1562                         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
1563         }
1564
1565         return err;
1566 }
1567
1568 int
1569 i40e_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1570 {
1571         struct i40e_rx_queue *rxq;
1572         int err;
1573         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1574
1575         if (rx_queue_id < dev->data->nb_rx_queues) {
1576                 rxq = dev->data->rx_queues[rx_queue_id];
1577
1578                 /*
1579                 * rx_queue_id is queue id application refers to, while
1580                 * rxq->reg_idx is the real queue index.
1581                 */
1582                 err = i40e_switch_rx_queue(hw, rxq->reg_idx, FALSE);
1583
1584                 if (err) {
1585                         PMD_DRV_LOG(ERR, "Failed to switch RX queue %u off",
1586                                     rx_queue_id);
1587                         return err;
1588                 }
1589                 i40e_rx_queue_release_mbufs(rxq);
1590                 i40e_reset_rx_queue(rxq);
1591                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
1592         }
1593
1594         return 0;
1595 }
1596
1597 int
1598 i40e_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
1599 {
1600         int err = -1;
1601         struct i40e_tx_queue *txq;
1602         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1603
1604         PMD_INIT_FUNC_TRACE();
1605
1606         if (tx_queue_id < dev->data->nb_tx_queues) {
1607                 txq = dev->data->tx_queues[tx_queue_id];
1608
1609                 /*
1610                 * tx_queue_id is queue id application refers to, while
1611                 * rxq->reg_idx is the real queue index.
1612                 */
1613                 err = i40e_switch_tx_queue(hw, txq->reg_idx, TRUE);
1614                 if (err)
1615                         PMD_DRV_LOG(ERR, "Failed to switch TX queue %u on",
1616                                     tx_queue_id);
1617                 else
1618                         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
1619         }
1620
1621         return err;
1622 }
1623
1624 int
1625 i40e_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
1626 {
1627         struct i40e_tx_queue *txq;
1628         int err;
1629         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1630
1631         if (tx_queue_id < dev->data->nb_tx_queues) {
1632                 txq = dev->data->tx_queues[tx_queue_id];
1633
1634                 /*
1635                 * tx_queue_id is queue id application refers to, while
1636                 * txq->reg_idx is the real queue index.
1637                 */
1638                 err = i40e_switch_tx_queue(hw, txq->reg_idx, FALSE);
1639
1640                 if (err) {
1641                         PMD_DRV_LOG(ERR, "Failed to switch TX queue %u of",
1642                                     tx_queue_id);
1643                         return err;
1644                 }
1645
1646                 i40e_tx_queue_release_mbufs(txq);
1647                 i40e_reset_tx_queue(txq);
1648                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
1649         }
1650
1651         return 0;
1652 }
1653
1654 const uint32_t *
1655 i40e_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1656 {
1657         static const uint32_t ptypes[] = {
1658                 /* refers to i40e_rxd_pkt_type_mapping() */
1659                 RTE_PTYPE_L2_ETHER,
1660                 RTE_PTYPE_L2_ETHER_TIMESYNC,
1661                 RTE_PTYPE_L2_ETHER_LLDP,
1662                 RTE_PTYPE_L2_ETHER_ARP,
1663                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
1664                 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
1665                 RTE_PTYPE_L4_FRAG,
1666                 RTE_PTYPE_L4_ICMP,
1667                 RTE_PTYPE_L4_NONFRAG,
1668                 RTE_PTYPE_L4_SCTP,
1669                 RTE_PTYPE_L4_TCP,
1670                 RTE_PTYPE_L4_UDP,
1671                 RTE_PTYPE_TUNNEL_GRENAT,
1672                 RTE_PTYPE_TUNNEL_IP,
1673                 RTE_PTYPE_INNER_L2_ETHER,
1674                 RTE_PTYPE_INNER_L2_ETHER_VLAN,
1675                 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN,
1676                 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN,
1677                 RTE_PTYPE_INNER_L4_FRAG,
1678                 RTE_PTYPE_INNER_L4_ICMP,
1679                 RTE_PTYPE_INNER_L4_NONFRAG,
1680                 RTE_PTYPE_INNER_L4_SCTP,
1681                 RTE_PTYPE_INNER_L4_TCP,
1682                 RTE_PTYPE_INNER_L4_UDP,
1683                 RTE_PTYPE_UNKNOWN
1684         };
1685
1686         if (dev->rx_pkt_burst == i40e_recv_pkts ||
1687 #ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC
1688             dev->rx_pkt_burst == i40e_recv_pkts_bulk_alloc ||
1689 #endif
1690             dev->rx_pkt_burst == i40e_recv_scattered_pkts ||
1691             dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec ||
1692             dev->rx_pkt_burst == i40e_recv_pkts_vec)
1693                 return ptypes;
1694         return NULL;
1695 }
1696
1697 int
1698 i40e_dev_rx_queue_setup(struct rte_eth_dev *dev,
1699                         uint16_t queue_idx,
1700                         uint16_t nb_desc,
1701                         unsigned int socket_id,
1702                         const struct rte_eth_rxconf *rx_conf,
1703                         struct rte_mempool *mp)
1704 {
1705         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1706         struct i40e_adapter *ad =
1707                 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
1708         struct i40e_vsi *vsi;
1709         struct i40e_pf *pf = NULL;
1710         struct i40e_vf *vf = NULL;
1711         struct i40e_rx_queue *rxq;
1712         const struct rte_memzone *rz;
1713         uint32_t ring_size;
1714         uint16_t len, i;
1715         uint16_t reg_idx, base, bsf, tc_mapping;
1716         int q_offset, use_def_burst_func = 1;
1717
1718         if (hw->mac.type == I40E_MAC_VF || hw->mac.type == I40E_MAC_X722_VF) {
1719                 vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1720                 vsi = &vf->vsi;
1721                 if (!vsi)
1722                         return -EINVAL;
1723                 reg_idx = queue_idx;
1724         } else {
1725                 pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1726                 vsi = i40e_pf_get_vsi_by_qindex(pf, queue_idx);
1727                 if (!vsi)
1728                         return -EINVAL;
1729                 q_offset = i40e_get_queue_offset_by_qindex(pf, queue_idx);
1730                 if (q_offset < 0)
1731                         return -EINVAL;
1732                 reg_idx = vsi->base_queue + q_offset;
1733         }
1734
1735         if (nb_desc % I40E_ALIGN_RING_DESC != 0 ||
1736             (nb_desc > I40E_MAX_RING_DESC) ||
1737             (nb_desc < I40E_MIN_RING_DESC)) {
1738                 PMD_DRV_LOG(ERR, "Number (%u) of receive descriptors is "
1739                             "invalid", nb_desc);
1740                 return -EINVAL;
1741         }
1742
1743         /* Free memory if needed */
1744         if (dev->data->rx_queues[queue_idx]) {
1745                 i40e_dev_rx_queue_release(dev->data->rx_queues[queue_idx]);
1746                 dev->data->rx_queues[queue_idx] = NULL;
1747         }
1748
1749         /* Allocate the rx queue data structure */
1750         rxq = rte_zmalloc_socket("i40e rx queue",
1751                                  sizeof(struct i40e_rx_queue),
1752                                  RTE_CACHE_LINE_SIZE,
1753                                  socket_id);
1754         if (!rxq) {
1755                 PMD_DRV_LOG(ERR, "Failed to allocate memory for "
1756                             "rx queue data structure");
1757                 return -ENOMEM;
1758         }
1759         rxq->mp = mp;
1760         rxq->nb_rx_desc = nb_desc;
1761         rxq->rx_free_thresh = rx_conf->rx_free_thresh;
1762         rxq->queue_id = queue_idx;
1763         rxq->reg_idx = reg_idx;
1764         rxq->port_id = dev->data->port_id;
1765         rxq->crc_len = (uint8_t) ((dev->data->dev_conf.rxmode.hw_strip_crc) ?
1766                                                         0 : ETHER_CRC_LEN);
1767         rxq->drop_en = rx_conf->rx_drop_en;
1768         rxq->vsi = vsi;
1769         rxq->rx_deferred_start = rx_conf->rx_deferred_start;
1770
1771         /* Allocate the maximun number of RX ring hardware descriptor. */
1772         len = I40E_MAX_RING_DESC;
1773
1774         /**
1775          * Allocating a little more memory because vectorized/bulk_alloc Rx
1776          * functions doesn't check boundaries each time.
1777          */
1778         len += RTE_PMD_I40E_RX_MAX_BURST;
1779
1780         ring_size = RTE_ALIGN(len * sizeof(union i40e_rx_desc),
1781                               I40E_DMA_MEM_ALIGN);
1782
1783         rz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx,
1784                               ring_size, I40E_RING_BASE_ALIGN, socket_id);
1785         if (!rz) {
1786                 i40e_dev_rx_queue_release(rxq);
1787                 PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for RX");
1788                 return -ENOMEM;
1789         }
1790
1791         /* Zero all the descriptors in the ring. */
1792         memset(rz->addr, 0, ring_size);
1793
1794         rxq->rx_ring_phys_addr = rz->iova;
1795         rxq->rx_ring = (union i40e_rx_desc *)rz->addr;
1796
1797         len = (uint16_t)(nb_desc + RTE_PMD_I40E_RX_MAX_BURST);
1798
1799         /* Allocate the software ring. */
1800         rxq->sw_ring =
1801                 rte_zmalloc_socket("i40e rx sw ring",
1802                                    sizeof(struct i40e_rx_entry) * len,
1803                                    RTE_CACHE_LINE_SIZE,
1804                                    socket_id);
1805         if (!rxq->sw_ring) {
1806                 i40e_dev_rx_queue_release(rxq);
1807                 PMD_DRV_LOG(ERR, "Failed to allocate memory for SW ring");
1808                 return -ENOMEM;
1809         }
1810
1811         i40e_reset_rx_queue(rxq);
1812         rxq->q_set = TRUE;
1813         dev->data->rx_queues[queue_idx] = rxq;
1814
1815         use_def_burst_func = check_rx_burst_bulk_alloc_preconditions(rxq);
1816
1817         if (!use_def_burst_func) {
1818 #ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC
1819                 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are "
1820                              "satisfied. Rx Burst Bulk Alloc function will be "
1821                              "used on port=%d, queue=%d.",
1822                              rxq->port_id, rxq->queue_id);
1823 #endif /* RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC */
1824         } else {
1825                 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are "
1826                              "not satisfied, Scattered Rx is requested, "
1827                              "or RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC is "
1828                              "not enabled on port=%d, queue=%d.",
1829                              rxq->port_id, rxq->queue_id);
1830                 ad->rx_bulk_alloc_allowed = false;
1831         }
1832
1833         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1834                 if (!(vsi->enabled_tc & (1 << i)))
1835                         continue;
1836                 tc_mapping = rte_le_to_cpu_16(vsi->info.tc_mapping[i]);
1837                 base = (tc_mapping & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) >>
1838                         I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT;
1839                 bsf = (tc_mapping & I40E_AQ_VSI_TC_QUE_NUMBER_MASK) >>
1840                         I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT;
1841
1842                 if (queue_idx >= base && queue_idx < (base + BIT(bsf)))
1843                         rxq->dcb_tc = i;
1844         }
1845
1846         return 0;
1847 }
1848
1849 void
1850 i40e_dev_rx_queue_release(void *rxq)
1851 {
1852         struct i40e_rx_queue *q = (struct i40e_rx_queue *)rxq;
1853
1854         if (!q) {
1855                 PMD_DRV_LOG(DEBUG, "Pointer to rxq is NULL");
1856                 return;
1857         }
1858
1859         i40e_rx_queue_release_mbufs(q);
1860         rte_free(q->sw_ring);
1861         rte_free(q);
1862 }
1863
1864 uint32_t
1865 i40e_dev_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1866 {
1867 #define I40E_RXQ_SCAN_INTERVAL 4
1868         volatile union i40e_rx_desc *rxdp;
1869         struct i40e_rx_queue *rxq;
1870         uint16_t desc = 0;
1871
1872         rxq = dev->data->rx_queues[rx_queue_id];
1873         rxdp = &(rxq->rx_ring[rxq->rx_tail]);
1874         while ((desc < rxq->nb_rx_desc) &&
1875                 ((rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len) &
1876                 I40E_RXD_QW1_STATUS_MASK) >> I40E_RXD_QW1_STATUS_SHIFT) &
1877                                 (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) {
1878                 /**
1879                  * Check the DD bit of a rx descriptor of each 4 in a group,
1880                  * to avoid checking too frequently and downgrading performance
1881                  * too much.
1882                  */
1883                 desc += I40E_RXQ_SCAN_INTERVAL;
1884                 rxdp += I40E_RXQ_SCAN_INTERVAL;
1885                 if (rxq->rx_tail + desc >= rxq->nb_rx_desc)
1886                         rxdp = &(rxq->rx_ring[rxq->rx_tail +
1887                                         desc - rxq->nb_rx_desc]);
1888         }
1889
1890         return desc;
1891 }
1892
1893 int
1894 i40e_dev_rx_descriptor_done(void *rx_queue, uint16_t offset)
1895 {
1896         volatile union i40e_rx_desc *rxdp;
1897         struct i40e_rx_queue *rxq = rx_queue;
1898         uint16_t desc;
1899         int ret;
1900
1901         if (unlikely(offset >= rxq->nb_rx_desc)) {
1902                 PMD_DRV_LOG(ERR, "Invalid RX descriptor id %u", offset);
1903                 return 0;
1904         }
1905
1906         desc = rxq->rx_tail + offset;
1907         if (desc >= rxq->nb_rx_desc)
1908                 desc -= rxq->nb_rx_desc;
1909
1910         rxdp = &(rxq->rx_ring[desc]);
1911
1912         ret = !!(((rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len) &
1913                 I40E_RXD_QW1_STATUS_MASK) >> I40E_RXD_QW1_STATUS_SHIFT) &
1914                                 (1 << I40E_RX_DESC_STATUS_DD_SHIFT));
1915
1916         return ret;
1917 }
1918
1919 int
1920 i40e_dev_rx_descriptor_status(void *rx_queue, uint16_t offset)
1921 {
1922         struct i40e_rx_queue *rxq = rx_queue;
1923         volatile uint64_t *status;
1924         uint64_t mask;
1925         uint32_t desc;
1926
1927         if (unlikely(offset >= rxq->nb_rx_desc))
1928                 return -EINVAL;
1929
1930         if (offset >= rxq->nb_rx_desc - rxq->nb_rx_hold)
1931                 return RTE_ETH_RX_DESC_UNAVAIL;
1932
1933         desc = rxq->rx_tail + offset;
1934         if (desc >= rxq->nb_rx_desc)
1935                 desc -= rxq->nb_rx_desc;
1936
1937         status = &rxq->rx_ring[desc].wb.qword1.status_error_len;
1938         mask = rte_le_to_cpu_64((1ULL << I40E_RX_DESC_STATUS_DD_SHIFT)
1939                 << I40E_RXD_QW1_STATUS_SHIFT);
1940         if (*status & mask)
1941                 return RTE_ETH_RX_DESC_DONE;
1942
1943         return RTE_ETH_RX_DESC_AVAIL;
1944 }
1945
1946 int
1947 i40e_dev_tx_descriptor_status(void *tx_queue, uint16_t offset)
1948 {
1949         struct i40e_tx_queue *txq = tx_queue;
1950         volatile uint64_t *status;
1951         uint64_t mask, expect;
1952         uint32_t desc;
1953
1954         if (unlikely(offset >= txq->nb_tx_desc))
1955                 return -EINVAL;
1956
1957         desc = txq->tx_tail + offset;
1958         /* go to next desc that has the RS bit */
1959         desc = ((desc + txq->tx_rs_thresh - 1) / txq->tx_rs_thresh) *
1960                 txq->tx_rs_thresh;
1961         if (desc >= txq->nb_tx_desc) {
1962                 desc -= txq->nb_tx_desc;
1963                 if (desc >= txq->nb_tx_desc)
1964                         desc -= txq->nb_tx_desc;
1965         }
1966
1967         status = &txq->tx_ring[desc].cmd_type_offset_bsz;
1968         mask = rte_le_to_cpu_64(I40E_TXD_QW1_DTYPE_MASK);
1969         expect = rte_cpu_to_le_64(
1970                 I40E_TX_DESC_DTYPE_DESC_DONE << I40E_TXD_QW1_DTYPE_SHIFT);
1971         if ((*status & mask) == expect)
1972                 return RTE_ETH_TX_DESC_DONE;
1973
1974         return RTE_ETH_TX_DESC_FULL;
1975 }
1976
1977 int
1978 i40e_dev_tx_queue_setup(struct rte_eth_dev *dev,
1979                         uint16_t queue_idx,
1980                         uint16_t nb_desc,
1981                         unsigned int socket_id,
1982                         const struct rte_eth_txconf *tx_conf)
1983 {
1984         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1985         struct i40e_vsi *vsi;
1986         struct i40e_pf *pf = NULL;
1987         struct i40e_vf *vf = NULL;
1988         struct i40e_tx_queue *txq;
1989         const struct rte_memzone *tz;
1990         uint32_t ring_size;
1991         uint16_t tx_rs_thresh, tx_free_thresh;
1992         uint16_t reg_idx, i, base, bsf, tc_mapping;
1993         int q_offset;
1994
1995         if (hw->mac.type == I40E_MAC_VF || hw->mac.type == I40E_MAC_X722_VF) {
1996                 vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1997                 vsi = &vf->vsi;
1998                 if (!vsi)
1999                         return -EINVAL;
2000                 reg_idx = queue_idx;
2001         } else {
2002                 pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2003                 vsi = i40e_pf_get_vsi_by_qindex(pf, queue_idx);
2004                 if (!vsi)
2005                         return -EINVAL;
2006                 q_offset = i40e_get_queue_offset_by_qindex(pf, queue_idx);
2007                 if (q_offset < 0)
2008                         return -EINVAL;
2009                 reg_idx = vsi->base_queue + q_offset;
2010         }
2011
2012         if (nb_desc % I40E_ALIGN_RING_DESC != 0 ||
2013             (nb_desc > I40E_MAX_RING_DESC) ||
2014             (nb_desc < I40E_MIN_RING_DESC)) {
2015                 PMD_DRV_LOG(ERR, "Number (%u) of transmit descriptors is "
2016                             "invalid", nb_desc);
2017                 return -EINVAL;
2018         }
2019
2020         /**
2021          * The following two parameters control the setting of the RS bit on
2022          * transmit descriptors. TX descriptors will have their RS bit set
2023          * after txq->tx_rs_thresh descriptors have been used. The TX
2024          * descriptor ring will be cleaned after txq->tx_free_thresh
2025          * descriptors are used or if the number of descriptors required to
2026          * transmit a packet is greater than the number of free TX descriptors.
2027          *
2028          * The following constraints must be satisfied:
2029          *  - tx_rs_thresh must be greater than 0.
2030          *  - tx_rs_thresh must be less than the size of the ring minus 2.
2031          *  - tx_rs_thresh must be less than or equal to tx_free_thresh.
2032          *  - tx_rs_thresh must be a divisor of the ring size.
2033          *  - tx_free_thresh must be greater than 0.
2034          *  - tx_free_thresh must be less than the size of the ring minus 3.
2035          *
2036          * One descriptor in the TX ring is used as a sentinel to avoid a H/W
2037          * race condition, hence the maximum threshold constraints. When set
2038          * to zero use default values.
2039          */
2040         tx_rs_thresh = (uint16_t)((tx_conf->tx_rs_thresh) ?
2041                 tx_conf->tx_rs_thresh : DEFAULT_TX_RS_THRESH);
2042         tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ?
2043                 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH);
2044         if (tx_rs_thresh >= (nb_desc - 2)) {
2045                 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than the "
2046                              "number of TX descriptors minus 2. "
2047                              "(tx_rs_thresh=%u port=%d queue=%d)",
2048                              (unsigned int)tx_rs_thresh,
2049                              (int)dev->data->port_id,
2050                              (int)queue_idx);
2051                 return I40E_ERR_PARAM;
2052         }
2053         if (tx_free_thresh >= (nb_desc - 3)) {
2054                 PMD_INIT_LOG(ERR, "tx_free_thresh must be less than the "
2055                              "number of TX descriptors minus 3. "
2056                              "(tx_free_thresh=%u port=%d queue=%d)",
2057                              (unsigned int)tx_free_thresh,
2058                              (int)dev->data->port_id,
2059                              (int)queue_idx);
2060                 return I40E_ERR_PARAM;
2061         }
2062         if (tx_rs_thresh > tx_free_thresh) {
2063                 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than or "
2064                              "equal to tx_free_thresh. (tx_free_thresh=%u"
2065                              " tx_rs_thresh=%u port=%d queue=%d)",
2066                              (unsigned int)tx_free_thresh,
2067                              (unsigned int)tx_rs_thresh,
2068                              (int)dev->data->port_id,
2069                              (int)queue_idx);
2070                 return I40E_ERR_PARAM;
2071         }
2072         if ((nb_desc % tx_rs_thresh) != 0) {
2073                 PMD_INIT_LOG(ERR, "tx_rs_thresh must be a divisor of the "
2074                              "number of TX descriptors. (tx_rs_thresh=%u"
2075                              " port=%d queue=%d)",
2076                              (unsigned int)tx_rs_thresh,
2077                              (int)dev->data->port_id,
2078                              (int)queue_idx);
2079                 return I40E_ERR_PARAM;
2080         }
2081         if ((tx_rs_thresh > 1) && (tx_conf->tx_thresh.wthresh != 0)) {
2082                 PMD_INIT_LOG(ERR, "TX WTHRESH must be set to 0 if "
2083                              "tx_rs_thresh is greater than 1. "
2084                              "(tx_rs_thresh=%u port=%d queue=%d)",
2085                              (unsigned int)tx_rs_thresh,
2086                              (int)dev->data->port_id,
2087                              (int)queue_idx);
2088                 return I40E_ERR_PARAM;
2089         }
2090
2091         /* Free memory if needed. */
2092         if (dev->data->tx_queues[queue_idx]) {
2093                 i40e_dev_tx_queue_release(dev->data->tx_queues[queue_idx]);
2094                 dev->data->tx_queues[queue_idx] = NULL;
2095         }
2096
2097         /* Allocate the TX queue data structure. */
2098         txq = rte_zmalloc_socket("i40e tx queue",
2099                                   sizeof(struct i40e_tx_queue),
2100                                   RTE_CACHE_LINE_SIZE,
2101                                   socket_id);
2102         if (!txq) {
2103                 PMD_DRV_LOG(ERR, "Failed to allocate memory for "
2104                             "tx queue structure");
2105                 return -ENOMEM;
2106         }
2107
2108         /* Allocate TX hardware ring descriptors. */
2109         ring_size = sizeof(struct i40e_tx_desc) * I40E_MAX_RING_DESC;
2110         ring_size = RTE_ALIGN(ring_size, I40E_DMA_MEM_ALIGN);
2111         tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx,
2112                               ring_size, I40E_RING_BASE_ALIGN, socket_id);
2113         if (!tz) {
2114                 i40e_dev_tx_queue_release(txq);
2115                 PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for TX");
2116                 return -ENOMEM;
2117         }
2118
2119         txq->nb_tx_desc = nb_desc;
2120         txq->tx_rs_thresh = tx_rs_thresh;
2121         txq->tx_free_thresh = tx_free_thresh;
2122         txq->pthresh = tx_conf->tx_thresh.pthresh;
2123         txq->hthresh = tx_conf->tx_thresh.hthresh;
2124         txq->wthresh = tx_conf->tx_thresh.wthresh;
2125         txq->queue_id = queue_idx;
2126         txq->reg_idx = reg_idx;
2127         txq->port_id = dev->data->port_id;
2128         txq->txq_flags = tx_conf->txq_flags;
2129         txq->vsi = vsi;
2130         txq->tx_deferred_start = tx_conf->tx_deferred_start;
2131
2132         txq->tx_ring_phys_addr = tz->iova;
2133         txq->tx_ring = (struct i40e_tx_desc *)tz->addr;
2134
2135         /* Allocate software ring */
2136         txq->sw_ring =
2137                 rte_zmalloc_socket("i40e tx sw ring",
2138                                    sizeof(struct i40e_tx_entry) * nb_desc,
2139                                    RTE_CACHE_LINE_SIZE,
2140                                    socket_id);
2141         if (!txq->sw_ring) {
2142                 i40e_dev_tx_queue_release(txq);
2143                 PMD_DRV_LOG(ERR, "Failed to allocate memory for SW TX ring");
2144                 return -ENOMEM;
2145         }
2146
2147         i40e_reset_tx_queue(txq);
2148         txq->q_set = TRUE;
2149         dev->data->tx_queues[queue_idx] = txq;
2150
2151         /* Use a simple TX queue without offloads or multi segs if possible */
2152         i40e_set_tx_function_flag(dev, txq);
2153
2154         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
2155                 if (!(vsi->enabled_tc & (1 << i)))
2156                         continue;
2157                 tc_mapping = rte_le_to_cpu_16(vsi->info.tc_mapping[i]);
2158                 base = (tc_mapping & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) >>
2159                         I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT;
2160                 bsf = (tc_mapping & I40E_AQ_VSI_TC_QUE_NUMBER_MASK) >>
2161                         I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT;
2162
2163                 if (queue_idx >= base && queue_idx < (base + BIT(bsf)))
2164                         txq->dcb_tc = i;
2165         }
2166
2167         return 0;
2168 }
2169
2170 void
2171 i40e_dev_tx_queue_release(void *txq)
2172 {
2173         struct i40e_tx_queue *q = (struct i40e_tx_queue *)txq;
2174
2175         if (!q) {
2176                 PMD_DRV_LOG(DEBUG, "Pointer to TX queue is NULL");
2177                 return;
2178         }
2179
2180         i40e_tx_queue_release_mbufs(q);
2181         rte_free(q->sw_ring);
2182         rte_free(q);
2183 }
2184
2185 const struct rte_memzone *
2186 i40e_memzone_reserve(const char *name, uint32_t len, int socket_id)
2187 {
2188         const struct rte_memzone *mz;
2189
2190         mz = rte_memzone_lookup(name);
2191         if (mz)
2192                 return mz;
2193
2194         mz = rte_memzone_reserve_aligned(name, len,
2195                                          socket_id, 0, I40E_RING_BASE_ALIGN);
2196         return mz;
2197 }
2198
2199 void
2200 i40e_rx_queue_release_mbufs(struct i40e_rx_queue *rxq)
2201 {
2202         uint16_t i;
2203
2204         /* SSE Vector driver has a different way of releasing mbufs. */
2205         if (rxq->rx_using_sse) {
2206                 i40e_rx_queue_release_mbufs_vec(rxq);
2207                 return;
2208         }
2209
2210         if (!rxq->sw_ring) {
2211                 PMD_DRV_LOG(DEBUG, "Pointer to sw_ring is NULL");
2212                 return;
2213         }
2214
2215         for (i = 0; i < rxq->nb_rx_desc; i++) {
2216                 if (rxq->sw_ring[i].mbuf) {
2217                         rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
2218                         rxq->sw_ring[i].mbuf = NULL;
2219                 }
2220         }
2221 #ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC
2222         if (rxq->rx_nb_avail == 0)
2223                 return;
2224         for (i = 0; i < rxq->rx_nb_avail; i++) {
2225                 struct rte_mbuf *mbuf;
2226
2227                 mbuf = rxq->rx_stage[rxq->rx_next_avail + i];
2228                 rte_pktmbuf_free_seg(mbuf);
2229         }
2230         rxq->rx_nb_avail = 0;
2231 #endif /* RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC */
2232 }
2233
2234 void
2235 i40e_reset_rx_queue(struct i40e_rx_queue *rxq)
2236 {
2237         unsigned i;
2238         uint16_t len;
2239
2240         if (!rxq) {
2241                 PMD_DRV_LOG(DEBUG, "Pointer to rxq is NULL");
2242                 return;
2243         }
2244
2245 #ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC
2246         if (check_rx_burst_bulk_alloc_preconditions(rxq) == 0)
2247                 len = (uint16_t)(rxq->nb_rx_desc + RTE_PMD_I40E_RX_MAX_BURST);
2248         else
2249 #endif /* RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC */
2250                 len = rxq->nb_rx_desc;
2251
2252         for (i = 0; i < len * sizeof(union i40e_rx_desc); i++)
2253                 ((volatile char *)rxq->rx_ring)[i] = 0;
2254
2255         memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf));
2256         for (i = 0; i < RTE_PMD_I40E_RX_MAX_BURST; ++i)
2257                 rxq->sw_ring[rxq->nb_rx_desc + i].mbuf = &rxq->fake_mbuf;
2258
2259 #ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC
2260         rxq->rx_nb_avail = 0;
2261         rxq->rx_next_avail = 0;
2262         rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1);
2263 #endif /* RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC */
2264         rxq->rx_tail = 0;
2265         rxq->nb_rx_hold = 0;
2266         rxq->pkt_first_seg = NULL;
2267         rxq->pkt_last_seg = NULL;
2268
2269         rxq->rxrearm_start = 0;
2270         rxq->rxrearm_nb = 0;
2271 }
2272
2273 void
2274 i40e_tx_queue_release_mbufs(struct i40e_tx_queue *txq)
2275 {
2276         struct rte_eth_dev *dev;
2277         uint16_t i;
2278
2279         dev = &rte_eth_devices[txq->port_id];
2280
2281         if (!txq || !txq->sw_ring) {
2282                 PMD_DRV_LOG(DEBUG, "Pointer to rxq or sw_ring is NULL");
2283                 return;
2284         }
2285
2286         /**
2287          *  vPMD tx will not set sw_ring's mbuf to NULL after free,
2288          *  so need to free remains more carefully.
2289          */
2290         if (dev->tx_pkt_burst == i40e_xmit_pkts_vec) {
2291                 i = txq->tx_next_dd - txq->tx_rs_thresh + 1;
2292                 if (txq->tx_tail < i) {
2293                         for (; i < txq->nb_tx_desc; i++) {
2294                                 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
2295                                 txq->sw_ring[i].mbuf = NULL;
2296                         }
2297                         i = 0;
2298                 }
2299                 for (; i < txq->tx_tail; i++) {
2300                         rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
2301                         txq->sw_ring[i].mbuf = NULL;
2302                 }
2303         } else {
2304                 for (i = 0; i < txq->nb_tx_desc; i++) {
2305                         if (txq->sw_ring[i].mbuf) {
2306                                 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
2307                                 txq->sw_ring[i].mbuf = NULL;
2308                         }
2309                 }
2310         }
2311 }
2312
2313 void
2314 i40e_reset_tx_queue(struct i40e_tx_queue *txq)
2315 {
2316         struct i40e_tx_entry *txe;
2317         uint16_t i, prev, size;
2318
2319         if (!txq) {
2320                 PMD_DRV_LOG(DEBUG, "Pointer to txq is NULL");
2321                 return;
2322         }
2323
2324         txe = txq->sw_ring;
2325         size = sizeof(struct i40e_tx_desc) * txq->nb_tx_desc;
2326         for (i = 0; i < size; i++)
2327                 ((volatile char *)txq->tx_ring)[i] = 0;
2328
2329         prev = (uint16_t)(txq->nb_tx_desc - 1);
2330         for (i = 0; i < txq->nb_tx_desc; i++) {
2331                 volatile struct i40e_tx_desc *txd = &txq->tx_ring[i];
2332
2333                 txd->cmd_type_offset_bsz =
2334                         rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE);
2335                 txe[i].mbuf =  NULL;
2336                 txe[i].last_id = i;
2337                 txe[prev].next_id = i;
2338                 prev = i;
2339         }
2340
2341         txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
2342         txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
2343
2344         txq->tx_tail = 0;
2345         txq->nb_tx_used = 0;
2346
2347         txq->last_desc_cleaned = (uint16_t)(txq->nb_tx_desc - 1);
2348         txq->nb_tx_free = (uint16_t)(txq->nb_tx_desc - 1);
2349 }
2350
2351 /* Init the TX queue in hardware */
2352 int
2353 i40e_tx_queue_init(struct i40e_tx_queue *txq)
2354 {
2355         enum i40e_status_code err = I40E_SUCCESS;
2356         struct i40e_vsi *vsi = txq->vsi;
2357         struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2358         uint16_t pf_q = txq->reg_idx;
2359         struct i40e_hmc_obj_txq tx_ctx;
2360         uint32_t qtx_ctl;
2361
2362         /* clear the context structure first */
2363         memset(&tx_ctx, 0, sizeof(tx_ctx));
2364         tx_ctx.new_context = 1;
2365         tx_ctx.base = txq->tx_ring_phys_addr / I40E_QUEUE_BASE_ADDR_UNIT;
2366         tx_ctx.qlen = txq->nb_tx_desc;
2367
2368 #ifdef RTE_LIBRTE_IEEE1588
2369         tx_ctx.timesync_ena = 1;
2370 #endif
2371         tx_ctx.rdylist = rte_le_to_cpu_16(vsi->info.qs_handle[txq->dcb_tc]);
2372         if (vsi->type == I40E_VSI_FDIR)
2373                 tx_ctx.fd_ena = TRUE;
2374
2375         err = i40e_clear_lan_tx_queue_context(hw, pf_q);
2376         if (err != I40E_SUCCESS) {
2377                 PMD_DRV_LOG(ERR, "Failure of clean lan tx queue context");
2378                 return err;
2379         }
2380
2381         err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx);
2382         if (err != I40E_SUCCESS) {
2383                 PMD_DRV_LOG(ERR, "Failure of set lan tx queue context");
2384                 return err;
2385         }
2386
2387         /* Now associate this queue with this PCI function */
2388         qtx_ctl = I40E_QTX_CTL_PF_QUEUE;
2389         qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
2390                                         I40E_QTX_CTL_PF_INDX_MASK);
2391         I40E_WRITE_REG(hw, I40E_QTX_CTL(pf_q), qtx_ctl);
2392         I40E_WRITE_FLUSH(hw);
2393
2394         txq->qtx_tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);
2395
2396         return err;
2397 }
2398
2399 int
2400 i40e_alloc_rx_queue_mbufs(struct i40e_rx_queue *rxq)
2401 {
2402         struct i40e_rx_entry *rxe = rxq->sw_ring;
2403         uint64_t dma_addr;
2404         uint16_t i;
2405
2406         for (i = 0; i < rxq->nb_rx_desc; i++) {
2407                 volatile union i40e_rx_desc *rxd;
2408                 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mp);
2409
2410                 if (unlikely(!mbuf)) {
2411                         PMD_DRV_LOG(ERR, "Failed to allocate mbuf for RX");
2412                         return -ENOMEM;
2413                 }
2414
2415                 rte_mbuf_refcnt_set(mbuf, 1);
2416                 mbuf->next = NULL;
2417                 mbuf->data_off = RTE_PKTMBUF_HEADROOM;
2418                 mbuf->nb_segs = 1;
2419                 mbuf->port = rxq->port_id;
2420
2421                 dma_addr =
2422                         rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf));
2423
2424                 rxd = &rxq->rx_ring[i];
2425                 rxd->read.pkt_addr = dma_addr;
2426                 rxd->read.hdr_addr = 0;
2427 #ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC
2428                 rxd->read.rsvd1 = 0;
2429                 rxd->read.rsvd2 = 0;
2430 #endif /* RTE_LIBRTE_I40E_16BYTE_RX_DESC */
2431
2432                 rxe[i].mbuf = mbuf;
2433         }
2434
2435         return 0;
2436 }
2437
2438 /*
2439  * Calculate the buffer length, and check the jumbo frame
2440  * and maximum packet length.
2441  */
2442 static int
2443 i40e_rx_queue_config(struct i40e_rx_queue *rxq)
2444 {
2445         struct i40e_pf *pf = I40E_VSI_TO_PF(rxq->vsi);
2446         struct i40e_hw *hw = I40E_VSI_TO_HW(rxq->vsi);
2447         struct rte_eth_dev_data *data = pf->dev_data;
2448         uint16_t buf_size, len;
2449
2450         buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
2451                 RTE_PKTMBUF_HEADROOM);
2452
2453         switch (pf->flags & (I40E_FLAG_HEADER_SPLIT_DISABLED |
2454                         I40E_FLAG_HEADER_SPLIT_ENABLED)) {
2455         case I40E_FLAG_HEADER_SPLIT_ENABLED: /* Not supported */
2456                 rxq->rx_hdr_len = RTE_ALIGN(I40E_RXBUF_SZ_1024,
2457                                 (1 << I40E_RXQ_CTX_HBUFF_SHIFT));
2458                 rxq->rx_buf_len = RTE_ALIGN(I40E_RXBUF_SZ_2048,
2459                                 (1 << I40E_RXQ_CTX_DBUFF_SHIFT));
2460                 rxq->hs_mode = i40e_header_split_enabled;
2461                 break;
2462         case I40E_FLAG_HEADER_SPLIT_DISABLED:
2463         default:
2464                 rxq->rx_hdr_len = 0;
2465                 rxq->rx_buf_len = RTE_ALIGN_FLOOR(buf_size,
2466                         (1 << I40E_RXQ_CTX_DBUFF_SHIFT));
2467                 rxq->hs_mode = i40e_header_split_none;
2468                 break;
2469         }
2470
2471         len = hw->func_caps.rx_buf_chain_len * rxq->rx_buf_len;
2472         rxq->max_pkt_len = RTE_MIN(len, data->dev_conf.rxmode.max_rx_pkt_len);
2473         if (data->dev_conf.rxmode.jumbo_frame == 1) {
2474                 if (rxq->max_pkt_len <= ETHER_MAX_LEN ||
2475                         rxq->max_pkt_len > I40E_FRAME_SIZE_MAX) {
2476                         PMD_DRV_LOG(ERR, "maximum packet length must "
2477                                     "be larger than %u and smaller than %u,"
2478                                     "as jumbo frame is enabled",
2479                                     (uint32_t)ETHER_MAX_LEN,
2480                                     (uint32_t)I40E_FRAME_SIZE_MAX);
2481                         return I40E_ERR_CONFIG;
2482                 }
2483         } else {
2484                 if (rxq->max_pkt_len < ETHER_MIN_LEN ||
2485                         rxq->max_pkt_len > ETHER_MAX_LEN) {
2486                         PMD_DRV_LOG(ERR, "maximum packet length must be "
2487                                     "larger than %u and smaller than %u, "
2488                                     "as jumbo frame is disabled",
2489                                     (uint32_t)ETHER_MIN_LEN,
2490                                     (uint32_t)ETHER_MAX_LEN);
2491                         return I40E_ERR_CONFIG;
2492                 }
2493         }
2494
2495         return 0;
2496 }
2497
2498 /* Init the RX queue in hardware */
2499 int
2500 i40e_rx_queue_init(struct i40e_rx_queue *rxq)
2501 {
2502         int err = I40E_SUCCESS;
2503         struct i40e_hw *hw = I40E_VSI_TO_HW(rxq->vsi);
2504         struct rte_eth_dev_data *dev_data = I40E_VSI_TO_DEV_DATA(rxq->vsi);
2505         uint16_t pf_q = rxq->reg_idx;
2506         uint16_t buf_size;
2507         struct i40e_hmc_obj_rxq rx_ctx;
2508
2509         err = i40e_rx_queue_config(rxq);
2510         if (err < 0) {
2511                 PMD_DRV_LOG(ERR, "Failed to config RX queue");
2512                 return err;
2513         }
2514
2515         /* Clear the context structure first */
2516         memset(&rx_ctx, 0, sizeof(struct i40e_hmc_obj_rxq));
2517         rx_ctx.dbuff = rxq->rx_buf_len >> I40E_RXQ_CTX_DBUFF_SHIFT;
2518         rx_ctx.hbuff = rxq->rx_hdr_len >> I40E_RXQ_CTX_HBUFF_SHIFT;
2519
2520         rx_ctx.base = rxq->rx_ring_phys_addr / I40E_QUEUE_BASE_ADDR_UNIT;
2521         rx_ctx.qlen = rxq->nb_rx_desc;
2522 #ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC
2523         rx_ctx.dsize = 1;
2524 #endif
2525         rx_ctx.dtype = rxq->hs_mode;
2526         if (rxq->hs_mode)
2527                 rx_ctx.hsplit_0 = I40E_HEADER_SPLIT_ALL;
2528         else
2529                 rx_ctx.hsplit_0 = I40E_HEADER_SPLIT_NONE;
2530         rx_ctx.rxmax = rxq->max_pkt_len;
2531         rx_ctx.tphrdesc_ena = 1;
2532         rx_ctx.tphwdesc_ena = 1;
2533         rx_ctx.tphdata_ena = 1;
2534         rx_ctx.tphhead_ena = 1;
2535         rx_ctx.lrxqthresh = 2;
2536         rx_ctx.crcstrip = (rxq->crc_len == 0) ? 1 : 0;
2537         rx_ctx.l2tsel = 1;
2538         /* showiv indicates if inner VLAN is stripped inside of tunnel
2539          * packet. When set it to 1, vlan information is stripped from
2540          * the inner header, but the hardware does not put it in the
2541          * descriptor. So set it zero by default.
2542          */
2543         rx_ctx.showiv = 0;
2544         rx_ctx.prefena = 1;
2545
2546         err = i40e_clear_lan_rx_queue_context(hw, pf_q);
2547         if (err != I40E_SUCCESS) {
2548                 PMD_DRV_LOG(ERR, "Failed to clear LAN RX queue context");
2549                 return err;
2550         }
2551         err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx);
2552         if (err != I40E_SUCCESS) {
2553                 PMD_DRV_LOG(ERR, "Failed to set LAN RX queue context");
2554                 return err;
2555         }
2556
2557         rxq->qrx_tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
2558
2559         buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
2560                 RTE_PKTMBUF_HEADROOM);
2561
2562         /* Check if scattered RX needs to be used. */
2563         if ((rxq->max_pkt_len + 2 * I40E_VLAN_TAG_SIZE) > buf_size) {
2564                 dev_data->scattered_rx = 1;
2565         }
2566
2567         /* Init the RX tail regieter. */
2568         I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
2569
2570         return 0;
2571 }
2572
2573 void
2574 i40e_dev_clear_queues(struct rte_eth_dev *dev)
2575 {
2576         uint16_t i;
2577
2578         PMD_INIT_FUNC_TRACE();
2579
2580         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2581                 if (!dev->data->tx_queues[i])
2582                         continue;
2583                 i40e_tx_queue_release_mbufs(dev->data->tx_queues[i]);
2584                 i40e_reset_tx_queue(dev->data->tx_queues[i]);
2585         }
2586
2587         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2588                 if (!dev->data->rx_queues[i])
2589                         continue;
2590                 i40e_rx_queue_release_mbufs(dev->data->rx_queues[i]);
2591                 i40e_reset_rx_queue(dev->data->rx_queues[i]);
2592         }
2593 }
2594
2595 void
2596 i40e_dev_free_queues(struct rte_eth_dev *dev)
2597 {
2598         uint16_t i;
2599
2600         PMD_INIT_FUNC_TRACE();
2601
2602         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2603                 if (!dev->data->rx_queues[i])
2604                         continue;
2605                 i40e_dev_rx_queue_release(dev->data->rx_queues[i]);
2606                 dev->data->rx_queues[i] = NULL;
2607         }
2608         dev->data->nb_rx_queues = 0;
2609
2610         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2611                 if (!dev->data->tx_queues[i])
2612                         continue;
2613                 i40e_dev_tx_queue_release(dev->data->tx_queues[i]);
2614                 dev->data->tx_queues[i] = NULL;
2615         }
2616         dev->data->nb_tx_queues = 0;
2617 }
2618
2619 #define I40E_FDIR_NUM_TX_DESC  I40E_MIN_RING_DESC
2620 #define I40E_FDIR_NUM_RX_DESC  I40E_MIN_RING_DESC
2621
2622 enum i40e_status_code
2623 i40e_fdir_setup_tx_resources(struct i40e_pf *pf)
2624 {
2625         struct i40e_tx_queue *txq;
2626         const struct rte_memzone *tz = NULL;
2627         uint32_t ring_size;
2628         struct rte_eth_dev *dev;
2629
2630         if (!pf) {
2631                 PMD_DRV_LOG(ERR, "PF is not available");
2632                 return I40E_ERR_BAD_PTR;
2633         }
2634
2635         dev = pf->adapter->eth_dev;
2636
2637         /* Allocate the TX queue data structure. */
2638         txq = rte_zmalloc_socket("i40e fdir tx queue",
2639                                   sizeof(struct i40e_tx_queue),
2640                                   RTE_CACHE_LINE_SIZE,
2641                                   SOCKET_ID_ANY);
2642         if (!txq) {
2643                 PMD_DRV_LOG(ERR, "Failed to allocate memory for "
2644                                         "tx queue structure.");
2645                 return I40E_ERR_NO_MEMORY;
2646         }
2647
2648         /* Allocate TX hardware ring descriptors. */
2649         ring_size = sizeof(struct i40e_tx_desc) * I40E_FDIR_NUM_TX_DESC;
2650         ring_size = RTE_ALIGN(ring_size, I40E_DMA_MEM_ALIGN);
2651
2652         tz = rte_eth_dma_zone_reserve(dev, "fdir_tx_ring",
2653                                       I40E_FDIR_QUEUE_ID, ring_size,
2654                                       I40E_RING_BASE_ALIGN, SOCKET_ID_ANY);
2655         if (!tz) {
2656                 i40e_dev_tx_queue_release(txq);
2657                 PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for TX.");
2658                 return I40E_ERR_NO_MEMORY;
2659         }
2660
2661         txq->nb_tx_desc = I40E_FDIR_NUM_TX_DESC;
2662         txq->queue_id = I40E_FDIR_QUEUE_ID;
2663         txq->reg_idx = pf->fdir.fdir_vsi->base_queue;
2664         txq->vsi = pf->fdir.fdir_vsi;
2665
2666         txq->tx_ring_phys_addr = tz->iova;
2667         txq->tx_ring = (struct i40e_tx_desc *)tz->addr;
2668         /*
2669          * don't need to allocate software ring and reset for the fdir
2670          * program queue just set the queue has been configured.
2671          */
2672         txq->q_set = TRUE;
2673         pf->fdir.txq = txq;
2674
2675         return I40E_SUCCESS;
2676 }
2677
2678 enum i40e_status_code
2679 i40e_fdir_setup_rx_resources(struct i40e_pf *pf)
2680 {
2681         struct i40e_rx_queue *rxq;
2682         const struct rte_memzone *rz = NULL;
2683         uint32_t ring_size;
2684         struct rte_eth_dev *dev;
2685
2686         if (!pf) {
2687                 PMD_DRV_LOG(ERR, "PF is not available");
2688                 return I40E_ERR_BAD_PTR;
2689         }
2690
2691         dev = pf->adapter->eth_dev;
2692
2693         /* Allocate the RX queue data structure. */
2694         rxq = rte_zmalloc_socket("i40e fdir rx queue",
2695                                   sizeof(struct i40e_rx_queue),
2696                                   RTE_CACHE_LINE_SIZE,
2697                                   SOCKET_ID_ANY);
2698         if (!rxq) {
2699                 PMD_DRV_LOG(ERR, "Failed to allocate memory for "
2700                                         "rx queue structure.");
2701                 return I40E_ERR_NO_MEMORY;
2702         }
2703
2704         /* Allocate RX hardware ring descriptors. */
2705         ring_size = sizeof(union i40e_rx_desc) * I40E_FDIR_NUM_RX_DESC;
2706         ring_size = RTE_ALIGN(ring_size, I40E_DMA_MEM_ALIGN);
2707
2708         rz = rte_eth_dma_zone_reserve(dev, "fdir_rx_ring",
2709                                       I40E_FDIR_QUEUE_ID, ring_size,
2710                                       I40E_RING_BASE_ALIGN, SOCKET_ID_ANY);
2711         if (!rz) {
2712                 i40e_dev_rx_queue_release(rxq);
2713                 PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for RX.");
2714                 return I40E_ERR_NO_MEMORY;
2715         }
2716
2717         rxq->nb_rx_desc = I40E_FDIR_NUM_RX_DESC;
2718         rxq->queue_id = I40E_FDIR_QUEUE_ID;
2719         rxq->reg_idx = pf->fdir.fdir_vsi->base_queue;
2720         rxq->vsi = pf->fdir.fdir_vsi;
2721
2722         rxq->rx_ring_phys_addr = rz->iova;
2723         rxq->rx_ring = (union i40e_rx_desc *)rz->addr;
2724
2725         /*
2726          * Don't need to allocate software ring and reset for the fdir
2727          * rx queue, just set the queue has been configured.
2728          */
2729         rxq->q_set = TRUE;
2730         pf->fdir.rxq = rxq;
2731
2732         return I40E_SUCCESS;
2733 }
2734
2735 void
2736 i40e_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
2737         struct rte_eth_rxq_info *qinfo)
2738 {
2739         struct i40e_rx_queue *rxq;
2740
2741         rxq = dev->data->rx_queues[queue_id];
2742
2743         qinfo->mp = rxq->mp;
2744         qinfo->scattered_rx = dev->data->scattered_rx;
2745         qinfo->nb_desc = rxq->nb_rx_desc;
2746
2747         qinfo->conf.rx_free_thresh = rxq->rx_free_thresh;
2748         qinfo->conf.rx_drop_en = rxq->drop_en;
2749         qinfo->conf.rx_deferred_start = rxq->rx_deferred_start;
2750 }
2751
2752 void
2753 i40e_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
2754         struct rte_eth_txq_info *qinfo)
2755 {
2756         struct i40e_tx_queue *txq;
2757
2758         txq = dev->data->tx_queues[queue_id];
2759
2760         qinfo->nb_desc = txq->nb_tx_desc;
2761
2762         qinfo->conf.tx_thresh.pthresh = txq->pthresh;
2763         qinfo->conf.tx_thresh.hthresh = txq->hthresh;
2764         qinfo->conf.tx_thresh.wthresh = txq->wthresh;
2765
2766         qinfo->conf.tx_free_thresh = txq->tx_free_thresh;
2767         qinfo->conf.tx_rs_thresh = txq->tx_rs_thresh;
2768         qinfo->conf.txq_flags = txq->txq_flags;
2769         qinfo->conf.tx_deferred_start = txq->tx_deferred_start;
2770 }
2771
2772 void __attribute__((cold))
2773 i40e_set_rx_function(struct rte_eth_dev *dev)
2774 {
2775         struct i40e_adapter *ad =
2776                 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
2777         uint16_t rx_using_sse, i;
2778         /* In order to allow Vector Rx there are a few configuration
2779          * conditions to be met and Rx Bulk Allocation should be allowed.
2780          */
2781         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
2782                 if (i40e_rx_vec_dev_conf_condition_check(dev) ||
2783                     !ad->rx_bulk_alloc_allowed) {
2784                         PMD_INIT_LOG(DEBUG, "Port[%d] doesn't meet"
2785                                      " Vector Rx preconditions",
2786                                      dev->data->port_id);
2787
2788                         ad->rx_vec_allowed = false;
2789                 }
2790                 if (ad->rx_vec_allowed) {
2791                         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2792                                 struct i40e_rx_queue *rxq =
2793                                         dev->data->rx_queues[i];
2794
2795                                 if (rxq && i40e_rxq_vec_setup(rxq)) {
2796                                         ad->rx_vec_allowed = false;
2797                                         break;
2798                                 }
2799                         }
2800                 }
2801         }
2802
2803         if (dev->data->scattered_rx) {
2804                 /* Set the non-LRO scattered callback: there are Vector and
2805                  * single allocation versions.
2806                  */
2807                 if (ad->rx_vec_allowed) {
2808                         PMD_INIT_LOG(DEBUG, "Using Vector Scattered Rx "
2809                                             "callback (port=%d).",
2810                                      dev->data->port_id);
2811
2812                         dev->rx_pkt_burst = i40e_recv_scattered_pkts_vec;
2813                 } else {
2814                         PMD_INIT_LOG(DEBUG, "Using a Scattered with bulk "
2815                                            "allocation callback (port=%d).",
2816                                      dev->data->port_id);
2817                         dev->rx_pkt_burst = i40e_recv_scattered_pkts;
2818                 }
2819         /* If parameters allow we are going to choose between the following
2820          * callbacks:
2821          *    - Vector
2822          *    - Bulk Allocation
2823          *    - Single buffer allocation (the simplest one)
2824          */
2825         } else if (ad->rx_vec_allowed) {
2826                 PMD_INIT_LOG(DEBUG, "Vector rx enabled, please make sure RX "
2827                                     "burst size no less than %d (port=%d).",
2828                              RTE_I40E_DESCS_PER_LOOP,
2829                              dev->data->port_id);
2830
2831                 dev->rx_pkt_burst = i40e_recv_pkts_vec;
2832         } else if (ad->rx_bulk_alloc_allowed) {
2833                 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are "
2834                                     "satisfied. Rx Burst Bulk Alloc function "
2835                                     "will be used on port=%d.",
2836                              dev->data->port_id);
2837
2838                 dev->rx_pkt_burst = i40e_recv_pkts_bulk_alloc;
2839         } else {
2840                 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are not "
2841                                     "satisfied, or Scattered Rx is requested "
2842                                     "(port=%d).",
2843                              dev->data->port_id);
2844
2845                 dev->rx_pkt_burst = i40e_recv_pkts;
2846         }
2847
2848         /* Propagate information about RX function choice through all queues. */
2849         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
2850                 rx_using_sse =
2851                         (dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec ||
2852                          dev->rx_pkt_burst == i40e_recv_pkts_vec);
2853
2854                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
2855                         struct i40e_rx_queue *rxq = dev->data->rx_queues[i];
2856
2857                         if (rxq)
2858                                 rxq->rx_using_sse = rx_using_sse;
2859                 }
2860         }
2861 }
2862
2863 void __attribute__((cold))
2864 i40e_set_tx_function_flag(struct rte_eth_dev *dev, struct i40e_tx_queue *txq)
2865 {
2866         struct i40e_adapter *ad =
2867                 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
2868
2869         /* Use a simple Tx queue (no offloads, no multi segs) if possible */
2870         if (((txq->txq_flags & I40E_SIMPLE_FLAGS) == I40E_SIMPLE_FLAGS)
2871                         && (txq->tx_rs_thresh >= RTE_PMD_I40E_TX_MAX_BURST)) {
2872                 if (txq->tx_rs_thresh <= RTE_I40E_TX_MAX_FREE_BUF_SZ) {
2873                         PMD_INIT_LOG(DEBUG, "Vector tx"
2874                                      " can be enabled on this txq.");
2875
2876                 } else {
2877                         ad->tx_vec_allowed = false;
2878                 }
2879         } else {
2880                 ad->tx_simple_allowed = false;
2881         }
2882 }
2883
2884 void __attribute__((cold))
2885 i40e_set_tx_function(struct rte_eth_dev *dev)
2886 {
2887         struct i40e_adapter *ad =
2888                 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
2889         int i;
2890
2891         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
2892                 if (ad->tx_vec_allowed) {
2893                         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2894                                 struct i40e_tx_queue *txq =
2895                                         dev->data->tx_queues[i];
2896
2897                                 if (txq && i40e_txq_vec_setup(txq)) {
2898                                         ad->tx_vec_allowed = false;
2899                                         break;
2900                                 }
2901                         }
2902                 }
2903         }
2904
2905         if (ad->tx_simple_allowed) {
2906                 if (ad->tx_vec_allowed) {
2907                         PMD_INIT_LOG(DEBUG, "Vector tx finally be used.");
2908                         dev->tx_pkt_burst = i40e_xmit_pkts_vec;
2909                 } else {
2910                         PMD_INIT_LOG(DEBUG, "Simple tx finally be used.");
2911                         dev->tx_pkt_burst = i40e_xmit_pkts_simple;
2912                 }
2913                 dev->tx_pkt_prepare = NULL;
2914         } else {
2915                 PMD_INIT_LOG(DEBUG, "Xmit tx finally be used.");
2916                 dev->tx_pkt_burst = i40e_xmit_pkts;
2917                 dev->tx_pkt_prepare = i40e_prep_pkts;
2918         }
2919 }
2920
2921 void __attribute__((cold))
2922 i40e_set_default_ptype_table(struct rte_eth_dev *dev)
2923 {
2924         struct i40e_adapter *ad =
2925                 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
2926         int i;
2927
2928         for (i = 0; i < I40E_MAX_PKT_TYPE; i++)
2929                 ad->ptype_tbl[i] = i40e_get_default_pkt_type(i);
2930 }
2931
2932 void __attribute__((cold))
2933 i40e_set_default_pctype_table(struct rte_eth_dev *dev)
2934 {
2935         struct i40e_adapter *ad =
2936                         I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
2937         struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2938         int i;
2939
2940         for (i = 0; i < I40E_FLOW_TYPE_MAX; i++)
2941                 ad->pctypes_tbl[i] = 0ULL;
2942         ad->flow_types_mask = 0ULL;
2943         ad->pctypes_mask = 0ULL;
2944
2945         ad->pctypes_tbl[RTE_ETH_FLOW_FRAG_IPV4] =
2946                                 (1ULL << I40E_FILTER_PCTYPE_FRAG_IPV4);
2947         ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV4_UDP] =
2948                                 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_UDP);
2949         ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV4_TCP] =
2950                                 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP);
2951         ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV4_SCTP] =
2952                                 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP);
2953         ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV4_OTHER] =
2954                                 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER);
2955         ad->pctypes_tbl[RTE_ETH_FLOW_FRAG_IPV6] =
2956                                 (1ULL << I40E_FILTER_PCTYPE_FRAG_IPV6);
2957         ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV6_UDP] =
2958                                 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_UDP);
2959         ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV6_TCP] =
2960                                 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP);
2961         ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV6_SCTP] =
2962                                 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP);
2963         ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV6_OTHER] =
2964                                 (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER);
2965         ad->pctypes_tbl[RTE_ETH_FLOW_L2_PAYLOAD] =
2966                                 (1ULL << I40E_FILTER_PCTYPE_L2_PAYLOAD);
2967
2968         if (hw->mac.type == I40E_MAC_X722) {
2969                 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV4_UDP] |=
2970                         (1ULL << I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP);
2971                 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV4_UDP] |=
2972                         (1ULL << I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP);
2973                 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV4_TCP] |=
2974                         (1ULL << I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK);
2975                 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV6_UDP] |=
2976                         (1ULL << I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP);
2977                 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV6_UDP] |=
2978                         (1ULL << I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP);
2979                 ad->pctypes_tbl[RTE_ETH_FLOW_NONFRAG_IPV6_TCP] |=
2980                         (1ULL << I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK);
2981         }
2982
2983         for (i = 0; i < I40E_FLOW_TYPE_MAX; i++) {
2984                 if (ad->pctypes_tbl[i])
2985                         ad->flow_types_mask |= (1ULL << i);
2986                 ad->pctypes_mask |= ad->pctypes_tbl[i];
2987         }
2988 }
2989
2990 /* Stubs needed for linkage when CONFIG_RTE_I40E_INC_VECTOR is set to 'n' */
2991 int __attribute__((weak))
2992 i40e_rx_vec_dev_conf_condition_check(struct rte_eth_dev __rte_unused *dev)
2993 {
2994         return -1;
2995 }
2996
2997 uint16_t __attribute__((weak))
2998 i40e_recv_pkts_vec(
2999         void __rte_unused *rx_queue,
3000         struct rte_mbuf __rte_unused **rx_pkts,
3001         uint16_t __rte_unused nb_pkts)
3002 {
3003         return 0;
3004 }
3005
3006 uint16_t __attribute__((weak))
3007 i40e_recv_scattered_pkts_vec(
3008         void __rte_unused *rx_queue,
3009         struct rte_mbuf __rte_unused **rx_pkts,
3010         uint16_t __rte_unused nb_pkts)
3011 {
3012         return 0;
3013 }
3014
3015 int __attribute__((weak))
3016 i40e_rxq_vec_setup(struct i40e_rx_queue __rte_unused *rxq)
3017 {
3018         return -1;
3019 }
3020
3021 int __attribute__((weak))
3022 i40e_txq_vec_setup(struct i40e_tx_queue __rte_unused *txq)
3023 {
3024         return -1;
3025 }
3026
3027 void __attribute__((weak))
3028 i40e_rx_queue_release_mbufs_vec(struct i40e_rx_queue __rte_unused*rxq)
3029 {
3030         return;
3031 }
3032
3033 uint16_t __attribute__((weak))
3034 i40e_xmit_fixed_burst_vec(void __rte_unused * tx_queue,
3035                           struct rte_mbuf __rte_unused **tx_pkts,
3036                           uint16_t __rte_unused nb_pkts)
3037 {
3038         return 0;
3039 }