drivers/net/iavf/iavf_rxtx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <stdlib.h>
   7 #include <string.h>
   8 #include <errno.h>
   9 #include <stdint.h>
  10 #include <stdarg.h>
  11 #include <unistd.h>
  12 #include <inttypes.h>
  13 #include <sys/queue.h>
  14
  15 #include <rte_string_fns.h>
  16 #include <rte_memzone.h>
  17 #include <rte_mbuf.h>
  18 #include <rte_malloc.h>
  19 #include <rte_ether.h>
  20 #include <ethdev_driver.h>
  21 #include <rte_tcp.h>
  22 #include <rte_sctp.h>
  23 #include <rte_udp.h>
  24 #include <rte_ip.h>
  25 #include <rte_net.h>
  26 #include <rte_vect.h>
  27
  28 #include "iavf.h"
  29 #include "iavf_rxtx.h"
  30 #include "iavf_ipsec_crypto.h"
  31 #include "rte_pmd_iavf.h"
  32
  33 /* Offset of mbuf dynamic field for protocol extraction's metadata */
  34 int rte_pmd_ifd_dynfield_proto_xtr_metadata_offs = -1;
  35
  36 /* Mask of mbuf dynamic flags for protocol extraction's type */
  37 uint64_t rte_pmd_ifd_dynflag_proto_xtr_vlan_mask;
  38 uint64_t rte_pmd_ifd_dynflag_proto_xtr_ipv4_mask;
  39 uint64_t rte_pmd_ifd_dynflag_proto_xtr_ipv6_mask;
  40 uint64_t rte_pmd_ifd_dynflag_proto_xtr_ipv6_flow_mask;
  41 uint64_t rte_pmd_ifd_dynflag_proto_xtr_tcp_mask;
  42 uint64_t rte_pmd_ifd_dynflag_proto_xtr_ip_offset_mask;
  43 uint64_t rte_pmd_ifd_dynflag_proto_xtr_ipsec_crypto_said_mask;
  44
  45 uint8_t
  46 iavf_proto_xtr_type_to_rxdid(uint8_t flex_type)
  47 {
  48         static uint8_t rxdid_map[] = {
  49                 [IAVF_PROTO_XTR_NONE]      = IAVF_RXDID_COMMS_OVS_1,
  50                 [IAVF_PROTO_XTR_VLAN]      = IAVF_RXDID_COMMS_AUX_VLAN,
  51                 [IAVF_PROTO_XTR_IPV4]      = IAVF_RXDID_COMMS_AUX_IPV4,
  52                 [IAVF_PROTO_XTR_IPV6]      = IAVF_RXDID_COMMS_AUX_IPV6,
  53                 [IAVF_PROTO_XTR_IPV6_FLOW] = IAVF_RXDID_COMMS_AUX_IPV6_FLOW,
  54                 [IAVF_PROTO_XTR_TCP]       = IAVF_RXDID_COMMS_AUX_TCP,
  55                 [IAVF_PROTO_XTR_IP_OFFSET] = IAVF_RXDID_COMMS_AUX_IP_OFFSET,
  56                 [IAVF_PROTO_XTR_IPSEC_CRYPTO_SAID] =
  57                                 IAVF_RXDID_COMMS_IPSEC_CRYPTO,
  58         };
  59
  60         return flex_type < RTE_DIM(rxdid_map) ?
  61                                 rxdid_map[flex_type] : IAVF_RXDID_COMMS_OVS_1;
  62 }
  63
  64 static int
  65 iavf_monitor_callback(const uint64_t value,
  66                 const uint64_t arg[RTE_POWER_MONITOR_OPAQUE_SZ] __rte_unused)
  67 {
  68         const uint64_t m = rte_cpu_to_le_64(1 << IAVF_RX_DESC_STATUS_DD_SHIFT);
  69         /*
  70          * we expect the DD bit to be set to 1 if this descriptor was already
  71          * written to.
  72          */
  73         return (value & m) == m ? -1 : 0;
  74 }
  75
  76 int
  77 iavf_get_monitor_addr(void *rx_queue, struct rte_power_monitor_cond *pmc)
  78 {
  79         struct iavf_rx_queue *rxq = rx_queue;
  80         volatile union iavf_rx_desc *rxdp;
  81         uint16_t desc;
  82
  83         desc = rxq->rx_tail;
  84         rxdp = &rxq->rx_ring[desc];
  85         /* watch for changes in status bit */
  86         pmc->addr = &rxdp->wb.qword1.status_error_len;
  87
  88         /* comparison callback */
  89         pmc->fn = iavf_monitor_callback;
  90
  91         /* registers are 64-bit */
  92         pmc->size = sizeof(uint64_t);
  93
  94         return 0;
  95 }
  96
  97 static inline int
  98 check_rx_thresh(uint16_t nb_desc, uint16_t thresh)
  99 {
 100         /* The following constraints must be satisfied:
 101          *   thresh < rxq->nb_rx_desc
 102          */
 103         if (thresh >= nb_desc) {
 104                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be less than %u",
 105                              thresh, nb_desc);
 106                 return -EINVAL;
 107         }
 108         return 0;
 109 }
 110
 111 static inline int
 112 check_tx_thresh(uint16_t nb_desc, uint16_t tx_rs_thresh,
 113                 uint16_t tx_free_thresh)
 114 {
 115         /* TX descriptors will have their RS bit set after tx_rs_thresh
 116          * descriptors have been used. The TX descriptor ring will be cleaned
 117          * after tx_free_thresh descriptors are used or if the number of
 118          * descriptors required to transmit a packet is greater than the
 119          * number of free TX descriptors.
 120          *
 121          * The following constraints must be satisfied:
 122          *  - tx_rs_thresh must be less than the size of the ring minus 2.
 123          *  - tx_free_thresh must be less than the size of the ring minus 3.
 124          *  - tx_rs_thresh must be less than or equal to tx_free_thresh.
 125          *  - tx_rs_thresh must be a divisor of the ring size.
 126          *
 127          * One descriptor in the TX ring is used as a sentinel to avoid a H/W
 128          * race condition, hence the maximum threshold constraints. When set
 129          * to zero use default values.
 130          */
 131         if (tx_rs_thresh >= (nb_desc - 2)) {
 132                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be less than the "
 133                              "number of TX descriptors (%u) minus 2",
 134                              tx_rs_thresh, nb_desc);
 135                 return -EINVAL;
 136         }
 137         if (tx_free_thresh >= (nb_desc - 3)) {
 138                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be less than the "
 139                              "number of TX descriptors (%u) minus 3.",
 140                              tx_free_thresh, nb_desc);
 141                 return -EINVAL;
 142         }
 143         if (tx_rs_thresh > tx_free_thresh) {
 144                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be less than or "
 145                              "equal to tx_free_thresh (%u).",
 146                              tx_rs_thresh, tx_free_thresh);
 147                 return -EINVAL;
 148         }
 149         if ((nb_desc % tx_rs_thresh) != 0) {
 150                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be a divisor of the "
 151                              "number of TX descriptors (%u).",
 152                              tx_rs_thresh, nb_desc);
 153                 return -EINVAL;
 154         }
 155
 156         return 0;
 157 }
 158
 159 static inline bool
 160 check_rx_vec_allow(struct iavf_rx_queue *rxq)
 161 {
 162         if (rxq->rx_free_thresh >= IAVF_VPMD_RX_MAX_BURST &&
 163             rxq->nb_rx_desc % rxq->rx_free_thresh == 0) {
 164                 PMD_INIT_LOG(DEBUG, "Vector Rx can be enabled on this rxq.");
 165                 return true;
 166         }
 167
 168         PMD_INIT_LOG(DEBUG, "Vector Rx cannot be enabled on this rxq.");
 169         return false;
 170 }
 171
 172 static inline bool
 173 check_tx_vec_allow(struct iavf_tx_queue *txq)
 174 {
 175         if (!(txq->offloads & IAVF_TX_NO_VECTOR_FLAGS) &&
 176             txq->rs_thresh >= IAVF_VPMD_TX_MAX_BURST &&
 177             txq->rs_thresh <= IAVF_VPMD_TX_MAX_FREE_BUF) {
 178                 PMD_INIT_LOG(DEBUG, "Vector tx can be enabled on this txq.");
 179                 return true;
 180         }
 181         PMD_INIT_LOG(DEBUG, "Vector Tx cannot be enabled on this txq.");
 182         return false;
 183 }
 184
 185 static inline bool
 186 check_rx_bulk_allow(struct iavf_rx_queue *rxq)
 187 {
 188         int ret = true;
 189
 190         if (!(rxq->rx_free_thresh >= IAVF_RX_MAX_BURST)) {
 191                 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
 192                              "rxq->rx_free_thresh=%d, "
 193                              "IAVF_RX_MAX_BURST=%d",
 194                              rxq->rx_free_thresh, IAVF_RX_MAX_BURST);
 195                 ret = false;
 196         } else if (rxq->nb_rx_desc % rxq->rx_free_thresh != 0) {
 197                 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
 198                              "rxq->nb_rx_desc=%d, "
 199                              "rxq->rx_free_thresh=%d",
 200                              rxq->nb_rx_desc, rxq->rx_free_thresh);
 201                 ret = false;
 202         }
 203         return ret;
 204 }
 205
 206 static inline void
 207 reset_rx_queue(struct iavf_rx_queue *rxq)
 208 {
 209         uint16_t len;
 210         uint32_t i;
 211
 212         if (!rxq)
 213                 return;
 214
 215         len = rxq->nb_rx_desc + IAVF_RX_MAX_BURST;
 216
 217         for (i = 0; i < len * sizeof(union iavf_rx_desc); i++)
 218                 ((volatile char *)rxq->rx_ring)[i] = 0;
 219
 220         memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf));
 221
 222         for (i = 0; i < IAVF_RX_MAX_BURST; i++)
 223                 rxq->sw_ring[rxq->nb_rx_desc + i] = &rxq->fake_mbuf;
 224
 225         /* for rx bulk */
 226         rxq->rx_nb_avail = 0;
 227         rxq->rx_next_avail = 0;
 228         rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1);
 229
 230         rxq->rx_tail = 0;
 231         rxq->nb_rx_hold = 0;
 232
 233         rte_pktmbuf_free(rxq->pkt_first_seg);
 234
 235         rxq->pkt_first_seg = NULL;
 236         rxq->pkt_last_seg = NULL;
 237         rxq->rxrearm_nb = 0;
 238         rxq->rxrearm_start = 0;
 239 }
 240
 241 static inline void
 242 reset_tx_queue(struct iavf_tx_queue *txq)
 243 {
 244         struct iavf_tx_entry *txe;
 245         uint32_t i, size;
 246         uint16_t prev;
 247
 248         if (!txq) {
 249                 PMD_DRV_LOG(DEBUG, "Pointer to txq is NULL");
 250                 return;
 251         }
 252
 253         txe = txq->sw_ring;
 254         size = sizeof(struct iavf_tx_desc) * txq->nb_tx_desc;
 255         for (i = 0; i < size; i++)
 256                 ((volatile char *)txq->tx_ring)[i] = 0;
 257
 258         prev = (uint16_t)(txq->nb_tx_desc - 1);
 259         for (i = 0; i < txq->nb_tx_desc; i++) {
 260                 txq->tx_ring[i].cmd_type_offset_bsz =
 261                         rte_cpu_to_le_64(IAVF_TX_DESC_DTYPE_DESC_DONE);
 262                 txe[i].mbuf =  NULL;
 263                 txe[i].last_id = i;
 264                 txe[prev].next_id = i;
 265                 prev = i;
 266         }
 267
 268         txq->tx_tail = 0;
 269         txq->nb_used = 0;
 270
 271         txq->last_desc_cleaned = txq->nb_tx_desc - 1;
 272         txq->nb_free = txq->nb_tx_desc - 1;
 273
 274         txq->next_dd = txq->rs_thresh - 1;
 275         txq->next_rs = txq->rs_thresh - 1;
 276 }
 277
 278 static int
 279 alloc_rxq_mbufs(struct iavf_rx_queue *rxq)
 280 {
 281         volatile union iavf_rx_desc *rxd;
 282         struct rte_mbuf *mbuf = NULL;
 283         uint64_t dma_addr;
 284         uint16_t i, j;
 285
 286         for (i = 0; i < rxq->nb_rx_desc; i++) {
 287                 mbuf = rte_mbuf_raw_alloc(rxq->mp);
 288                 if (unlikely(!mbuf)) {
 289                         for (j = 0; j < i; j++) {
 290                                 rte_pktmbuf_free_seg(rxq->sw_ring[j]);
 291                                 rxq->sw_ring[j] = NULL;
 292                         }
 293                         PMD_DRV_LOG(ERR, "Failed to allocate mbuf for RX");
 294                         return -ENOMEM;
 295                 }
 296
 297                 rte_mbuf_refcnt_set(mbuf, 1);
 298                 mbuf->next = NULL;
 299                 mbuf->data_off = RTE_PKTMBUF_HEADROOM;
 300                 mbuf->nb_segs = 1;
 301                 mbuf->port = rxq->port_id;
 302
 303                 dma_addr =
 304                         rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf));
 305
 306                 rxd = &rxq->rx_ring[i];
 307                 rxd->read.pkt_addr = dma_addr;
 308                 rxd->read.hdr_addr = 0;
 309 #ifndef RTE_LIBRTE_IAVF_16BYTE_RX_DESC
 310                 rxd->read.rsvd1 = 0;
 311                 rxd->read.rsvd2 = 0;
 312 #endif
 313
 314                 rxq->sw_ring[i] = mbuf;
 315         }
 316
 317         return 0;
 318 }
 319
 320 static inline void
 321 release_rxq_mbufs(struct iavf_rx_queue *rxq)
 322 {
 323         uint16_t i;
 324
 325         if (!rxq->sw_ring)
 326                 return;
 327
 328         for (i = 0; i < rxq->nb_rx_desc; i++) {
 329                 if (rxq->sw_ring[i]) {
 330                         rte_pktmbuf_free_seg(rxq->sw_ring[i]);
 331                         rxq->sw_ring[i] = NULL;
 332                 }
 333         }
 334
 335         /* for rx bulk */
 336         if (rxq->rx_nb_avail == 0)
 337                 return;
 338         for (i = 0; i < rxq->rx_nb_avail; i++) {
 339                 struct rte_mbuf *mbuf;
 340
 341                 mbuf = rxq->rx_stage[rxq->rx_next_avail + i];
 342                 rte_pktmbuf_free_seg(mbuf);
 343         }
 344         rxq->rx_nb_avail = 0;
 345 }
 346
 347 static inline void
 348 release_txq_mbufs(struct iavf_tx_queue *txq)
 349 {
 350         uint16_t i;
 351
 352         if (!txq || !txq->sw_ring) {
 353                 PMD_DRV_LOG(DEBUG, "Pointer to rxq or sw_ring is NULL");
 354                 return;
 355         }
 356
 357         for (i = 0; i < txq->nb_tx_desc; i++) {
 358                 if (txq->sw_ring[i].mbuf) {
 359                         rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
 360                         txq->sw_ring[i].mbuf = NULL;
 361                 }
 362         }
 363 }
 364
 365 static const
 366 struct iavf_rxq_ops iavf_rxq_release_mbufs_ops[] = {
 367         [IAVF_REL_MBUFS_DEFAULT].release_mbufs = release_rxq_mbufs,
 368 #ifdef RTE_ARCH_X86
 369         [IAVF_REL_MBUFS_SSE_VEC].release_mbufs = iavf_rx_queue_release_mbufs_sse,
 370 #endif
 371 };
 372
 373 static const
 374 struct iavf_txq_ops iavf_txq_release_mbufs_ops[] = {
 375         [IAVF_REL_MBUFS_DEFAULT].release_mbufs = release_txq_mbufs,
 376 #ifdef RTE_ARCH_X86
 377         [IAVF_REL_MBUFS_SSE_VEC].release_mbufs = iavf_tx_queue_release_mbufs_sse,
 378 #ifdef CC_AVX512_SUPPORT
 379         [IAVF_REL_MBUFS_AVX512_VEC].release_mbufs = iavf_tx_queue_release_mbufs_avx512,
 380 #endif
 381 #endif
 382
 383 };
 384
 385 static inline void
 386 iavf_rxd_to_pkt_fields_by_comms_ovs(__rte_unused struct iavf_rx_queue *rxq,
 387                                     struct rte_mbuf *mb,
 388                                     volatile union iavf_rx_flex_desc *rxdp)
 389 {
 390         volatile struct iavf_32b_rx_flex_desc_comms_ovs *desc =
 391                         (volatile struct iavf_32b_rx_flex_desc_comms_ovs *)rxdp;
 392 #ifndef RTE_LIBRTE_IAVF_16BYTE_RX_DESC
 393         uint16_t stat_err;
 394 #endif
 395
 396         if (desc->flow_id != 0xFFFFFFFF) {
 397                 mb->ol_flags |= RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID;
 398                 mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id);
 399         }
 400
 401 #ifndef RTE_LIBRTE_IAVF_16BYTE_RX_DESC
 402         stat_err = rte_le_to_cpu_16(desc->status_error0);
 403         if (likely(stat_err & (1 << IAVF_RX_FLEX_DESC_STATUS0_RSS_VALID_S))) {
 404                 mb->ol_flags |= RTE_MBUF_F_RX_RSS_HASH;
 405                 mb->hash.rss = rte_le_to_cpu_32(desc->rss_hash);
 406         }
 407 #endif
 408 }
 409
 410 static inline void
 411 iavf_rxd_to_pkt_fields_by_comms_aux_v1(struct iavf_rx_queue *rxq,
 412                                        struct rte_mbuf *mb,
 413                                        volatile union iavf_rx_flex_desc *rxdp)
 414 {
 415         volatile struct iavf_32b_rx_flex_desc_comms *desc =
 416                         (volatile struct iavf_32b_rx_flex_desc_comms *)rxdp;
 417         uint16_t stat_err;
 418
 419         stat_err = rte_le_to_cpu_16(desc->status_error0);
 420         if (likely(stat_err & (1 << IAVF_RX_FLEX_DESC_STATUS0_RSS_VALID_S))) {
 421                 mb->ol_flags |= RTE_MBUF_F_RX_RSS_HASH;
 422                 mb->hash.rss = rte_le_to_cpu_32(desc->rss_hash);
 423         }
 424
 425 #ifndef RTE_LIBRTE_IAVF_16BYTE_RX_DESC
 426         if (desc->flow_id != 0xFFFFFFFF) {
 427                 mb->ol_flags |= RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID;
 428                 mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id);
 429         }
 430
 431         if (rxq->xtr_ol_flag) {
 432                 uint32_t metadata = 0;
 433
 434                 stat_err = rte_le_to_cpu_16(desc->status_error1);
 435
 436                 if (stat_err & (1 << IAVF_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S))
 437                         metadata = rte_le_to_cpu_16(desc->flex_ts.flex.aux0);
 438
 439                 if (stat_err & (1 << IAVF_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S))
 440                         metadata |=
 441                                 rte_le_to_cpu_16(desc->flex_ts.flex.aux1) << 16;
 442
 443                 if (metadata) {
 444                         mb->ol_flags |= rxq->xtr_ol_flag;
 445
 446                         *RTE_PMD_IFD_DYNF_PROTO_XTR_METADATA(mb) = metadata;
 447                 }
 448         }
 449 #endif
 450 }
 451
 452 static inline void
 453 iavf_rxd_to_pkt_fields_by_comms_aux_v2(struct iavf_rx_queue *rxq,
 454                                        struct rte_mbuf *mb,
 455                                        volatile union iavf_rx_flex_desc *rxdp)
 456 {
 457         volatile struct iavf_32b_rx_flex_desc_comms *desc =
 458                         (volatile struct iavf_32b_rx_flex_desc_comms *)rxdp;
 459         uint16_t stat_err;
 460
 461         stat_err = rte_le_to_cpu_16(desc->status_error0);
 462         if (likely(stat_err & (1 << IAVF_RX_FLEX_DESC_STATUS0_RSS_VALID_S))) {
 463                 mb->ol_flags |= RTE_MBUF_F_RX_RSS_HASH;
 464                 mb->hash.rss = rte_le_to_cpu_32(desc->rss_hash);
 465         }
 466
 467 #ifndef RTE_LIBRTE_IAVF_16BYTE_RX_DESC
 468         if (desc->flow_id != 0xFFFFFFFF) {
 469                 mb->ol_flags |= RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID;
 470                 mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id);
 471         }
 472
 473         if (rxq->xtr_ol_flag) {
 474                 uint32_t metadata = 0;
 475
 476                 if (desc->flex_ts.flex.aux0 != 0xFFFF)
 477                         metadata = rte_le_to_cpu_16(desc->flex_ts.flex.aux0);
 478                 else if (desc->flex_ts.flex.aux1 != 0xFFFF)
 479                         metadata = rte_le_to_cpu_16(desc->flex_ts.flex.aux1);
 480
 481                 if (metadata) {
 482                         mb->ol_flags |= rxq->xtr_ol_flag;
 483
 484                         *RTE_PMD_IFD_DYNF_PROTO_XTR_METADATA(mb) = metadata;
 485                 }
 486         }
 487 #endif
 488 }
 489
 490 static const
 491 iavf_rxd_to_pkt_fields_t rxd_to_pkt_fields_ops[IAVF_RXDID_LAST + 1] = {
 492         [IAVF_RXDID_LEGACY_0] = iavf_rxd_to_pkt_fields_by_comms_ovs,
 493         [IAVF_RXDID_LEGACY_1] = iavf_rxd_to_pkt_fields_by_comms_ovs,
 494         [IAVF_RXDID_COMMS_AUX_VLAN] = iavf_rxd_to_pkt_fields_by_comms_aux_v1,
 495         [IAVF_RXDID_COMMS_AUX_IPV4] = iavf_rxd_to_pkt_fields_by_comms_aux_v1,
 496         [IAVF_RXDID_COMMS_AUX_IPV6] = iavf_rxd_to_pkt_fields_by_comms_aux_v1,
 497         [IAVF_RXDID_COMMS_AUX_IPV6_FLOW] =
 498                 iavf_rxd_to_pkt_fields_by_comms_aux_v1,
 499         [IAVF_RXDID_COMMS_AUX_TCP] = iavf_rxd_to_pkt_fields_by_comms_aux_v1,
 500         [IAVF_RXDID_COMMS_AUX_IP_OFFSET] =
 501                 iavf_rxd_to_pkt_fields_by_comms_aux_v2,
 502         [IAVF_RXDID_COMMS_IPSEC_CRYPTO] =
 503                 iavf_rxd_to_pkt_fields_by_comms_aux_v2,
 504         [IAVF_RXDID_COMMS_OVS_1] = iavf_rxd_to_pkt_fields_by_comms_ovs,
 505 };
 506
 507 static void
 508 iavf_select_rxd_to_pkt_fields_handler(struct iavf_rx_queue *rxq, uint32_t rxdid)
 509 {
 510         rxq->rxdid = rxdid;
 511
 512         switch (rxdid) {
 513         case IAVF_RXDID_COMMS_AUX_VLAN:
 514                 rxq->xtr_ol_flag = rte_pmd_ifd_dynflag_proto_xtr_vlan_mask;
 515                 break;
 516         case IAVF_RXDID_COMMS_AUX_IPV4:
 517                 rxq->xtr_ol_flag = rte_pmd_ifd_dynflag_proto_xtr_ipv4_mask;
 518                 break;
 519         case IAVF_RXDID_COMMS_AUX_IPV6:
 520                 rxq->xtr_ol_flag = rte_pmd_ifd_dynflag_proto_xtr_ipv6_mask;
 521                 break;
 522         case IAVF_RXDID_COMMS_AUX_IPV6_FLOW:
 523                 rxq->xtr_ol_flag =
 524                         rte_pmd_ifd_dynflag_proto_xtr_ipv6_flow_mask;
 525                 break;
 526         case IAVF_RXDID_COMMS_AUX_TCP:
 527                 rxq->xtr_ol_flag = rte_pmd_ifd_dynflag_proto_xtr_tcp_mask;
 528                 break;
 529         case IAVF_RXDID_COMMS_AUX_IP_OFFSET:
 530                 rxq->xtr_ol_flag =
 531                         rte_pmd_ifd_dynflag_proto_xtr_ip_offset_mask;
 532                 break;
 533         case IAVF_RXDID_COMMS_IPSEC_CRYPTO:
 534                 rxq->xtr_ol_flag =
 535                         rte_pmd_ifd_dynflag_proto_xtr_ipsec_crypto_said_mask;
 536                 break;
 537         case IAVF_RXDID_COMMS_OVS_1:
 538         case IAVF_RXDID_LEGACY_0:
 539         case IAVF_RXDID_LEGACY_1:
 540                 break;
 541         default:
 542                 /* update this according to the RXDID for FLEX_DESC_NONE */
 543                 rxq->rxdid = IAVF_RXDID_COMMS_OVS_1;
 544                 break;
 545         }
 546
 547         if (!rte_pmd_ifd_dynf_proto_xtr_metadata_avail())
 548                 rxq->xtr_ol_flag = 0;
 549 }
 550
 551 int
 552 iavf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 553                        uint16_t nb_desc, unsigned int socket_id,
 554                        const struct rte_eth_rxconf *rx_conf,
 555                        struct rte_mempool *mp)
 556 {
 557         struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 558         struct iavf_adapter *ad =
 559                 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 560         struct iavf_info *vf =
 561                 IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 562         struct iavf_vsi *vsi = &vf->vsi;
 563         struct iavf_rx_queue *rxq;
 564         const struct rte_memzone *mz;
 565         uint32_t ring_size;
 566         uint8_t proto_xtr;
 567         uint16_t len;
 568         uint16_t rx_free_thresh;
 569         uint64_t offloads;
 570
 571         PMD_INIT_FUNC_TRACE();
 572
 573         if (ad->closed)
 574                 return -EIO;
 575
 576         offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads;
 577
 578         if (nb_desc % IAVF_ALIGN_RING_DESC != 0 ||
 579             nb_desc > IAVF_MAX_RING_DESC ||
 580             nb_desc < IAVF_MIN_RING_DESC) {
 581                 PMD_INIT_LOG(ERR, "Number (%u) of receive descriptors is "
 582                              "invalid", nb_desc);
 583                 return -EINVAL;
 584         }
 585
 586         /* Check free threshold */
 587         rx_free_thresh = (rx_conf->rx_free_thresh == 0) ?
 588                          IAVF_DEFAULT_RX_FREE_THRESH :
 589                          rx_conf->rx_free_thresh;
 590         if (check_rx_thresh(nb_desc, rx_free_thresh) != 0)
 591                 return -EINVAL;
 592
 593         /* Free memory if needed */
 594         if (dev->data->rx_queues[queue_idx]) {
 595                 iavf_dev_rx_queue_release(dev, queue_idx);
 596                 dev->data->rx_queues[queue_idx] = NULL;
 597         }
 598
 599         /* Allocate the rx queue data structure */
 600         rxq = rte_zmalloc_socket("iavf rxq",
 601                                  sizeof(struct iavf_rx_queue),
 602                                  RTE_CACHE_LINE_SIZE,
 603                                  socket_id);
 604         if (!rxq) {
 605                 PMD_INIT_LOG(ERR, "Failed to allocate memory for "
 606                              "rx queue data structure");
 607                 return -ENOMEM;
 608         }
 609
 610         if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) {
 611                 proto_xtr = vf->proto_xtr ? vf->proto_xtr[queue_idx] :
 612                                 IAVF_PROTO_XTR_NONE;
 613                 rxq->rxdid = iavf_proto_xtr_type_to_rxdid(proto_xtr);
 614                 rxq->proto_xtr = proto_xtr;
 615         } else {
 616                 rxq->rxdid = IAVF_RXDID_LEGACY_1;
 617                 rxq->proto_xtr = IAVF_PROTO_XTR_NONE;
 618         }
 619
 620         if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN_V2) {
 621                 struct virtchnl_vlan_supported_caps *stripping_support =
 622                                 &vf->vlan_v2_caps.offloads.stripping_support;
 623                 uint32_t stripping_cap;
 624
 625                 if (stripping_support->outer)
 626                         stripping_cap = stripping_support->outer;
 627                 else
 628                         stripping_cap = stripping_support->inner;
 629
 630                 if (stripping_cap & VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)
 631                         rxq->rx_flags = IAVF_RX_FLAGS_VLAN_TAG_LOC_L2TAG1;
 632                 else if (stripping_cap & VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2_2)
 633                         rxq->rx_flags = IAVF_RX_FLAGS_VLAN_TAG_LOC_L2TAG2_2;
 634         } else {
 635                 rxq->rx_flags = IAVF_RX_FLAGS_VLAN_TAG_LOC_L2TAG1;
 636         }
 637
 638         iavf_select_rxd_to_pkt_fields_handler(rxq, rxq->rxdid);
 639
 640         rxq->mp = mp;
 641         rxq->nb_rx_desc = nb_desc;
 642         rxq->rx_free_thresh = rx_free_thresh;
 643         rxq->queue_id = queue_idx;
 644         rxq->port_id = dev->data->port_id;
 645         rxq->rx_deferred_start = rx_conf->rx_deferred_start;
 646         rxq->rx_hdr_len = 0;
 647         rxq->vsi = vsi;
 648         rxq->offloads = offloads;
 649
 650         if (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC)
 651                 rxq->crc_len = RTE_ETHER_CRC_LEN;
 652         else
 653                 rxq->crc_len = 0;
 654
 655         len = rte_pktmbuf_data_room_size(rxq->mp) - RTE_PKTMBUF_HEADROOM;
 656         rxq->rx_buf_len = RTE_ALIGN_FLOOR(len, (1 << IAVF_RXQ_CTX_DBUFF_SHIFT));
 657
 658         /* Allocate the software ring. */
 659         len = nb_desc + IAVF_RX_MAX_BURST;
 660         rxq->sw_ring =
 661                 rte_zmalloc_socket("iavf rx sw ring",
 662                                    sizeof(struct rte_mbuf *) * len,
 663                                    RTE_CACHE_LINE_SIZE,
 664                                    socket_id);
 665         if (!rxq->sw_ring) {
 666                 PMD_INIT_LOG(ERR, "Failed to allocate memory for SW ring");
 667                 rte_free(rxq);
 668                 return -ENOMEM;
 669         }
 670
 671         /* Allocate the maximum number of RX ring hardware descriptor with
 672          * a little more to support bulk allocate.
 673          */
 674         len = IAVF_MAX_RING_DESC + IAVF_RX_MAX_BURST;
 675         ring_size = RTE_ALIGN(len * sizeof(union iavf_rx_desc),
 676                               IAVF_DMA_MEM_ALIGN);
 677         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx,
 678                                       ring_size, IAVF_RING_BASE_ALIGN,
 679                                       socket_id);
 680         if (!mz) {
 681                 PMD_INIT_LOG(ERR, "Failed to reserve DMA memory for RX");
 682                 rte_free(rxq->sw_ring);
 683                 rte_free(rxq);
 684                 return -ENOMEM;
 685         }
 686         /* Zero all the descriptors in the ring. */
 687         memset(mz->addr, 0, ring_size);
 688         rxq->rx_ring_phys_addr = mz->iova;
 689         rxq->rx_ring = (union iavf_rx_desc *)mz->addr;
 690
 691         rxq->mz = mz;
 692         reset_rx_queue(rxq);
 693         rxq->q_set = true;
 694         dev->data->rx_queues[queue_idx] = rxq;
 695         rxq->qrx_tail = hw->hw_addr + IAVF_QRX_TAIL1(rxq->queue_id);
 696         rxq->rel_mbufs_type = IAVF_REL_MBUFS_DEFAULT;
 697
 698         if (check_rx_bulk_allow(rxq) == true) {
 699                 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are "
 700                              "satisfied. Rx Burst Bulk Alloc function will be "
 701                              "used on port=%d, queue=%d.",
 702                              rxq->port_id, rxq->queue_id);
 703         } else {
 704                 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are "
 705                              "not satisfied, Scattered Rx is requested "
 706                              "on port=%d, queue=%d.",
 707                              rxq->port_id, rxq->queue_id);
 708                 ad->rx_bulk_alloc_allowed = false;
 709         }
 710
 711         if (check_rx_vec_allow(rxq) == false)
 712                 ad->rx_vec_allowed = false;
 713
 714         return 0;
 715 }
 716
 717 int
 718 iavf_dev_tx_queue_setup(struct rte_eth_dev *dev,
 719                        uint16_t queue_idx,
 720                        uint16_t nb_desc,
 721                        unsigned int socket_id,
 722                        const struct rte_eth_txconf *tx_conf)
 723 {
 724         struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 725         struct iavf_adapter *adapter =
 726                 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 727         struct iavf_info *vf =
 728                 IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 729         struct iavf_tx_queue *txq;
 730         const struct rte_memzone *mz;
 731         uint32_t ring_size;
 732         uint16_t tx_rs_thresh, tx_free_thresh;
 733         uint64_t offloads;
 734
 735         PMD_INIT_FUNC_TRACE();
 736
 737         if (adapter->closed)
 738                 return -EIO;
 739
 740         offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads;
 741
 742         if (nb_desc % IAVF_ALIGN_RING_DESC != 0 ||
 743             nb_desc > IAVF_MAX_RING_DESC ||
 744             nb_desc < IAVF_MIN_RING_DESC) {
 745                 PMD_INIT_LOG(ERR, "Number (%u) of transmit descriptors is "
 746                             "invalid", nb_desc);
 747                 return -EINVAL;
 748         }
 749
 750         tx_rs_thresh = (uint16_t)((tx_conf->tx_rs_thresh) ?
 751                 tx_conf->tx_rs_thresh : DEFAULT_TX_RS_THRESH);
 752         tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ?
 753                 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH);
 754         if (check_tx_thresh(nb_desc, tx_rs_thresh, tx_free_thresh) != 0)
 755                 return -EINVAL;
 756
 757         /* Free memory if needed. */
 758         if (dev->data->tx_queues[queue_idx]) {
 759                 iavf_dev_tx_queue_release(dev, queue_idx);
 760                 dev->data->tx_queues[queue_idx] = NULL;
 761         }
 762
 763         /* Allocate the TX queue data structure. */
 764         txq = rte_zmalloc_socket("iavf txq",
 765                                  sizeof(struct iavf_tx_queue),
 766                                  RTE_CACHE_LINE_SIZE,
 767                                  socket_id);
 768         if (!txq) {
 769                 PMD_INIT_LOG(ERR, "Failed to allocate memory for "
 770                              "tx queue structure");
 771                 return -ENOMEM;
 772         }
 773
 774         if (adapter->vf.vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN_V2) {
 775                 struct virtchnl_vlan_supported_caps *insertion_support =
 776                         &adapter->vf.vlan_v2_caps.offloads.insertion_support;
 777                 uint32_t insertion_cap;
 778
 779                 if (insertion_support->outer)
 780                         insertion_cap = insertion_support->outer;
 781                 else
 782                         insertion_cap = insertion_support->inner;
 783
 784                 if (insertion_cap & VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)
 785                         txq->vlan_flag = IAVF_TX_FLAGS_VLAN_TAG_LOC_L2TAG1;
 786                 else if (insertion_cap & VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2)
 787                         txq->vlan_flag = IAVF_TX_FLAGS_VLAN_TAG_LOC_L2TAG2;
 788         } else {
 789                 txq->vlan_flag = IAVF_TX_FLAGS_VLAN_TAG_LOC_L2TAG1;
 790         }
 791
 792         txq->nb_tx_desc = nb_desc;
 793         txq->rs_thresh = tx_rs_thresh;
 794         txq->free_thresh = tx_free_thresh;
 795         txq->queue_id = queue_idx;
 796         txq->port_id = dev->data->port_id;
 797         txq->offloads = offloads;
 798         txq->tx_deferred_start = tx_conf->tx_deferred_start;
 799
 800         if (iavf_ipsec_crypto_supported(adapter))
 801                 txq->ipsec_crypto_pkt_md_offset =
 802                         iavf_security_get_pkt_md_offset(adapter);
 803
 804         /* Allocate software ring */
 805         txq->sw_ring =
 806                 rte_zmalloc_socket("iavf tx sw ring",
 807                                    sizeof(struct iavf_tx_entry) * nb_desc,
 808                                    RTE_CACHE_LINE_SIZE,
 809                                    socket_id);
 810         if (!txq->sw_ring) {
 811                 PMD_INIT_LOG(ERR, "Failed to allocate memory for SW TX ring");
 812                 rte_free(txq);
 813                 return -ENOMEM;
 814         }
 815
 816         /* Allocate TX hardware ring descriptors. */
 817         ring_size = sizeof(struct iavf_tx_desc) * IAVF_MAX_RING_DESC;
 818         ring_size = RTE_ALIGN(ring_size, IAVF_DMA_MEM_ALIGN);
 819         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx,
 820                                       ring_size, IAVF_RING_BASE_ALIGN,
 821                                       socket_id);
 822         if (!mz) {
 823                 PMD_INIT_LOG(ERR, "Failed to reserve DMA memory for TX");
 824                 rte_free(txq->sw_ring);
 825                 rte_free(txq);
 826                 return -ENOMEM;
 827         }
 828         txq->tx_ring_phys_addr = mz->iova;
 829         txq->tx_ring = (struct iavf_tx_desc *)mz->addr;
 830
 831         txq->mz = mz;
 832         reset_tx_queue(txq);
 833         txq->q_set = true;
 834         dev->data->tx_queues[queue_idx] = txq;
 835         txq->qtx_tail = hw->hw_addr + IAVF_QTX_TAIL1(queue_idx);
 836         txq->rel_mbufs_type = IAVF_REL_MBUFS_DEFAULT;
 837
 838         if (check_tx_vec_allow(txq) == false) {
 839                 struct iavf_adapter *ad =
 840                         IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 841                 ad->tx_vec_allowed = false;
 842         }
 843
 844         if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_QOS &&
 845             vf->tm_conf.committed) {
 846                 int tc;
 847                 for (tc = 0; tc < vf->qos_cap->num_elem; tc++) {
 848                         if (txq->queue_id >= vf->qtc_map[tc].start_queue_id &&
 849                             txq->queue_id < (vf->qtc_map[tc].start_queue_id +
 850                             vf->qtc_map[tc].queue_count))
 851                                 break;
 852                 }
 853                 if (tc >= vf->qos_cap->num_elem) {
 854                         PMD_INIT_LOG(ERR, "Queue TC mapping is not correct");
 855                         return -EINVAL;
 856                 }
 857                 txq->tc = tc;
 858         }
 859
 860         return 0;
 861 }
 862
 863 int
 864 iavf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 865 {
 866         struct iavf_adapter *adapter =
 867                 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 868         struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 869         struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 870         struct iavf_rx_queue *rxq;
 871         int err = 0;
 872
 873         PMD_DRV_FUNC_TRACE();
 874
 875         if (rx_queue_id >= dev->data->nb_rx_queues)
 876                 return -EINVAL;
 877
 878         rxq = dev->data->rx_queues[rx_queue_id];
 879
 880         err = alloc_rxq_mbufs(rxq);
 881         if (err) {
 882                 PMD_DRV_LOG(ERR, "Failed to allocate RX queue mbuf");
 883                 return err;
 884         }
 885
 886         rte_wmb();
 887
 888         /* Init the RX tail register. */
 889         IAVF_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
 890         IAVF_WRITE_FLUSH(hw);
 891
 892         /* Ready to switch the queue on */
 893         if (!vf->lv_enabled)
 894                 err = iavf_switch_queue(adapter, rx_queue_id, true, true);
 895         else
 896                 err = iavf_switch_queue_lv(adapter, rx_queue_id, true, true);
 897
 898         if (err) {
 899                 release_rxq_mbufs(rxq);
 900                 PMD_DRV_LOG(ERR, "Failed to switch RX queue %u on",
 901                             rx_queue_id);
 902         } else {
 903                 dev->data->rx_queue_state[rx_queue_id] =
 904                         RTE_ETH_QUEUE_STATE_STARTED;
 905         }
 906
 907         return err;
 908 }
 909
 910 int
 911 iavf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 912 {
 913         struct iavf_adapter *adapter =
 914                 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 915         struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 916         struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 917         struct iavf_tx_queue *txq;
 918         int err = 0;
 919
 920         PMD_DRV_FUNC_TRACE();
 921
 922         if (tx_queue_id >= dev->data->nb_tx_queues)
 923                 return -EINVAL;
 924
 925         txq = dev->data->tx_queues[tx_queue_id];
 926
 927         /* Init the RX tail register. */
 928         IAVF_PCI_REG_WRITE(txq->qtx_tail, 0);
 929         IAVF_WRITE_FLUSH(hw);
 930
 931         /* Ready to switch the queue on */
 932         if (!vf->lv_enabled)
 933                 err = iavf_switch_queue(adapter, tx_queue_id, false, true);
 934         else
 935                 err = iavf_switch_queue_lv(adapter, tx_queue_id, false, true);
 936
 937         if (err)
 938                 PMD_DRV_LOG(ERR, "Failed to switch TX queue %u on",
 939                             tx_queue_id);
 940         else
 941                 dev->data->tx_queue_state[tx_queue_id] =
 942                         RTE_ETH_QUEUE_STATE_STARTED;
 943
 944         return err;
 945 }
 946
 947 int
 948 iavf_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 949 {
 950         struct iavf_adapter *adapter =
 951                 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 952         struct iavf_rx_queue *rxq;
 953         int err;
 954
 955         PMD_DRV_FUNC_TRACE();
 956
 957         if (rx_queue_id >= dev->data->nb_rx_queues)
 958                 return -EINVAL;
 959
 960         err = iavf_switch_queue(adapter, rx_queue_id, true, false);
 961         if (err) {
 962                 PMD_DRV_LOG(ERR, "Failed to switch RX queue %u off",
 963                             rx_queue_id);
 964                 return err;
 965         }
 966
 967         rxq = dev->data->rx_queues[rx_queue_id];
 968         iavf_rxq_release_mbufs_ops[rxq->rel_mbufs_type].release_mbufs(rxq);
 969         reset_rx_queue(rxq);
 970         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 971
 972         return 0;
 973 }
 974
 975 int
 976 iavf_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 977 {
 978         struct iavf_adapter *adapter =
 979                 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 980         struct iavf_tx_queue *txq;
 981         int err;
 982
 983         PMD_DRV_FUNC_TRACE();
 984
 985         if (tx_queue_id >= dev->data->nb_tx_queues)
 986                 return -EINVAL;
 987
 988         err = iavf_switch_queue(adapter, tx_queue_id, false, false);
 989         if (err) {
 990                 PMD_DRV_LOG(ERR, "Failed to switch TX queue %u off",
 991                             tx_queue_id);
 992                 return err;
 993         }
 994
 995         txq = dev->data->tx_queues[tx_queue_id];
 996         iavf_txq_release_mbufs_ops[txq->rel_mbufs_type].release_mbufs(txq);
 997         reset_tx_queue(txq);
 998         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 999
1000         return 0;
1001 }
1002
1003 void
1004 iavf_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
1005 {
1006         struct iavf_rx_queue *q = dev->data->rx_queues[qid];
1007
1008         if (!q)
1009                 return;
1010
1011         iavf_rxq_release_mbufs_ops[q->rel_mbufs_type].release_mbufs(q);
1012         rte_free(q->sw_ring);
1013         rte_memzone_free(q->mz);
1014         rte_free(q);
1015 }
1016
1017 void
1018 iavf_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
1019 {
1020         struct iavf_tx_queue *q = dev->data->tx_queues[qid];
1021
1022         if (!q)
1023                 return;
1024
1025         iavf_txq_release_mbufs_ops[q->rel_mbufs_type].release_mbufs(q);
1026         rte_free(q->sw_ring);
1027         rte_memzone_free(q->mz);
1028         rte_free(q);
1029 }
1030
1031 void
1032 iavf_stop_queues(struct rte_eth_dev *dev)
1033 {
1034         struct iavf_adapter *adapter =
1035                 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
1036         struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1037         struct iavf_rx_queue *rxq;
1038         struct iavf_tx_queue *txq;
1039         int ret, i;
1040
1041         /* Stop All queues */
1042         if (!vf->lv_enabled) {
1043                 ret = iavf_disable_queues(adapter);
1044                 if (ret)
1045                         PMD_DRV_LOG(WARNING, "Fail to stop queues");
1046         } else {
1047                 ret = iavf_disable_queues_lv(adapter);
1048                 if (ret)
1049                         PMD_DRV_LOG(WARNING, "Fail to stop queues for large VF");
1050         }
1051
1052         if (ret)
1053                 PMD_DRV_LOG(WARNING, "Fail to stop queues");
1054
1055         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1056                 txq = dev->data->tx_queues[i];
1057                 if (!txq)
1058                         continue;
1059                 iavf_txq_release_mbufs_ops[txq->rel_mbufs_type].release_mbufs(txq);
1060                 reset_tx_queue(txq);
1061                 dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED;
1062         }
1063         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1064                 rxq = dev->data->rx_queues[i];
1065                 if (!rxq)
1066                         continue;
1067                 iavf_rxq_release_mbufs_ops[rxq->rel_mbufs_type].release_mbufs(rxq);
1068                 reset_rx_queue(rxq);
1069                 dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED;
1070         }
1071 }
1072
1073 #define IAVF_RX_FLEX_ERR0_BITS  \
1074         ((1 << IAVF_RX_FLEX_DESC_STATUS0_HBO_S) |       \
1075          (1 << IAVF_RX_FLEX_DESC_STATUS0_XSUM_IPE_S) |  \
1076          (1 << IAVF_RX_FLEX_DESC_STATUS0_XSUM_L4E_S) |  \
1077          (1 << IAVF_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S) | \
1078          (1 << IAVF_RX_FLEX_DESC_STATUS0_XSUM_EUDPE_S) |        \
1079          (1 << IAVF_RX_FLEX_DESC_STATUS0_RXE_S))
1080
1081 static inline void
1082 iavf_rxd_to_vlan_tci(struct rte_mbuf *mb, volatile union iavf_rx_desc *rxdp)
1083 {
1084         if (rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len) &
1085                 (1 << IAVF_RX_DESC_STATUS_L2TAG1P_SHIFT)) {
1086                 mb->ol_flags |= RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED;
1087                 mb->vlan_tci =
1088                         rte_le_to_cpu_16(rxdp->wb.qword0.lo_dword.l2tag1);
1089         } else {
1090                 mb->vlan_tci = 0;
1091         }
1092 }
1093
1094 static inline void
1095 iavf_flex_rxd_to_vlan_tci(struct rte_mbuf *mb,
1096                           volatile union iavf_rx_flex_desc *rxdp)
1097 {
1098         if (rte_le_to_cpu_64(rxdp->wb.status_error0) &
1099                 (1 << IAVF_RX_FLEX_DESC_STATUS0_L2TAG1P_S)) {
1100                 mb->ol_flags |= RTE_MBUF_F_RX_VLAN |
1101                                 RTE_MBUF_F_RX_VLAN_STRIPPED;
1102                 mb->vlan_tci =
1103                         rte_le_to_cpu_16(rxdp->wb.l2tag1);
1104         } else {
1105                 mb->vlan_tci = 0;
1106         }
1107
1108 #ifndef RTE_LIBRTE_IAVF_16BYTE_RX_DESC
1109         if (rte_le_to_cpu_16(rxdp->wb.status_error1) &
1110             (1 << IAVF_RX_FLEX_DESC_STATUS1_L2TAG2P_S)) {
1111                 mb->ol_flags |= RTE_MBUF_F_RX_QINQ_STRIPPED |
1112                                 RTE_MBUF_F_RX_QINQ |
1113                                 RTE_MBUF_F_RX_VLAN_STRIPPED |
1114                                 RTE_MBUF_F_RX_VLAN;
1115                 mb->vlan_tci_outer = mb->vlan_tci;
1116                 mb->vlan_tci = rte_le_to_cpu_16(rxdp->wb.l2tag2_2nd);
1117                 PMD_RX_LOG(DEBUG, "Descriptor l2tag2_1: %u, l2tag2_2: %u",
1118                            rte_le_to_cpu_16(rxdp->wb.l2tag2_1st),
1119                            rte_le_to_cpu_16(rxdp->wb.l2tag2_2nd));
1120         } else {
1121                 mb->vlan_tci_outer = 0;
1122         }
1123 #endif
1124 }
1125
1126 static inline void
1127 iavf_flex_rxd_to_ipsec_crypto_said_get(struct rte_mbuf *mb,
1128                           volatile union iavf_rx_flex_desc *rxdp)
1129 {
1130         volatile struct iavf_32b_rx_flex_desc_comms_ipsec *desc =
1131                 (volatile struct iavf_32b_rx_flex_desc_comms_ipsec *)rxdp;
1132
1133         mb->dynfield1[0] = desc->ipsec_said &
1134                          IAVF_RX_FLEX_DESC_IPSEC_CRYPTO_SAID_MASK;
1135         }
1136
1137 static inline void
1138 iavf_flex_rxd_to_ipsec_crypto_status(struct rte_mbuf *mb,
1139                           volatile union iavf_rx_flex_desc *rxdp,
1140                           struct iavf_ipsec_crypto_stats *stats)
1141 {
1142         uint16_t status1 = rte_le_to_cpu_64(rxdp->wb.status_error1);
1143
1144         if (status1 & BIT(IAVF_RX_FLEX_DESC_STATUS1_IPSEC_CRYPTO_PROCESSED)) {
1145                 uint16_t ipsec_status;
1146
1147                 mb->ol_flags |= RTE_MBUF_F_RX_SEC_OFFLOAD;
1148
1149                 ipsec_status = status1 &
1150                         IAVF_RX_FLEX_DESC_IPSEC_CRYPTO_STATUS_MASK;
1151
1152
1153                 if (unlikely(ipsec_status !=
1154                         IAVF_IPSEC_CRYPTO_STATUS_SUCCESS)) {
1155                         mb->ol_flags |= RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED;
1156
1157                         switch (ipsec_status) {
1158                         case IAVF_IPSEC_CRYPTO_STATUS_SAD_MISS:
1159                                 stats->ierrors.sad_miss++;
1160                                 break;
1161                         case IAVF_IPSEC_CRYPTO_STATUS_NOT_PROCESSED:
1162                                 stats->ierrors.not_processed++;
1163                                 break;
1164                         case IAVF_IPSEC_CRYPTO_STATUS_ICV_CHECK_FAIL:
1165                                 stats->ierrors.icv_check++;
1166                                 break;
1167                         case IAVF_IPSEC_CRYPTO_STATUS_LENGTH_ERR:
1168                                 stats->ierrors.ipsec_length++;
1169                                 break;
1170                         case IAVF_IPSEC_CRYPTO_STATUS_MISC_ERR:
1171                                 stats->ierrors.misc++;
1172                                 break;
1173 }
1174
1175                         stats->ierrors.count++;
1176                         return;
1177                 }
1178
1179                 stats->icount++;
1180                 stats->ibytes += rxdp->wb.pkt_len & 0x3FFF;
1181
1182                 if (rxdp->wb.rxdid == IAVF_RXDID_COMMS_IPSEC_CRYPTO &&
1183                         ipsec_status !=
1184                                 IAVF_IPSEC_CRYPTO_STATUS_SAD_MISS)
1185                         iavf_flex_rxd_to_ipsec_crypto_said_get(mb, rxdp);
1186         }
1187 }
1188
1189
1190 /* Translate the rx descriptor status and error fields to pkt flags */
1191 static inline uint64_t
1192 iavf_rxd_to_pkt_flags(uint64_t qword)
1193 {
1194         uint64_t flags;
1195         uint64_t error_bits = (qword >> IAVF_RXD_QW1_ERROR_SHIFT);
1196
1197 #define IAVF_RX_ERR_BITS 0x3f
1198
1199         /* Check if RSS_HASH */
1200         flags = (((qword >> IAVF_RX_DESC_STATUS_FLTSTAT_SHIFT) &
1201                                         IAVF_RX_DESC_FLTSTAT_RSS_HASH) ==
1202                         IAVF_RX_DESC_FLTSTAT_RSS_HASH) ? RTE_MBUF_F_RX_RSS_HASH : 0;
1203
1204         /* Check if FDIR Match */
1205         flags |= (qword & (1 << IAVF_RX_DESC_STATUS_FLM_SHIFT) ?
1206                                 RTE_MBUF_F_RX_FDIR : 0);
1207
1208         if (likely((error_bits & IAVF_RX_ERR_BITS) == 0)) {
1209                 flags |= (RTE_MBUF_F_RX_IP_CKSUM_GOOD | RTE_MBUF_F_RX_L4_CKSUM_GOOD);
1210                 return flags;
1211         }
1212
1213         if (unlikely(error_bits & (1 << IAVF_RX_DESC_ERROR_IPE_SHIFT)))
1214                 flags |= RTE_MBUF_F_RX_IP_CKSUM_BAD;
1215         else
1216                 flags |= RTE_MBUF_F_RX_IP_CKSUM_GOOD;
1217
1218         if (unlikely(error_bits & (1 << IAVF_RX_DESC_ERROR_L4E_SHIFT)))
1219                 flags |= RTE_MBUF_F_RX_L4_CKSUM_BAD;
1220         else
1221                 flags |= RTE_MBUF_F_RX_L4_CKSUM_GOOD;
1222
1223         /* TODO: Oversize error bit is not processed here */
1224
1225         return flags;
1226 }
1227
1228 static inline uint64_t
1229 iavf_rxd_build_fdir(volatile union iavf_rx_desc *rxdp, struct rte_mbuf *mb)
1230 {
1231         uint64_t flags = 0;
1232 #ifndef RTE_LIBRTE_IAVF_16BYTE_RX_DESC
1233         uint16_t flexbh;
1234
1235         flexbh = (rte_le_to_cpu_32(rxdp->wb.qword2.ext_status) >>
1236                 IAVF_RX_DESC_EXT_STATUS_FLEXBH_SHIFT) &
1237                 IAVF_RX_DESC_EXT_STATUS_FLEXBH_MASK;
1238
1239         if (flexbh == IAVF_RX_DESC_EXT_STATUS_FLEXBH_FD_ID) {
1240                 mb->hash.fdir.hi =
1241                         rte_le_to_cpu_32(rxdp->wb.qword3.hi_dword.fd_id);
1242                 flags |= RTE_MBUF_F_RX_FDIR_ID;
1243         }
1244 #else
1245         mb->hash.fdir.hi =
1246                 rte_le_to_cpu_32(rxdp->wb.qword0.hi_dword.fd_id);
1247         flags |= RTE_MBUF_F_RX_FDIR_ID;
1248 #endif
1249         return flags;
1250 }
1251
1252 #define IAVF_RX_FLEX_ERR0_BITS  \
1253         ((1 << IAVF_RX_FLEX_DESC_STATUS0_HBO_S) |       \
1254          (1 << IAVF_RX_FLEX_DESC_STATUS0_XSUM_IPE_S) |  \
1255          (1 << IAVF_RX_FLEX_DESC_STATUS0_XSUM_L4E_S) |  \
1256          (1 << IAVF_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S) | \
1257          (1 << IAVF_RX_FLEX_DESC_STATUS0_XSUM_EUDPE_S) |        \
1258          (1 << IAVF_RX_FLEX_DESC_STATUS0_RXE_S))
1259
1260 /* Rx L3/L4 checksum */
1261 static inline uint64_t
1262 iavf_flex_rxd_error_to_pkt_flags(uint16_t stat_err0)
1263 {
1264         uint64_t flags = 0;
1265
1266         /* check if HW has decoded the packet and checksum */
1267         if (unlikely(!(stat_err0 & (1 << IAVF_RX_FLEX_DESC_STATUS0_L3L4P_S))))
1268                 return 0;
1269
1270         if (likely(!(stat_err0 & IAVF_RX_FLEX_ERR0_BITS))) {
1271                 flags |= (RTE_MBUF_F_RX_IP_CKSUM_GOOD | RTE_MBUF_F_RX_L4_CKSUM_GOOD);
1272                 return flags;
1273         }
1274
1275         if (unlikely(stat_err0 & (1 << IAVF_RX_FLEX_DESC_STATUS0_XSUM_IPE_S)))
1276                 flags |= RTE_MBUF_F_RX_IP_CKSUM_BAD;
1277         else
1278                 flags |= RTE_MBUF_F_RX_IP_CKSUM_GOOD;
1279
1280         if (unlikely(stat_err0 & (1 << IAVF_RX_FLEX_DESC_STATUS0_XSUM_L4E_S)))
1281                 flags |= RTE_MBUF_F_RX_L4_CKSUM_BAD;
1282         else
1283                 flags |= RTE_MBUF_F_RX_L4_CKSUM_GOOD;
1284
1285         if (unlikely(stat_err0 & (1 << IAVF_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S)))
1286                 flags |= RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD;
1287
1288         return flags;
1289 }
1290
1291 /* If the number of free RX descriptors is greater than the RX free
1292  * threshold of the queue, advance the Receive Descriptor Tail (RDT)
1293  * register. Update the RDT with the value of the last processed RX
1294  * descriptor minus 1, to guarantee that the RDT register is never
1295  * equal to the RDH register, which creates a "full" ring situation
1296  * from the hardware point of view.
1297  */
1298 static inline void
1299 iavf_update_rx_tail(struct iavf_rx_queue *rxq, uint16_t nb_hold, uint16_t rx_id)
1300 {
1301         nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold);
1302
1303         if (nb_hold > rxq->rx_free_thresh) {
1304                 PMD_RX_LOG(DEBUG,
1305                            "port_id=%u queue_id=%u rx_tail=%u nb_hold=%u",
1306                            rxq->port_id, rxq->queue_id, rx_id, nb_hold);
1307                 rx_id = (uint16_t)((rx_id == 0) ?
1308                         (rxq->nb_rx_desc - 1) : (rx_id - 1));
1309                 IAVF_PCI_REG_WC_WRITE(rxq->qrx_tail, rx_id);
1310                 nb_hold = 0;
1311         }
1312         rxq->nb_rx_hold = nb_hold;
1313 }
1314
1315 /* implement recv_pkts */
1316 uint16_t
1317 iavf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
1318 {
1319         volatile union iavf_rx_desc *rx_ring;
1320         volatile union iavf_rx_desc *rxdp;
1321         struct iavf_rx_queue *rxq;
1322         union iavf_rx_desc rxd;
1323         struct rte_mbuf *rxe;
1324         struct rte_eth_dev *dev;
1325         struct rte_mbuf *rxm;
1326         struct rte_mbuf *nmb;
1327         uint16_t nb_rx;
1328         uint32_t rx_status;
1329         uint64_t qword1;
1330         uint16_t rx_packet_len;
1331         uint16_t rx_id, nb_hold;
1332         uint64_t dma_addr;
1333         uint64_t pkt_flags;
1334         const uint32_t *ptype_tbl;
1335
1336         nb_rx = 0;
1337         nb_hold = 0;
1338         rxq = rx_queue;
1339         rx_id = rxq->rx_tail;
1340         rx_ring = rxq->rx_ring;
1341         ptype_tbl = rxq->vsi->adapter->ptype_tbl;
1342
1343         while (nb_rx < nb_pkts) {
1344                 rxdp = &rx_ring[rx_id];
1345                 qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len);
1346                 rx_status = (qword1 & IAVF_RXD_QW1_STATUS_MASK) >>
1347                             IAVF_RXD_QW1_STATUS_SHIFT;
1348
1349                 /* Check the DD bit first */
1350                 if (!(rx_status & (1 << IAVF_RX_DESC_STATUS_DD_SHIFT)))
1351                         break;
1352                 IAVF_DUMP_RX_DESC(rxq, rxdp, rx_id);
1353
1354                 nmb = rte_mbuf_raw_alloc(rxq->mp);
1355                 if (unlikely(!nmb)) {
1356                         dev = &rte_eth_devices[rxq->port_id];
1357                         dev->data->rx_mbuf_alloc_failed++;
1358                         PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u "
1359                                    "queue_id=%u", rxq->port_id, rxq->queue_id);
1360                         break;
1361                 }
1362
1363                 rxd = *rxdp;
1364                 nb_hold++;
1365                 rxe = rxq->sw_ring[rx_id];
1366                 rxq->sw_ring[rx_id] = nmb;
1367                 rx_id++;
1368                 if (unlikely(rx_id == rxq->nb_rx_desc))
1369                         rx_id = 0;
1370
1371                 /* Prefetch next mbuf */
1372                 rte_prefetch0(rxq->sw_ring[rx_id]);
1373
1374                 /* When next RX descriptor is on a cache line boundary,
1375                  * prefetch the next 4 RX descriptors and next 8 pointers
1376                  * to mbufs.
1377                  */
1378                 if ((rx_id & 0x3) == 0) {
1379                         rte_prefetch0(&rx_ring[rx_id]);
1380                         rte_prefetch0(rxq->sw_ring[rx_id]);
1381                 }
1382                 rxm = rxe;
1383                 dma_addr =
1384                         rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
1385                 rxdp->read.hdr_addr = 0;
1386                 rxdp->read.pkt_addr = dma_addr;
1387
1388                 rx_packet_len = ((qword1 & IAVF_RXD_QW1_LENGTH_PBUF_MASK) >>
1389                                 IAVF_RXD_QW1_LENGTH_PBUF_SHIFT) - rxq->crc_len;
1390
1391                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1392                 rte_prefetch0(RTE_PTR_ADD(rxm->buf_addr, RTE_PKTMBUF_HEADROOM));
1393                 rxm->nb_segs = 1;
1394                 rxm->next = NULL;
1395                 rxm->pkt_len = rx_packet_len;
1396                 rxm->data_len = rx_packet_len;
1397                 rxm->port = rxq->port_id;
1398                 rxm->ol_flags = 0;
1399                 iavf_rxd_to_vlan_tci(rxm, &rxd);
1400                 pkt_flags = iavf_rxd_to_pkt_flags(qword1);
1401                 rxm->packet_type =
1402                         ptype_tbl[(uint8_t)((qword1 &
1403                         IAVF_RXD_QW1_PTYPE_MASK) >> IAVF_RXD_QW1_PTYPE_SHIFT)];
1404
1405                 if (pkt_flags & RTE_MBUF_F_RX_RSS_HASH)
1406                         rxm->hash.rss =
1407                                 rte_le_to_cpu_32(rxd.wb.qword0.hi_dword.rss);
1408
1409                 if (pkt_flags & RTE_MBUF_F_RX_FDIR)
1410                         pkt_flags |= iavf_rxd_build_fdir(&rxd, rxm);
1411
1412                 rxm->ol_flags |= pkt_flags;
1413
1414                 rx_pkts[nb_rx++] = rxm;
1415         }
1416         rxq->rx_tail = rx_id;
1417
1418         iavf_update_rx_tail(rxq, nb_hold, rx_id);
1419
1420         return nb_rx;
1421 }
1422
1423 /* implement recv_pkts for flexible Rx descriptor */
1424 uint16_t
1425 iavf_recv_pkts_flex_rxd(void *rx_queue,
1426                         struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
1427 {
1428         volatile union iavf_rx_desc *rx_ring;
1429         volatile union iavf_rx_flex_desc *rxdp;
1430         struct iavf_rx_queue *rxq;
1431         union iavf_rx_flex_desc rxd;
1432         struct rte_mbuf *rxe;
1433         struct rte_eth_dev *dev;
1434         struct rte_mbuf *rxm;
1435         struct rte_mbuf *nmb;
1436         uint16_t nb_rx;
1437         uint16_t rx_stat_err0;
1438         uint16_t rx_packet_len;
1439         uint16_t rx_id, nb_hold;
1440         uint64_t dma_addr;
1441         uint64_t pkt_flags;
1442         const uint32_t *ptype_tbl;
1443
1444         nb_rx = 0;
1445         nb_hold = 0;
1446         rxq = rx_queue;
1447         rx_id = rxq->rx_tail;
1448         rx_ring = rxq->rx_ring;
1449         ptype_tbl = rxq->vsi->adapter->ptype_tbl;
1450
1451         struct iavf_adapter *ad = rxq->vsi->adapter;
1452         uint64_t ts_ns;
1453
1454         if (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) {
1455                 uint64_t sw_cur_time = rte_get_timer_cycles() / (rte_get_timer_hz() / 1000);
1456                 if (sw_cur_time - ad->hw_time_update > 4) {
1457                         if (iavf_get_phc_time(ad))
1458                                 PMD_DRV_LOG(ERR, "get physical time failed");
1459                         ad->hw_time_update = sw_cur_time;
1460                 }
1461         }
1462
1463         while (nb_rx < nb_pkts) {
1464                 rxdp = (volatile union iavf_rx_flex_desc *)&rx_ring[rx_id];
1465                 rx_stat_err0 = rte_le_to_cpu_16(rxdp->wb.status_error0);
1466
1467                 /* Check the DD bit first */
1468                 if (!(rx_stat_err0 & (1 << IAVF_RX_FLEX_DESC_STATUS0_DD_S)))
1469                         break;
1470                 IAVF_DUMP_RX_DESC(rxq, rxdp, rx_id);
1471
1472                 nmb = rte_mbuf_raw_alloc(rxq->mp);
1473                 if (unlikely(!nmb)) {
1474                         dev = &rte_eth_devices[rxq->port_id];
1475                         dev->data->rx_mbuf_alloc_failed++;
1476                         PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u "
1477                                    "queue_id=%u", rxq->port_id, rxq->queue_id);
1478                         break;
1479                 }
1480
1481                 rxd = *rxdp;
1482                 nb_hold++;
1483                 rxe = rxq->sw_ring[rx_id];
1484                 rxq->sw_ring[rx_id] = nmb;
1485                 rx_id++;
1486                 if (unlikely(rx_id == rxq->nb_rx_desc))
1487                         rx_id = 0;
1488
1489                 /* Prefetch next mbuf */
1490                 rte_prefetch0(rxq->sw_ring[rx_id]);
1491
1492                 /* When next RX descriptor is on a cache line boundary,
1493                  * prefetch the next 4 RX descriptors and next 8 pointers
1494                  * to mbufs.
1495                  */
1496                 if ((rx_id & 0x3) == 0) {
1497                         rte_prefetch0(&rx_ring[rx_id]);
1498                         rte_prefetch0(rxq->sw_ring[rx_id]);
1499                 }
1500                 rxm = rxe;
1501                 dma_addr =
1502                         rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
1503                 rxdp->read.hdr_addr = 0;
1504                 rxdp->read.pkt_addr = dma_addr;
1505
1506                 rx_packet_len = (rte_le_to_cpu_16(rxd.wb.pkt_len) &
1507                                 IAVF_RX_FLX_DESC_PKT_LEN_M) - rxq->crc_len;
1508
1509                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1510                 rte_prefetch0(RTE_PTR_ADD(rxm->buf_addr, RTE_PKTMBUF_HEADROOM));
1511                 rxm->nb_segs = 1;
1512                 rxm->next = NULL;
1513                 rxm->pkt_len = rx_packet_len;
1514                 rxm->data_len = rx_packet_len;
1515                 rxm->port = rxq->port_id;
1516                 rxm->ol_flags = 0;
1517                 rxm->packet_type = ptype_tbl[IAVF_RX_FLEX_DESC_PTYPE_M &
1518                         rte_le_to_cpu_16(rxd.wb.ptype_flex_flags0)];
1519                 iavf_flex_rxd_to_vlan_tci(rxm, &rxd);
1520                 iavf_flex_rxd_to_ipsec_crypto_status(rxm, &rxd,
1521                                 &rxq->stats.ipsec_crypto);
1522                 rxd_to_pkt_fields_ops[rxq->rxdid](rxq, rxm, &rxd);
1523                 pkt_flags = iavf_flex_rxd_error_to_pkt_flags(rx_stat_err0);
1524
1525                 if (iavf_timestamp_dynflag > 0) {
1526                         ts_ns = iavf_tstamp_convert_32b_64b(ad->phc_time,
1527                                 rte_le_to_cpu_32(rxd.wb.flex_ts.ts_high));
1528
1529                         ad->phc_time = ts_ns;
1530                         ad->hw_time_update = rte_get_timer_cycles() / (rte_get_timer_hz() / 1000);
1531
1532                         *RTE_MBUF_DYNFIELD(rxm,
1533                                 iavf_timestamp_dynfield_offset,
1534                                 rte_mbuf_timestamp_t *) = ts_ns;
1535                         rxm->ol_flags |= iavf_timestamp_dynflag;
1536                 }
1537
1538                 rxm->ol_flags |= pkt_flags;
1539
1540                 rx_pkts[nb_rx++] = rxm;
1541         }
1542         rxq->rx_tail = rx_id;
1543
1544         iavf_update_rx_tail(rxq, nb_hold, rx_id);
1545
1546         return nb_rx;
1547 }
1548
1549 /* implement recv_scattered_pkts for flexible Rx descriptor */
1550 uint16_t
1551 iavf_recv_scattered_pkts_flex_rxd(void *rx_queue, struct rte_mbuf **rx_pkts,
1552                                   uint16_t nb_pkts)
1553 {
1554         struct iavf_rx_queue *rxq = rx_queue;
1555         union iavf_rx_flex_desc rxd;
1556         struct rte_mbuf *rxe;
1557         struct rte_mbuf *first_seg = rxq->pkt_first_seg;
1558         struct rte_mbuf *last_seg = rxq->pkt_last_seg;
1559         struct rte_mbuf *nmb, *rxm;
1560         uint16_t rx_id = rxq->rx_tail;
1561         uint16_t nb_rx = 0, nb_hold = 0, rx_packet_len;
1562         struct rte_eth_dev *dev;
1563         uint16_t rx_stat_err0;
1564         uint64_t dma_addr;
1565         uint64_t pkt_flags;
1566         struct iavf_adapter *ad = rxq->vsi->adapter;
1567         uint64_t ts_ns;
1568
1569         volatile union iavf_rx_desc *rx_ring = rxq->rx_ring;
1570         volatile union iavf_rx_flex_desc *rxdp;
1571         const uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl;
1572
1573         if (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) {
1574                 uint64_t sw_cur_time = rte_get_timer_cycles() / (rte_get_timer_hz() / 1000);
1575                 if (sw_cur_time - ad->hw_time_update > 4) {
1576                         if (iavf_get_phc_time(ad))
1577                                 PMD_DRV_LOG(ERR, "get physical time failed");
1578                         ad->hw_time_update = sw_cur_time;
1579                 }
1580         }
1581
1582         while (nb_rx < nb_pkts) {
1583                 rxdp = (volatile union iavf_rx_flex_desc *)&rx_ring[rx_id];
1584                 rx_stat_err0 = rte_le_to_cpu_16(rxdp->wb.status_error0);
1585
1586                 /* Check the DD bit */
1587                 if (!(rx_stat_err0 & (1 << IAVF_RX_FLEX_DESC_STATUS0_DD_S)))
1588                         break;
1589                 IAVF_DUMP_RX_DESC(rxq, rxdp, rx_id);
1590
1591                 nmb = rte_mbuf_raw_alloc(rxq->mp);
1592                 if (unlikely(!nmb)) {
1593                         PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u "
1594                                    "queue_id=%u", rxq->port_id, rxq->queue_id);
1595                         dev = &rte_eth_devices[rxq->port_id];
1596                         dev->data->rx_mbuf_alloc_failed++;
1597                         break;
1598                 }
1599
1600                 rxd = *rxdp;
1601                 nb_hold++;
1602                 rxe = rxq->sw_ring[rx_id];
1603                 rxq->sw_ring[rx_id] = nmb;
1604                 rx_id++;
1605                 if (rx_id == rxq->nb_rx_desc)
1606                         rx_id = 0;
1607
1608                 /* Prefetch next mbuf */
1609                 rte_prefetch0(rxq->sw_ring[rx_id]);
1610
1611                 /* When next RX descriptor is on a cache line boundary,
1612                  * prefetch the next 4 RX descriptors and next 8 pointers
1613                  * to mbufs.
1614                  */
1615                 if ((rx_id & 0x3) == 0) {
1616                         rte_prefetch0(&rx_ring[rx_id]);
1617                         rte_prefetch0(rxq->sw_ring[rx_id]);
1618                 }
1619
1620                 rxm = rxe;
1621                 dma_addr =
1622                         rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
1623
1624                 /* Set data buffer address and data length of the mbuf */
1625                 rxdp->read.hdr_addr = 0;
1626                 rxdp->read.pkt_addr = dma_addr;
1627                 rx_packet_len = rte_le_to_cpu_16(rxd.wb.pkt_len) &
1628                                 IAVF_RX_FLX_DESC_PKT_LEN_M;
1629                 rxm->data_len = rx_packet_len;
1630                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1631
1632                 /* If this is the first buffer of the received packet, set the
1633                  * pointer to the first mbuf of the packet and initialize its
1634                  * context. Otherwise, update the total length and the number
1635                  * of segments of the current scattered packet, and update the
1636                  * pointer to the last mbuf of the current packet.
1637                  */
1638                 if (!first_seg) {
1639                         first_seg = rxm;
1640                         first_seg->nb_segs = 1;
1641                         first_seg->pkt_len = rx_packet_len;
1642                 } else {
1643                         first_seg->pkt_len =
1644                                 (uint16_t)(first_seg->pkt_len +
1645                                                 rx_packet_len);
1646                         first_seg->nb_segs++;
1647                         last_seg->next = rxm;
1648                 }
1649
1650                 /* If this is not the last buffer of the received packet,
1651                  * update the pointer to the last mbuf of the current scattered
1652                  * packet and continue to parse the RX ring.
1653                  */
1654                 if (!(rx_stat_err0 & (1 << IAVF_RX_FLEX_DESC_STATUS0_EOF_S))) {
1655                         last_seg = rxm;
1656                         continue;
1657                 }
1658
1659                 /* This is the last buffer of the received packet. If the CRC
1660                  * is not stripped by the hardware:
1661                  *  - Subtract the CRC length from the total packet length.
1662                  *  - If the last buffer only contains the whole CRC or a part
1663                  *  of it, free the mbuf associated to the last buffer. If part
1664                  *  of the CRC is also contained in the previous mbuf, subtract
1665                  *  the length of that CRC part from the data length of the
1666                  *  previous mbuf.
1667                  */
1668                 rxm->next = NULL;
1669                 if (unlikely(rxq->crc_len > 0)) {
1670                         first_seg->pkt_len -= RTE_ETHER_CRC_LEN;
1671                         if (rx_packet_len <= RTE_ETHER_CRC_LEN) {
1672                                 rte_pktmbuf_free_seg(rxm);
1673                                 first_seg->nb_segs--;
1674                                 last_seg->data_len =
1675                                         (uint16_t)(last_seg->data_len -
1676                                         (RTE_ETHER_CRC_LEN - rx_packet_len));
1677                                 last_seg->next = NULL;
1678                         } else {
1679                                 rxm->data_len = (uint16_t)(rx_packet_len -
1680                                                         RTE_ETHER_CRC_LEN);
1681                         }
1682                 }
1683
1684                 first_seg->port = rxq->port_id;
1685                 first_seg->ol_flags = 0;
1686                 first_seg->packet_type = ptype_tbl[IAVF_RX_FLEX_DESC_PTYPE_M &
1687                         rte_le_to_cpu_16(rxd.wb.ptype_flex_flags0)];
1688                 iavf_flex_rxd_to_vlan_tci(first_seg, &rxd);
1689                 iavf_flex_rxd_to_ipsec_crypto_status(first_seg, &rxd,
1690                                 &rxq->stats.ipsec_crypto);
1691                 rxd_to_pkt_fields_ops[rxq->rxdid](rxq, first_seg, &rxd);
1692                 pkt_flags = iavf_flex_rxd_error_to_pkt_flags(rx_stat_err0);
1693
1694                 if (iavf_timestamp_dynflag > 0) {
1695                         ts_ns = iavf_tstamp_convert_32b_64b(ad->phc_time,
1696                                 rte_le_to_cpu_32(rxd.wb.flex_ts.ts_high));
1697
1698                         ad->phc_time = ts_ns;
1699                         ad->hw_time_update = rte_get_timer_cycles() / (rte_get_timer_hz() / 1000);
1700
1701                         *RTE_MBUF_DYNFIELD(first_seg,
1702                                 iavf_timestamp_dynfield_offset,
1703                                 rte_mbuf_timestamp_t *) = ts_ns;
1704                         first_seg->ol_flags |= iavf_timestamp_dynflag;
1705                 }
1706
1707                 first_seg->ol_flags |= pkt_flags;
1708
1709                 /* Prefetch data of first segment, if configured to do so. */
1710                 rte_prefetch0(RTE_PTR_ADD(first_seg->buf_addr,
1711                                           first_seg->data_off));
1712                 rx_pkts[nb_rx++] = first_seg;
1713                 first_seg = NULL;
1714         }
1715
1716         /* Record index of the next RX descriptor to probe. */
1717         rxq->rx_tail = rx_id;
1718         rxq->pkt_first_seg = first_seg;
1719         rxq->pkt_last_seg = last_seg;
1720
1721         iavf_update_rx_tail(rxq, nb_hold, rx_id);
1722
1723         return nb_rx;
1724 }
1725
1726 /* implement recv_scattered_pkts  */
1727 uint16_t
1728 iavf_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
1729                         uint16_t nb_pkts)
1730 {
1731         struct iavf_rx_queue *rxq = rx_queue;
1732         union iavf_rx_desc rxd;
1733         struct rte_mbuf *rxe;
1734         struct rte_mbuf *first_seg = rxq->pkt_first_seg;
1735         struct rte_mbuf *last_seg = rxq->pkt_last_seg;
1736         struct rte_mbuf *nmb, *rxm;
1737         uint16_t rx_id = rxq->rx_tail;
1738         uint16_t nb_rx = 0, nb_hold = 0, rx_packet_len;
1739         struct rte_eth_dev *dev;
1740         uint32_t rx_status;
1741         uint64_t qword1;
1742         uint64_t dma_addr;
1743         uint64_t pkt_flags;
1744
1745         volatile union iavf_rx_desc *rx_ring = rxq->rx_ring;
1746         volatile union iavf_rx_desc *rxdp;
1747         const uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl;
1748
1749         while (nb_rx < nb_pkts) {
1750                 rxdp = &rx_ring[rx_id];
1751                 qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len);
1752                 rx_status = (qword1 & IAVF_RXD_QW1_STATUS_MASK) >>
1753                             IAVF_RXD_QW1_STATUS_SHIFT;
1754
1755                 /* Check the DD bit */
1756                 if (!(rx_status & (1 << IAVF_RX_DESC_STATUS_DD_SHIFT)))
1757                         break;
1758                 IAVF_DUMP_RX_DESC(rxq, rxdp, rx_id);
1759
1760                 nmb = rte_mbuf_raw_alloc(rxq->mp);
1761                 if (unlikely(!nmb)) {
1762                         PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u "
1763                                    "queue_id=%u", rxq->port_id, rxq->queue_id);
1764                         dev = &rte_eth_devices[rxq->port_id];
1765                         dev->data->rx_mbuf_alloc_failed++;
1766                         break;
1767                 }
1768
1769                 rxd = *rxdp;
1770                 nb_hold++;
1771                 rxe = rxq->sw_ring[rx_id];
1772                 rxq->sw_ring[rx_id] = nmb;
1773                 rx_id++;
1774                 if (rx_id == rxq->nb_rx_desc)
1775                         rx_id = 0;
1776
1777                 /* Prefetch next mbuf */
1778                 rte_prefetch0(rxq->sw_ring[rx_id]);
1779
1780                 /* When next RX descriptor is on a cache line boundary,
1781                  * prefetch the next 4 RX descriptors and next 8 pointers
1782                  * to mbufs.
1783                  */
1784                 if ((rx_id & 0x3) == 0) {
1785                         rte_prefetch0(&rx_ring[rx_id]);
1786                         rte_prefetch0(rxq->sw_ring[rx_id]);
1787                 }
1788
1789                 rxm = rxe;
1790                 dma_addr =
1791                         rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
1792
1793                 /* Set data buffer address and data length of the mbuf */
1794                 rxdp->read.hdr_addr = 0;
1795                 rxdp->read.pkt_addr = dma_addr;
1796                 rx_packet_len = (qword1 & IAVF_RXD_QW1_LENGTH_PBUF_MASK) >>
1797                                  IAVF_RXD_QW1_LENGTH_PBUF_SHIFT;
1798                 rxm->data_len = rx_packet_len;
1799                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1800
1801                 /* If this is the first buffer of the received packet, set the
1802                  * pointer to the first mbuf of the packet and initialize its
1803                  * context. Otherwise, update the total length and the number
1804                  * of segments of the current scattered packet, and update the
1805                  * pointer to the last mbuf of the current packet.
1806                  */
1807                 if (!first_seg) {
1808                         first_seg = rxm;
1809                         first_seg->nb_segs = 1;
1810                         first_seg->pkt_len = rx_packet_len;
1811                 } else {
1812                         first_seg->pkt_len =
1813                                 (uint16_t)(first_seg->pkt_len +
1814                                                 rx_packet_len);
1815                         first_seg->nb_segs++;
1816                         last_seg->next = rxm;
1817                 }
1818
1819                 /* If this is not the last buffer of the received packet,
1820                  * update the pointer to the last mbuf of the current scattered
1821                  * packet and continue to parse the RX ring.
1822                  */
1823                 if (!(rx_status & (1 << IAVF_RX_DESC_STATUS_EOF_SHIFT))) {
1824                         last_seg = rxm;
1825                         continue;
1826                 }
1827
1828                 /* This is the last buffer of the received packet. If the CRC
1829                  * is not stripped by the hardware:
1830                  *  - Subtract the CRC length from the total packet length.
1831                  *  - If the last buffer only contains the whole CRC or a part
1832                  *  of it, free the mbuf associated to the last buffer. If part
1833                  *  of the CRC is also contained in the previous mbuf, subtract
1834                  *  the length of that CRC part from the data length of the
1835                  *  previous mbuf.
1836                  */
1837                 rxm->next = NULL;
1838                 if (unlikely(rxq->crc_len > 0)) {
1839                         first_seg->pkt_len -= RTE_ETHER_CRC_LEN;
1840                         if (rx_packet_len <= RTE_ETHER_CRC_LEN) {
1841                                 rte_pktmbuf_free_seg(rxm);
1842                                 first_seg->nb_segs--;
1843                                 last_seg->data_len =
1844                                         (uint16_t)(last_seg->data_len -
1845                                         (RTE_ETHER_CRC_LEN - rx_packet_len));
1846                                 last_seg->next = NULL;
1847                         } else
1848                                 rxm->data_len = (uint16_t)(rx_packet_len -
1849                                                         RTE_ETHER_CRC_LEN);
1850                 }
1851
1852                 first_seg->port = rxq->port_id;
1853                 first_seg->ol_flags = 0;
1854                 iavf_rxd_to_vlan_tci(first_seg, &rxd);
1855                 pkt_flags = iavf_rxd_to_pkt_flags(qword1);
1856                 first_seg->packet_type =
1857                         ptype_tbl[(uint8_t)((qword1 &
1858                         IAVF_RXD_QW1_PTYPE_MASK) >> IAVF_RXD_QW1_PTYPE_SHIFT)];
1859
1860                 if (pkt_flags & RTE_MBUF_F_RX_RSS_HASH)
1861                         first_seg->hash.rss =
1862                                 rte_le_to_cpu_32(rxd.wb.qword0.hi_dword.rss);
1863
1864                 if (pkt_flags & RTE_MBUF_F_RX_FDIR)
1865                         pkt_flags |= iavf_rxd_build_fdir(&rxd, first_seg);
1866
1867                 first_seg->ol_flags |= pkt_flags;
1868
1869                 /* Prefetch data of first segment, if configured to do so. */
1870                 rte_prefetch0(RTE_PTR_ADD(first_seg->buf_addr,
1871                                           first_seg->data_off));
1872                 rx_pkts[nb_rx++] = first_seg;
1873                 first_seg = NULL;
1874         }
1875
1876         /* Record index of the next RX descriptor to probe. */
1877         rxq->rx_tail = rx_id;
1878         rxq->pkt_first_seg = first_seg;
1879         rxq->pkt_last_seg = last_seg;
1880
1881         iavf_update_rx_tail(rxq, nb_hold, rx_id);
1882
1883         return nb_rx;
1884 }
1885
1886 #define IAVF_LOOK_AHEAD 8
1887 static inline int
1888 iavf_rx_scan_hw_ring_flex_rxd(struct iavf_rx_queue *rxq,
1889                             struct rte_mbuf **rx_pkts,
1890                             uint16_t nb_pkts)
1891 {
1892         volatile union iavf_rx_flex_desc *rxdp;
1893         struct rte_mbuf **rxep;
1894         struct rte_mbuf *mb;
1895         uint16_t stat_err0;
1896         uint16_t pkt_len;
1897         int32_t s[IAVF_LOOK_AHEAD], var, nb_dd;
1898         int32_t i, j, nb_rx = 0;
1899         int32_t nb_staged = 0;
1900         uint64_t pkt_flags;
1901         const uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl;
1902         struct iavf_adapter *ad = rxq->vsi->adapter;
1903         uint64_t ts_ns;
1904
1905         rxdp = (volatile union iavf_rx_flex_desc *)&rxq->rx_ring[rxq->rx_tail];
1906         rxep = &rxq->sw_ring[rxq->rx_tail];
1907
1908         stat_err0 = rte_le_to_cpu_16(rxdp->wb.status_error0);
1909
1910         /* Make sure there is at least 1 packet to receive */
1911         if (!(stat_err0 & (1 << IAVF_RX_FLEX_DESC_STATUS0_DD_S)))
1912                 return 0;
1913
1914         if (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) {
1915                 uint64_t sw_cur_time = rte_get_timer_cycles() / (rte_get_timer_hz() / 1000);
1916                 if (sw_cur_time - ad->hw_time_update > 4) {
1917                         if (iavf_get_phc_time(ad))
1918                                 PMD_DRV_LOG(ERR, "get physical time failed");
1919                         ad->hw_time_update = sw_cur_time;
1920                 }
1921         }
1922
1923         /* Scan LOOK_AHEAD descriptors at a time to determine which
1924          * descriptors reference packets that are ready to be received.
1925          */
1926         for (i = 0; i < IAVF_RX_MAX_BURST; i += IAVF_LOOK_AHEAD,
1927              rxdp += IAVF_LOOK_AHEAD, rxep += IAVF_LOOK_AHEAD) {
1928                 /* Read desc statuses backwards to avoid race condition */
1929                 for (j = IAVF_LOOK_AHEAD - 1; j >= 0; j--)
1930                         s[j] = rte_le_to_cpu_16(rxdp[j].wb.status_error0);
1931
1932                 /* This barrier is to order loads of different words in the descriptor */
1933                 rte_atomic_thread_fence(__ATOMIC_ACQUIRE);
1934
1935                 /* Compute how many contiguous DD bits were set */
1936                 for (j = 0, nb_dd = 0; j < IAVF_LOOK_AHEAD; j++) {
1937                         var = s[j] & (1 << IAVF_RX_FLEX_DESC_STATUS0_DD_S);
1938 #ifdef RTE_ARCH_ARM
1939                         /* For Arm platforms, count only contiguous descriptors
1940                          * whose DD bit is set to 1. On Arm platforms, reads of
1941                          * descriptors can be reordered. Since the CPU may
1942                          * be reading the descriptors as the NIC updates them
1943                          * in memory, it is possbile that the DD bit for a
1944                          * descriptor earlier in the queue is read as not set
1945                          * while the DD bit for a descriptor later in the queue
1946                          * is read as set.
1947                          */
1948                         if (var)
1949                                 nb_dd += 1;
1950                         else
1951                                 break;
1952 #else
1953                         nb_dd += var;
1954 #endif
1955                 }
1956
1957                 /* Translate descriptor info to mbuf parameters */
1958                 for (j = 0; j < nb_dd; j++) {
1959                         IAVF_DUMP_RX_DESC(rxq, &rxdp[j],
1960                                           rxq->rx_tail +
1961                                           i * IAVF_LOOK_AHEAD + j);
1962
1963                         mb = rxep[j];
1964                         pkt_len = (rte_le_to_cpu_16(rxdp[j].wb.pkt_len) &
1965                                 IAVF_RX_FLX_DESC_PKT_LEN_M) - rxq->crc_len;
1966                         mb->data_len = pkt_len;
1967                         mb->pkt_len = pkt_len;
1968                         mb->ol_flags = 0;
1969
1970                         mb->packet_type = ptype_tbl[IAVF_RX_FLEX_DESC_PTYPE_M &
1971                                 rte_le_to_cpu_16(rxdp[j].wb.ptype_flex_flags0)];
1972                         iavf_flex_rxd_to_vlan_tci(mb, &rxdp[j]);
1973                         iavf_flex_rxd_to_ipsec_crypto_status(mb, &rxdp[j],
1974                                 &rxq->stats.ipsec_crypto);
1975                         rxd_to_pkt_fields_ops[rxq->rxdid](rxq, mb, &rxdp[j]);
1976                         stat_err0 = rte_le_to_cpu_16(rxdp[j].wb.status_error0);
1977                         pkt_flags = iavf_flex_rxd_error_to_pkt_flags(stat_err0);
1978
1979                         if (iavf_timestamp_dynflag > 0) {
1980                                 ts_ns = iavf_tstamp_convert_32b_64b(ad->phc_time,
1981                                         rte_le_to_cpu_32(rxdp[j].wb.flex_ts.ts_high));
1982
1983                                 ad->phc_time = ts_ns;
1984                                 ad->hw_time_update = rte_get_timer_cycles() / (rte_get_timer_hz() / 1000);
1985
1986                                 *RTE_MBUF_DYNFIELD(mb,
1987                                         iavf_timestamp_dynfield_offset,
1988                                         rte_mbuf_timestamp_t *) = ts_ns;
1989                                 mb->ol_flags |= iavf_timestamp_dynflag;
1990                         }
1991
1992                         mb->ol_flags |= pkt_flags;
1993
1994                         /* Put up to nb_pkts directly into buffers */
1995                         if ((i + j) < nb_pkts) {
1996                                 rx_pkts[i + j] = rxep[j];
1997                                 nb_rx++;
1998                         } else {
1999                                 /* Stage excess pkts received */
2000                                 rxq->rx_stage[nb_staged] = rxep[j];
2001                                 nb_staged++;
2002                         }
2003                 }
2004
2005                 if (nb_dd != IAVF_LOOK_AHEAD)
2006                         break;
2007         }
2008
2009         /* Update rxq->rx_nb_avail to reflect number of staged pkts */
2010         rxq->rx_nb_avail = nb_staged;
2011
2012         /* Clear software ring entries */
2013         for (i = 0; i < (nb_rx + nb_staged); i++)
2014                 rxq->sw_ring[rxq->rx_tail + i] = NULL;
2015
2016         return nb_rx;
2017 }
2018
2019 static inline int
2020 iavf_rx_scan_hw_ring(struct iavf_rx_queue *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
2021 {
2022         volatile union iavf_rx_desc *rxdp;
2023         struct rte_mbuf **rxep;
2024         struct rte_mbuf *mb;
2025         uint16_t pkt_len;
2026         uint64_t qword1;
2027         uint32_t rx_status;
2028         int32_t s[IAVF_LOOK_AHEAD], var, nb_dd;
2029         int32_t i, j, nb_rx = 0;
2030         int32_t nb_staged = 0;
2031         uint64_t pkt_flags;
2032         const uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl;
2033
2034         rxdp = &rxq->rx_ring[rxq->rx_tail];
2035         rxep = &rxq->sw_ring[rxq->rx_tail];
2036
2037         qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len);
2038         rx_status = (qword1 & IAVF_RXD_QW1_STATUS_MASK) >>
2039                     IAVF_RXD_QW1_STATUS_SHIFT;
2040
2041         /* Make sure there is at least 1 packet to receive */
2042         if (!(rx_status & (1 << IAVF_RX_DESC_STATUS_DD_SHIFT)))
2043                 return 0;
2044
2045         /* Scan LOOK_AHEAD descriptors at a time to determine which
2046          * descriptors reference packets that are ready to be received.
2047          */
2048         for (i = 0; i < IAVF_RX_MAX_BURST; i += IAVF_LOOK_AHEAD,
2049              rxdp += IAVF_LOOK_AHEAD, rxep += IAVF_LOOK_AHEAD) {
2050                 /* Read desc statuses backwards to avoid race condition */
2051                 for (j = IAVF_LOOK_AHEAD - 1; j >= 0; j--) {
2052                         qword1 = rte_le_to_cpu_64(
2053                                 rxdp[j].wb.qword1.status_error_len);
2054                         s[j] = (qword1 & IAVF_RXD_QW1_STATUS_MASK) >>
2055                                IAVF_RXD_QW1_STATUS_SHIFT;
2056                 }
2057
2058                 /* This barrier is to order loads of different words in the descriptor */
2059                 rte_atomic_thread_fence(__ATOMIC_ACQUIRE);
2060
2061                 /* Compute how many contiguous DD bits were set */
2062                 for (j = 0, nb_dd = 0; j < IAVF_LOOK_AHEAD; j++) {
2063                         var = s[j] & (1 << IAVF_RX_DESC_STATUS_DD_SHIFT);
2064 #ifdef RTE_ARCH_ARM
2065                         /* For Arm platforms, count only contiguous descriptors
2066                          * whose DD bit is set to 1. On Arm platforms, reads of
2067                          * descriptors can be reordered. Since the CPU may
2068                          * be reading the descriptors as the NIC updates them
2069                          * in memory, it is possbile that the DD bit for a
2070                          * descriptor earlier in the queue is read as not set
2071                          * while the DD bit for a descriptor later in the queue
2072                          * is read as set.
2073                          */
2074                         if (var)
2075                                 nb_dd += 1;
2076                         else
2077                                 break;
2078 #else
2079                         nb_dd += var;
2080 #endif
2081                 }
2082
2083                 /* Translate descriptor info to mbuf parameters */
2084                 for (j = 0; j < nb_dd; j++) {
2085                         IAVF_DUMP_RX_DESC(rxq, &rxdp[j],
2086                                          rxq->rx_tail + i * IAVF_LOOK_AHEAD + j);
2087
2088                         mb = rxep[j];
2089                         qword1 = rte_le_to_cpu_64
2090                                         (rxdp[j].wb.qword1.status_error_len);
2091                         pkt_len = ((qword1 & IAVF_RXD_QW1_LENGTH_PBUF_MASK) >>
2092                                   IAVF_RXD_QW1_LENGTH_PBUF_SHIFT) - rxq->crc_len;
2093                         mb->data_len = pkt_len;
2094                         mb->pkt_len = pkt_len;
2095                         mb->ol_flags = 0;
2096                         iavf_rxd_to_vlan_tci(mb, &rxdp[j]);
2097                         pkt_flags = iavf_rxd_to_pkt_flags(qword1);
2098                         mb->packet_type =
2099                                 ptype_tbl[(uint8_t)((qword1 &
2100                                 IAVF_RXD_QW1_PTYPE_MASK) >>
2101                                 IAVF_RXD_QW1_PTYPE_SHIFT)];
2102
2103                         if (pkt_flags & RTE_MBUF_F_RX_RSS_HASH)
2104                                 mb->hash.rss = rte_le_to_cpu_32(
2105                                         rxdp[j].wb.qword0.hi_dword.rss);
2106
2107                         if (pkt_flags & RTE_MBUF_F_RX_FDIR)
2108                                 pkt_flags |= iavf_rxd_build_fdir(&rxdp[j], mb);
2109
2110                         mb->ol_flags |= pkt_flags;
2111
2112                         /* Put up to nb_pkts directly into buffers */
2113                         if ((i + j) < nb_pkts) {
2114                                 rx_pkts[i + j] = rxep[j];
2115                                 nb_rx++;
2116                         } else { /* Stage excess pkts received */
2117                                 rxq->rx_stage[nb_staged] = rxep[j];
2118                                 nb_staged++;
2119                         }
2120                 }
2121
2122                 if (nb_dd != IAVF_LOOK_AHEAD)
2123                         break;
2124         }
2125
2126         /* Update rxq->rx_nb_avail to reflect number of staged pkts */
2127         rxq->rx_nb_avail = nb_staged;
2128
2129         /* Clear software ring entries */
2130         for (i = 0; i < (nb_rx + nb_staged); i++)
2131                 rxq->sw_ring[rxq->rx_tail + i] = NULL;
2132
2133         return nb_rx;
2134 }
2135
2136 static inline uint16_t
2137 iavf_rx_fill_from_stage(struct iavf_rx_queue *rxq,
2138                        struct rte_mbuf **rx_pkts,
2139                        uint16_t nb_pkts)
2140 {
2141         uint16_t i;
2142         struct rte_mbuf **stage = &rxq->rx_stage[rxq->rx_next_avail];
2143
2144         nb_pkts = (uint16_t)RTE_MIN(nb_pkts, rxq->rx_nb_avail);
2145
2146         for (i = 0; i < nb_pkts; i++)
2147                 rx_pkts[i] = stage[i];
2148
2149         rxq->rx_nb_avail = (uint16_t)(rxq->rx_nb_avail - nb_pkts);
2150         rxq->rx_next_avail = (uint16_t)(rxq->rx_next_avail + nb_pkts);
2151
2152         return nb_pkts;
2153 }
2154
2155 static inline int
2156 iavf_rx_alloc_bufs(struct iavf_rx_queue *rxq)
2157 {
2158         volatile union iavf_rx_desc *rxdp;
2159         struct rte_mbuf **rxep;
2160         struct rte_mbuf *mb;
2161         uint16_t alloc_idx, i;
2162         uint64_t dma_addr;
2163         int diag;
2164
2165         /* Allocate buffers in bulk */
2166         alloc_idx = (uint16_t)(rxq->rx_free_trigger -
2167                                 (rxq->rx_free_thresh - 1));
2168         rxep = &rxq->sw_ring[alloc_idx];
2169         diag = rte_mempool_get_bulk(rxq->mp, (void *)rxep,
2170                                     rxq->rx_free_thresh);
2171         if (unlikely(diag != 0)) {
2172                 PMD_RX_LOG(ERR, "Failed to get mbufs in bulk");
2173                 return -ENOMEM;
2174         }
2175
2176         rxdp = &rxq->rx_ring[alloc_idx];
2177         for (i = 0; i < rxq->rx_free_thresh; i++) {
2178                 if (likely(i < (rxq->rx_free_thresh - 1)))
2179                         /* Prefetch next mbuf */
2180                         rte_prefetch0(rxep[i + 1]);
2181
2182                 mb = rxep[i];
2183                 rte_mbuf_refcnt_set(mb, 1);
2184                 mb->next = NULL;
2185                 mb->data_off = RTE_PKTMBUF_HEADROOM;
2186                 mb->nb_segs = 1;
2187                 mb->port = rxq->port_id;
2188                 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mb));
2189                 rxdp[i].read.hdr_addr = 0;
2190                 rxdp[i].read.pkt_addr = dma_addr;
2191         }
2192
2193         /* Update rx tail register */
2194         rte_wmb();
2195         IAVF_PCI_REG_WC_WRITE_RELAXED(rxq->qrx_tail, rxq->rx_free_trigger);
2196
2197         rxq->rx_free_trigger =
2198                 (uint16_t)(rxq->rx_free_trigger + rxq->rx_free_thresh);
2199         if (rxq->rx_free_trigger >= rxq->nb_rx_desc)
2200                 rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1);
2201
2202         return 0;
2203 }
2204
2205 static inline uint16_t
2206 rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
2207 {
2208         struct iavf_rx_queue *rxq = (struct iavf_rx_queue *)rx_queue;
2209         uint16_t nb_rx = 0;
2210
2211         if (!nb_pkts)
2212                 return 0;
2213
2214         if (rxq->rx_nb_avail)
2215                 return iavf_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
2216
2217         if (rxq->rxdid >= IAVF_RXDID_FLEX_NIC && rxq->rxdid <= IAVF_RXDID_LAST)
2218                 nb_rx = (uint16_t)iavf_rx_scan_hw_ring_flex_rxd(rxq, rx_pkts, nb_pkts);
2219         else
2220                 nb_rx = (uint16_t)iavf_rx_scan_hw_ring(rxq, rx_pkts, nb_pkts);
2221
2222         rxq->rx_next_avail = 0;
2223         rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_rx + rxq->rx_nb_avail);
2224
2225         if (rxq->rx_tail > rxq->rx_free_trigger) {
2226                 if (iavf_rx_alloc_bufs(rxq) != 0) {
2227                         uint16_t i, j, nb_staged;
2228
2229                         /* TODO: count rx_mbuf_alloc_failed here */
2230
2231                         nb_staged = rxq->rx_nb_avail;
2232                         rxq->rx_nb_avail = 0;
2233
2234                         rxq->rx_tail = (uint16_t)(rxq->rx_tail - (nb_rx + nb_staged));
2235                         for (i = 0, j = rxq->rx_tail; i < nb_rx; i++, j++) {
2236                                 rxq->sw_ring[j] = rx_pkts[i];
2237                                 rx_pkts[i] = NULL;
2238                         }
2239                         for (i = 0, j = rxq->rx_tail + nb_rx; i < nb_staged; i++, j++) {
2240                                 rxq->sw_ring[j] = rxq->rx_stage[i];
2241                                 rx_pkts[i] = NULL;
2242                         }
2243
2244                         return 0;
2245                 }
2246         }
2247
2248         if (rxq->rx_tail >= rxq->nb_rx_desc)
2249                 rxq->rx_tail = 0;
2250
2251         PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u, nb_rx=%u",
2252                    rxq->port_id, rxq->queue_id,
2253                    rxq->rx_tail, nb_rx);
2254
2255         return nb_rx;
2256 }
2257
2258 static uint16_t
2259 iavf_recv_pkts_bulk_alloc(void *rx_queue,
2260                          struct rte_mbuf **rx_pkts,
2261                          uint16_t nb_pkts)
2262 {
2263         uint16_t nb_rx = 0, n, count;
2264
2265         if (unlikely(nb_pkts == 0))
2266                 return 0;
2267
2268         if (likely(nb_pkts <= IAVF_RX_MAX_BURST))
2269                 return rx_recv_pkts(rx_queue, rx_pkts, nb_pkts);
2270
2271         while (nb_pkts) {
2272                 n = RTE_MIN(nb_pkts, IAVF_RX_MAX_BURST);
2273                 count = rx_recv_pkts(rx_queue, &rx_pkts[nb_rx], n);
2274                 nb_rx = (uint16_t)(nb_rx + count);
2275                 nb_pkts = (uint16_t)(nb_pkts - count);
2276                 if (count < n)
2277                         break;
2278         }
2279
2280         return nb_rx;
2281 }
2282
2283 static inline int
2284 iavf_xmit_cleanup(struct iavf_tx_queue *txq)
2285 {
2286         struct iavf_tx_entry *sw_ring = txq->sw_ring;
2287         uint16_t last_desc_cleaned = txq->last_desc_cleaned;
2288         uint16_t nb_tx_desc = txq->nb_tx_desc;
2289         uint16_t desc_to_clean_to;
2290         uint16_t nb_tx_to_clean;
2291
2292         volatile struct iavf_tx_desc *txd = txq->tx_ring;
2293
2294         desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->rs_thresh);
2295         if (desc_to_clean_to >= nb_tx_desc)
2296                 desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc);
2297
2298         desc_to_clean_to = sw_ring[desc_to_clean_to].last_id;
2299         if ((txd[desc_to_clean_to].cmd_type_offset_bsz &
2300                         rte_cpu_to_le_64(IAVF_TXD_QW1_DTYPE_MASK)) !=
2301                         rte_cpu_to_le_64(IAVF_TX_DESC_DTYPE_DESC_DONE)) {
2302                 PMD_TX_LOG(DEBUG, "TX descriptor %4u is not done "
2303                            "(port=%d queue=%d)", desc_to_clean_to,
2304                            txq->port_id, txq->queue_id);
2305                 return -1;
2306         }
2307
2308         if (last_desc_cleaned > desc_to_clean_to)
2309                 nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) +
2310                                                         desc_to_clean_to);
2311         else
2312                 nb_tx_to_clean = (uint16_t)(desc_to_clean_to -
2313                                         last_desc_cleaned);
2314
2315         txd[desc_to_clean_to].cmd_type_offset_bsz = 0;
2316
2317         txq->last_desc_cleaned = desc_to_clean_to;
2318         txq->nb_free = (uint16_t)(txq->nb_free + nb_tx_to_clean);
2319
2320         return 0;
2321 }
2322
2323 /* Check if the context descriptor is needed for TX offloading */
2324 static inline uint16_t
2325 iavf_calc_context_desc(uint64_t flags, uint8_t vlan_flag)
2326 {
2327         if (flags & (RTE_MBUF_F_TX_TCP_SEG | RTE_MBUF_F_TX_UDP_SEG |
2328                         RTE_MBUF_F_TX_TUNNEL_MASK))
2329                 return 1;
2330         if (flags & RTE_MBUF_F_TX_VLAN &&
2331             vlan_flag & IAVF_TX_FLAGS_VLAN_TAG_LOC_L2TAG2)
2332                 return 1;
2333         return 0;
2334 }
2335
2336 static inline void
2337 iavf_fill_ctx_desc_cmd_field(volatile uint64_t *field, struct rte_mbuf *m,
2338                 uint8_t vlan_flag)
2339 {
2340         uint64_t cmd = 0;
2341
2342         /* TSO enabled */
2343         if (m->ol_flags & (RTE_MBUF_F_TX_TCP_SEG | RTE_MBUF_F_TX_UDP_SEG))
2344                 cmd = IAVF_TX_CTX_DESC_TSO << IAVF_TXD_CTX_QW1_CMD_SHIFT;
2345
2346         if (m->ol_flags & RTE_MBUF_F_TX_VLAN &&
2347                         vlan_flag & IAVF_TX_FLAGS_VLAN_TAG_LOC_L2TAG2) {
2348                 cmd |= IAVF_TX_CTX_DESC_IL2TAG2
2349                         << IAVF_TXD_CTX_QW1_CMD_SHIFT;
2350         }
2351
2352         *field |= cmd;
2353 }
2354
2355 static inline void
2356 iavf_fill_ctx_desc_ipsec_field(volatile uint64_t *field,
2357         struct iavf_ipsec_crypto_pkt_metadata *ipsec_md)
2358 {
2359         uint64_t ipsec_field =
2360                 (uint64_t)ipsec_md->ctx_desc_ipsec_params <<
2361                         IAVF_TXD_CTX_QW1_IPSEC_PARAMS_CIPHERBLK_SHIFT;
2362
2363         *field |= ipsec_field;
2364 }
2365
2366
2367 static inline void
2368 iavf_fill_ctx_desc_tunnelling_field(volatile uint64_t *qw0,
2369                 const struct rte_mbuf *m)
2370 {
2371         uint64_t eip_typ = IAVF_TX_CTX_DESC_EIPT_NONE;
2372         uint64_t eip_len = 0;
2373         uint64_t eip_noinc = 0;
2374         /* Default - IP_ID is increment in each segment of LSO */
2375
2376         switch (m->ol_flags & (RTE_MBUF_F_TX_OUTER_IPV4 |
2377                         RTE_MBUF_F_TX_OUTER_IPV6 |
2378                         RTE_MBUF_F_TX_OUTER_IP_CKSUM)) {
2379         case RTE_MBUF_F_TX_OUTER_IPV4:
2380                 eip_typ = IAVF_TX_CTX_DESC_EIPT_IPV4_NO_CHECKSUM_OFFLOAD;
2381                 eip_len = m->outer_l3_len >> 2;
2382         break;
2383         case RTE_MBUF_F_TX_OUTER_IPV4 | RTE_MBUF_F_TX_OUTER_IP_CKSUM:
2384                 eip_typ = IAVF_TX_CTX_DESC_EIPT_IPV4_CHECKSUM_OFFLOAD;
2385                 eip_len = m->outer_l3_len >> 2;
2386         break;
2387         case RTE_MBUF_F_TX_OUTER_IPV6:
2388                 eip_typ = IAVF_TX_CTX_DESC_EIPT_IPV6;
2389                 eip_len = m->outer_l3_len >> 2;
2390         break;
2391         }
2392
2393         *qw0 = eip_typ << IAVF_TXD_CTX_QW0_TUN_PARAMS_EIPT_SHIFT |
2394                 eip_len << IAVF_TXD_CTX_QW0_TUN_PARAMS_EIPLEN_SHIFT |
2395                 eip_noinc << IAVF_TXD_CTX_QW0_TUN_PARAMS_EIP_NOINC_SHIFT;
2396 }
2397
2398 static inline uint16_t
2399 iavf_fill_ctx_desc_segmentation_field(volatile uint64_t *field,
2400         struct rte_mbuf *m, struct iavf_ipsec_crypto_pkt_metadata *ipsec_md)
2401 {
2402         uint64_t segmentation_field = 0;
2403         uint64_t total_length = 0;
2404
2405         if (m->ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD) {
2406                 total_length = ipsec_md->l4_payload_len;
2407         } else {
2408                 total_length = m->pkt_len - (m->l2_len + m->l3_len + m->l4_len);
2409
2410                 if (m->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK)
2411                         total_length -= m->outer_l3_len;
2412         }
2413
2414 #ifdef RTE_LIBRTE_IAVF_DEBUG_TX
2415         if (!m->l4_len || !m->tso_segsz)
2416                 PMD_TX_LOG(DEBUG, "L4 length %d, LSO Segment size %d",
2417                          m->l4_len, m->tso_segsz);
2418         if (m->tso_segsz < 88)
2419                 PMD_TX_LOG(DEBUG, "LSO Segment size %d is less than minimum %d",
2420                         m->tso_segsz, 88);
2421 #endif
2422         segmentation_field =
2423                 (((uint64_t)total_length << IAVF_TXD_CTX_QW1_TSO_LEN_SHIFT) &
2424                                 IAVF_TXD_CTX_QW1_TSO_LEN_MASK) |
2425                 (((uint64_t)m->tso_segsz << IAVF_TXD_CTX_QW1_MSS_SHIFT) &
2426                                 IAVF_TXD_CTX_QW1_MSS_MASK);
2427
2428         *field |= segmentation_field;
2429
2430         return total_length;
2431 }
2432
2433
2434 struct iavf_tx_context_desc_qws {
2435         __le64 qw0;
2436         __le64 qw1;
2437 };
2438
2439 static inline void
2440 iavf_fill_context_desc(volatile struct iavf_tx_context_desc *desc,
2441         struct rte_mbuf *m, struct iavf_ipsec_crypto_pkt_metadata *ipsec_md,
2442         uint16_t *tlen, uint8_t vlan_flag)
2443 {
2444         volatile struct iavf_tx_context_desc_qws *desc_qws =
2445                         (volatile struct iavf_tx_context_desc_qws *)desc;
2446         /* fill descriptor type field */
2447         desc_qws->qw1 = IAVF_TX_DESC_DTYPE_CONTEXT;
2448
2449         /* fill command field */
2450         iavf_fill_ctx_desc_cmd_field(&desc_qws->qw1, m, vlan_flag);
2451
2452         /* fill segmentation field */
2453         if (m->ol_flags & (RTE_MBUF_F_TX_TCP_SEG | RTE_MBUF_F_TX_UDP_SEG)) {
2454                 /* fill IPsec field */
2455                 if (m->ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD)
2456                         iavf_fill_ctx_desc_ipsec_field(&desc_qws->qw1,
2457                                 ipsec_md);
2458
2459                 *tlen = iavf_fill_ctx_desc_segmentation_field(&desc_qws->qw1,
2460                                 m, ipsec_md);
2461         }
2462
2463         /* fill tunnelling field */
2464         if (m->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK)
2465                 iavf_fill_ctx_desc_tunnelling_field(&desc_qws->qw0, m);
2466         else
2467                 desc_qws->qw0 = 0;
2468
2469         desc_qws->qw0 = rte_cpu_to_le_64(desc_qws->qw0);
2470         desc_qws->qw1 = rte_cpu_to_le_64(desc_qws->qw1);
2471
2472         if (vlan_flag & IAVF_TX_FLAGS_VLAN_TAG_LOC_L2TAG2)
2473                 desc->l2tag2 = m->vlan_tci;
2474 }
2475
2476
2477 static inline void
2478 iavf_fill_ipsec_desc(volatile struct iavf_tx_ipsec_desc *desc,
2479         const struct iavf_ipsec_crypto_pkt_metadata *md, uint16_t *ipsec_len)
2480 {
2481         desc->qw0 = rte_cpu_to_le_64(((uint64_t)md->l4_payload_len <<
2482                 IAVF_IPSEC_TX_DESC_QW0_L4PAYLEN_SHIFT) |
2483                 ((uint64_t)md->esn << IAVF_IPSEC_TX_DESC_QW0_IPSECESN_SHIFT) |
2484                 ((uint64_t)md->esp_trailer_len <<
2485                                 IAVF_IPSEC_TX_DESC_QW0_TRAILERLEN_SHIFT));
2486
2487         desc->qw1 = rte_cpu_to_le_64(((uint64_t)md->sa_idx <<
2488                 IAVF_IPSEC_TX_DESC_QW1_IPSECSA_SHIFT) |
2489                 ((uint64_t)md->next_proto <<
2490                                 IAVF_IPSEC_TX_DESC_QW1_IPSECNH_SHIFT) |
2491                 ((uint64_t)(md->len_iv & 0x3) <<
2492                                 IAVF_IPSEC_TX_DESC_QW1_IVLEN_SHIFT) |
2493                 ((uint64_t)(md->ol_flags & IAVF_IPSEC_CRYPTO_OL_FLAGS_NATT ?
2494                                 1ULL : 0ULL) <<
2495                                 IAVF_IPSEC_TX_DESC_QW1_UDP_SHIFT) |
2496                 (uint64_t)IAVF_TX_DESC_DTYPE_IPSEC);
2497
2498         /**
2499          * TODO: Pre-calculate this in the Session initialization
2500          *
2501          * Calculate IPsec length required in data descriptor func when TSO
2502          * offload is enabled
2503          */
2504         *ipsec_len = sizeof(struct rte_esp_hdr) + (md->len_iv >> 2) +
2505                         (md->ol_flags & IAVF_IPSEC_CRYPTO_OL_FLAGS_NATT ?
2506                         sizeof(struct rte_udp_hdr) : 0);
2507 }
2508
2509 static inline void
2510 iavf_build_data_desc_cmd_offset_fields(volatile uint64_t *qw1,
2511                 struct rte_mbuf *m, uint8_t vlan_flag)
2512 {
2513         uint64_t command = 0;
2514         uint64_t offset = 0;
2515         uint64_t l2tag1 = 0;
2516
2517         *qw1 = IAVF_TX_DESC_DTYPE_DATA;
2518
2519         command = (uint64_t)IAVF_TX_DESC_CMD_ICRC;
2520
2521         /* Descriptor based VLAN insertion */
2522         if ((vlan_flag & IAVF_TX_FLAGS_VLAN_TAG_LOC_L2TAG1) &&
2523                         m->ol_flags & RTE_MBUF_F_TX_VLAN) {
2524                 command |= (uint64_t)IAVF_TX_DESC_CMD_IL2TAG1;
2525                 l2tag1 |= m->vlan_tci;
2526         }
2527
2528         /* Set MACLEN */
2529         offset |= (m->l2_len >> 1) << IAVF_TX_DESC_LENGTH_MACLEN_SHIFT;
2530
2531         /* Enable L3 checksum offloading inner */
2532         if (m->ol_flags & (RTE_MBUF_F_TX_IP_CKSUM | RTE_MBUF_F_TX_IPV4)) {
2533                 command |= IAVF_TX_DESC_CMD_IIPT_IPV4_CSUM;
2534                 offset |= (m->l3_len >> 2) << IAVF_TX_DESC_LENGTH_IPLEN_SHIFT;
2535         } else if (m->ol_flags & RTE_MBUF_F_TX_IPV4) {
2536                 command |= IAVF_TX_DESC_CMD_IIPT_IPV4;
2537                 offset |= (m->l3_len >> 2) << IAVF_TX_DESC_LENGTH_IPLEN_SHIFT;
2538         } else if (m->ol_flags & RTE_MBUF_F_TX_IPV6) {
2539                 command |= IAVF_TX_DESC_CMD_IIPT_IPV6;
2540                 offset |= (m->l3_len >> 2) << IAVF_TX_DESC_LENGTH_IPLEN_SHIFT;
2541         }
2542
2543         if (m->ol_flags & RTE_MBUF_F_TX_TCP_SEG) {
2544                 command |= IAVF_TX_DESC_CMD_L4T_EOFT_TCP;
2545                 offset |= (m->l4_len >> 2) <<
2546                               IAVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
2547         }
2548
2549         /* Enable L4 checksum offloads */
2550         switch (m->ol_flags & RTE_MBUF_F_TX_L4_MASK) {
2551         case RTE_MBUF_F_TX_TCP_CKSUM:
2552                 command |= IAVF_TX_DESC_CMD_L4T_EOFT_TCP;
2553                 offset |= (sizeof(struct rte_tcp_hdr) >> 2) <<
2554                                 IAVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
2555                 break;
2556         case RTE_MBUF_F_TX_SCTP_CKSUM:
2557                 command |= IAVF_TX_DESC_CMD_L4T_EOFT_SCTP;
2558                 offset |= (sizeof(struct rte_sctp_hdr) >> 2) <<
2559                                 IAVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
2560                 break;
2561         case RTE_MBUF_F_TX_UDP_CKSUM:
2562                 command |= IAVF_TX_DESC_CMD_L4T_EOFT_UDP;
2563                 offset |= (sizeof(struct rte_udp_hdr) >> 2) <<
2564                                 IAVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
2565                 break;
2566         }
2567
2568         *qw1 = rte_cpu_to_le_64((((uint64_t)command <<
2569                 IAVF_TXD_DATA_QW1_CMD_SHIFT) & IAVF_TXD_DATA_QW1_CMD_MASK) |
2570                 (((uint64_t)offset << IAVF_TXD_DATA_QW1_OFFSET_SHIFT) &
2571                 IAVF_TXD_DATA_QW1_OFFSET_MASK) |
2572                 ((uint64_t)l2tag1 << IAVF_TXD_DATA_QW1_L2TAG1_SHIFT));
2573 }
2574
2575 static inline void
2576 iavf_fill_data_desc(volatile struct iavf_tx_desc *desc,
2577         struct rte_mbuf *m, uint64_t desc_template,
2578         uint16_t tlen, uint16_t ipseclen)
2579 {
2580         uint32_t hdrlen = m->l2_len;
2581         uint32_t bufsz = 0;
2582
2583         /* fill data descriptor qw1 from template */
2584         desc->cmd_type_offset_bsz = desc_template;
2585
2586         /* set data buffer address */
2587         desc->buffer_addr = rte_mbuf_data_iova(m);
2588
2589         /* calculate data buffer size less set header lengths */
2590         if ((m->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) &&
2591                         (m->ol_flags & (RTE_MBUF_F_TX_TCP_SEG |
2592                                         RTE_MBUF_F_TX_UDP_SEG))) {
2593                 hdrlen += m->outer_l3_len;
2594                 if (m->ol_flags & RTE_MBUF_F_TX_L4_MASK)
2595                         hdrlen += m->l3_len + m->l4_len;
2596                 else
2597                         hdrlen += m->l3_len;
2598                 if (m->ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD)
2599                         hdrlen += ipseclen;
2600                 bufsz = hdrlen + tlen;
2601         } else if ((m->ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD) &&
2602                         (m->ol_flags & (RTE_MBUF_F_TX_TCP_SEG |
2603                                         RTE_MBUF_F_TX_UDP_SEG))) {
2604                 hdrlen += m->outer_l3_len + m->l3_len + ipseclen;
2605                 if (m->ol_flags & RTE_MBUF_F_TX_L4_MASK)
2606                         hdrlen += m->l4_len;
2607                 bufsz = hdrlen + tlen;
2608
2609         } else {
2610                 bufsz = m->data_len;
2611         }
2612
2613         /* set data buffer size */
2614         desc->cmd_type_offset_bsz |=
2615                 (((uint64_t)bufsz << IAVF_TXD_DATA_QW1_TX_BUF_SZ_SHIFT) &
2616                 IAVF_TXD_DATA_QW1_TX_BUF_SZ_MASK);
2617
2618         desc->buffer_addr = rte_cpu_to_le_64(desc->buffer_addr);
2619         desc->cmd_type_offset_bsz = rte_cpu_to_le_64(desc->cmd_type_offset_bsz);
2620 }
2621
2622
2623 static struct iavf_ipsec_crypto_pkt_metadata *
2624 iavf_ipsec_crypto_get_pkt_metadata(const struct iavf_tx_queue *txq,
2625                 struct rte_mbuf *m)
2626 {
2627         if (m->ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD)
2628                 return RTE_MBUF_DYNFIELD(m, txq->ipsec_crypto_pkt_md_offset,
2629                                 struct iavf_ipsec_crypto_pkt_metadata *);
2630
2631         return NULL;
2632 }
2633
2634 /* TX function */
2635 uint16_t
2636 iavf_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
2637 {
2638         struct iavf_tx_queue *txq = tx_queue;
2639         volatile struct iavf_tx_desc *txr = txq->tx_ring;
2640         struct iavf_tx_entry *txe_ring = txq->sw_ring;
2641         struct iavf_tx_entry *txe, *txn;
2642         struct rte_mbuf *mb, *mb_seg;
2643         uint16_t desc_idx, desc_idx_last;
2644         uint16_t idx;
2645
2646
2647         /* Check if the descriptor ring needs to be cleaned. */
2648         if (txq->nb_free < txq->free_thresh)
2649                 iavf_xmit_cleanup(txq);
2650
2651         desc_idx = txq->tx_tail;
2652         txe = &txe_ring[desc_idx];
2653
2654 #ifdef RTE_LIBRTE_IAVF_DEBUG_TX_DESC_RING
2655                 iavf_dump_tx_entry_ring(txq);
2656                 iavf_dump_tx_desc_ring(txq);
2657 #endif
2658
2659
2660         for (idx = 0; idx < nb_pkts; idx++) {
2661                 volatile struct iavf_tx_desc *ddesc;
2662                 struct iavf_ipsec_crypto_pkt_metadata *ipsec_md;
2663
2664                 uint16_t nb_desc_ctx, nb_desc_ipsec;
2665                 uint16_t nb_desc_data, nb_desc_required;
2666                 uint16_t tlen = 0, ipseclen = 0;
2667                 uint64_t ddesc_template = 0;
2668                 uint64_t ddesc_cmd = 0;
2669
2670                 mb = tx_pkts[idx];
2671
2672                 RTE_MBUF_PREFETCH_TO_FREE(txe->mbuf);
2673
2674                 /**
2675                  * Get metadata for ipsec crypto from mbuf dynamic fields if
2676                  * security offload is specified.
2677                  */
2678                 ipsec_md = iavf_ipsec_crypto_get_pkt_metadata(txq, mb);
2679
2680                 nb_desc_data = mb->nb_segs;
2681                 nb_desc_ctx =
2682                         iavf_calc_context_desc(mb->ol_flags, txq->vlan_flag);
2683                 nb_desc_ipsec = !!(mb->ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD);
2684
2685                 /**
2686                  * The number of descriptors that must be allocated for
2687                  * a packet equals to the number of the segments of that
2688                  * packet plus the context and ipsec descriptors if needed.
2689                  */
2690                 nb_desc_required = nb_desc_data + nb_desc_ctx + nb_desc_ipsec;
2691
2692                 desc_idx_last = (uint16_t)(desc_idx + nb_desc_required - 1);
2693
2694                 /* wrap descriptor ring */
2695                 if (desc_idx_last >= txq->nb_tx_desc)
2696                         desc_idx_last =
2697                                 (uint16_t)(desc_idx_last - txq->nb_tx_desc);
2698
2699                 PMD_TX_LOG(DEBUG,
2700                         "port_id=%u queue_id=%u tx_first=%u tx_last=%u",
2701                         txq->port_id, txq->queue_id, desc_idx, desc_idx_last);
2702
2703                 if (nb_desc_required > txq->nb_free) {
2704                         if (iavf_xmit_cleanup(txq)) {
2705                                 if (idx == 0)
2706                                         return 0;
2707                                 goto end_of_tx;
2708                         }
2709                         if (unlikely(nb_desc_required > txq->rs_thresh)) {
2710                                 while (nb_desc_required > txq->nb_free) {
2711                                         if (iavf_xmit_cleanup(txq)) {
2712                                                 if (idx == 0)
2713                                                         return 0;
2714                                                 goto end_of_tx;
2715                                         }
2716                                 }
2717                         }
2718                 }
2719
2720                 iavf_build_data_desc_cmd_offset_fields(&ddesc_template, mb,
2721                         txq->vlan_flag);
2722
2723                         /* Setup TX context descriptor if required */
2724                 if (nb_desc_ctx) {
2725                         volatile struct iavf_tx_context_desc *ctx_desc =
2726                                 (volatile struct iavf_tx_context_desc *)
2727                                         &txr[desc_idx];
2728
2729                         /* clear QW0 or the previous writeback value
2730                          * may impact next write
2731                          */
2732                         *(volatile uint64_t *)ctx_desc = 0;
2733
2734                         txn = &txe_ring[txe->next_id];
2735                         RTE_MBUF_PREFETCH_TO_FREE(txn->mbuf);
2736
2737                         if (txe->mbuf) {
2738                                 rte_pktmbuf_free_seg(txe->mbuf);
2739                                 txe->mbuf = NULL;
2740                         }
2741
2742                         iavf_fill_context_desc(ctx_desc, mb, ipsec_md, &tlen,
2743                                 txq->vlan_flag);
2744                         IAVF_DUMP_TX_DESC(txq, ctx_desc, desc_idx);
2745
2746                         txe->last_id = desc_idx_last;
2747                         desc_idx = txe->next_id;
2748                         txe = txn;
2749                         }
2750
2751                 if (nb_desc_ipsec) {
2752                         volatile struct iavf_tx_ipsec_desc *ipsec_desc =
2753                                 (volatile struct iavf_tx_ipsec_desc *)
2754                                         &txr[desc_idx];
2755
2756                         txn = &txe_ring[txe->next_id];
2757                         RTE_MBUF_PREFETCH_TO_FREE(txn->mbuf);
2758
2759                         if (txe->mbuf) {
2760                                 rte_pktmbuf_free_seg(txe->mbuf);
2761                                 txe->mbuf = NULL;
2762                 }
2763
2764                         iavf_fill_ipsec_desc(ipsec_desc, ipsec_md, &ipseclen);
2765
2766                         IAVF_DUMP_TX_DESC(txq, ipsec_desc, desc_idx);
2767
2768                         txe->last_id = desc_idx_last;
2769                         desc_idx = txe->next_id;
2770                         txe = txn;
2771                 }
2772
2773                 mb_seg = mb;
2774
2775                 do {
2776                         ddesc = (volatile struct iavf_tx_desc *)
2777                                         &txr[desc_idx];
2778
2779                         txn = &txe_ring[txe->next_id];
2780                         RTE_MBUF_PREFETCH_TO_FREE(txn->mbuf);
2781
2782                         if (txe->mbuf)
2783                                 rte_pktmbuf_free_seg(txe->mbuf);
2784
2785                         txe->mbuf = mb_seg;
2786                         iavf_fill_data_desc(ddesc, mb_seg,
2787                                         ddesc_template, tlen, ipseclen);
2788
2789                         IAVF_DUMP_TX_DESC(txq, ddesc, desc_idx);
2790
2791                         txe->last_id = desc_idx_last;
2792                         desc_idx = txe->next_id;
2793                         txe = txn;
2794                         mb_seg = mb_seg->next;
2795                 } while (mb_seg);
2796
2797                 /* The last packet data descriptor needs End Of Packet (EOP) */
2798                 ddesc_cmd = IAVF_TX_DESC_CMD_EOP;
2799
2800                 txq->nb_used = (uint16_t)(txq->nb_used + nb_desc_required);
2801                 txq->nb_free = (uint16_t)(txq->nb_free - nb_desc_required);
2802
2803                 if (txq->nb_used >= txq->rs_thresh) {
2804                         PMD_TX_LOG(DEBUG, "Setting RS bit on TXD id="
2805                                    "%4u (port=%d queue=%d)",
2806                                    desc_idx_last, txq->port_id, txq->queue_id);
2807
2808                         ddesc_cmd |= IAVF_TX_DESC_CMD_RS;
2809
2810                         /* Update txq RS bit counters */
2811                         txq->nb_used = 0;
2812                 }
2813
2814                 ddesc->cmd_type_offset_bsz |= rte_cpu_to_le_64(ddesc_cmd <<
2815                                 IAVF_TXD_DATA_QW1_CMD_SHIFT);
2816
2817                 IAVF_DUMP_TX_DESC(txq, ddesc, desc_idx - 1);
2818         }
2819
2820 end_of_tx:
2821         rte_wmb();
2822
2823         PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u",
2824                    txq->port_id, txq->queue_id, desc_idx, idx);
2825
2826         IAVF_PCI_REG_WRITE_RELAXED(txq->qtx_tail, desc_idx);
2827         txq->tx_tail = desc_idx;
2828
2829         return idx;
2830 }
2831
2832 /* Check if the packet with vlan user priority is transmitted in the
2833  * correct queue.
2834  */
2835 static int
2836 iavf_check_vlan_up2tc(struct iavf_tx_queue *txq, struct rte_mbuf *m)
2837 {
2838         struct rte_eth_dev *dev = &rte_eth_devices[txq->port_id];
2839         struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2840         uint16_t up;
2841
2842         up = m->vlan_tci >> IAVF_VLAN_TAG_PCP_OFFSET;
2843
2844         if (!(vf->qos_cap->cap[txq->tc].tc_prio & BIT(up))) {
2845                 PMD_TX_LOG(ERR, "packet with vlan pcp %u cannot transmit in queue %u\n",
2846                         up, txq->queue_id);
2847                 return -1;
2848         } else {
2849                 return 0;
2850         }
2851 }
2852
2853 /* TX prep functions */
2854 uint16_t
2855 iavf_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts,
2856               uint16_t nb_pkts)
2857 {
2858         int i, ret;
2859         uint64_t ol_flags;
2860         struct rte_mbuf *m;
2861         struct iavf_tx_queue *txq = tx_queue;
2862         struct rte_eth_dev *dev = &rte_eth_devices[txq->port_id];
2863         struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2864         struct iavf_adapter *adapter = IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
2865
2866         if (adapter->closed)
2867                 return 0;
2868
2869         for (i = 0; i < nb_pkts; i++) {
2870                 m = tx_pkts[i];
2871                 ol_flags = m->ol_flags;
2872
2873                 /* Check condition for nb_segs > IAVF_TX_MAX_MTU_SEG. */
2874                 if (!(ol_flags & RTE_MBUF_F_TX_TCP_SEG)) {
2875                         if (m->nb_segs > IAVF_TX_MAX_MTU_SEG) {
2876                                 rte_errno = EINVAL;
2877                                 return i;
2878                         }
2879                 } else if ((m->tso_segsz < IAVF_MIN_TSO_MSS) ||
2880                            (m->tso_segsz > IAVF_MAX_TSO_MSS)) {
2881                         /* MSS outside the range are considered malicious */
2882                         rte_errno = EINVAL;
2883                         return i;
2884                 }
2885
2886                 if (ol_flags & IAVF_TX_OFFLOAD_NOTSUP_MASK) {
2887                         rte_errno = ENOTSUP;
2888                         return i;
2889                 }
2890
2891 #ifdef RTE_ETHDEV_DEBUG_TX
2892                 ret = rte_validate_tx_offload(m);
2893                 if (ret != 0) {
2894                         rte_errno = -ret;
2895                         return i;
2896                 }
2897 #endif
2898                 ret = rte_net_intel_cksum_prepare(m);
2899                 if (ret != 0) {
2900                         rte_errno = -ret;
2901                         return i;
2902                 }
2903
2904                 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_QOS &&
2905                     ol_flags & (RTE_MBUF_F_RX_VLAN_STRIPPED | RTE_MBUF_F_RX_VLAN)) {
2906                         ret = iavf_check_vlan_up2tc(txq, m);
2907                         if (ret != 0) {
2908                                 rte_errno = -ret;
2909                                 return i;
2910                         }
2911                 }
2912         }
2913
2914         return i;
2915 }
2916
2917 /* choose rx function*/
2918 void
2919 iavf_set_rx_function(struct rte_eth_dev *dev)
2920 {
2921         struct iavf_adapter *adapter =
2922                 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
2923         struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2924         int i;
2925         struct iavf_rx_queue *rxq;
2926         bool use_flex = true;
2927
2928         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2929                 rxq = dev->data->rx_queues[i];
2930                 if (rxq->rxdid <= IAVF_RXDID_LEGACY_1) {
2931                         PMD_DRV_LOG(NOTICE, "request RXDID[%d] in Queue[%d] is legacy, "
2932                                 "set rx_pkt_burst as legacy for all queues", rxq->rxdid, i);
2933                         use_flex = false;
2934                 } else if (!(vf->supported_rxdid & BIT(rxq->rxdid))) {
2935                         PMD_DRV_LOG(NOTICE, "request RXDID[%d] in Queue[%d] is not supported, "
2936                                 "set rx_pkt_burst as legacy for all queues", rxq->rxdid, i);
2937                         use_flex = false;
2938                 }
2939         }
2940
2941 #ifdef RTE_ARCH_X86
2942         int check_ret;
2943         bool use_avx2 = false;
2944         bool use_avx512 = false;
2945
2946         check_ret = iavf_rx_vec_dev_check(dev);
2947         if (check_ret >= 0 &&
2948             rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) {
2949                 if ((rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) == 1 ||
2950                      rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1) &&
2951                     rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_256)
2952                         use_avx2 = true;
2953
2954 #ifdef CC_AVX512_SUPPORT
2955                 if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1 &&
2956                     rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) == 1 &&
2957                     rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_512)
2958                         use_avx512 = true;
2959 #endif
2960
2961                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
2962                         rxq = dev->data->rx_queues[i];
2963                         (void)iavf_rxq_vec_setup(rxq);
2964                 }
2965
2966                 if (dev->data->scattered_rx) {
2967                         if (!use_avx512) {
2968                                 PMD_DRV_LOG(DEBUG,
2969                                             "Using %sVector Scattered Rx (port %d).",
2970                                             use_avx2 ? "avx2 " : "",
2971                                             dev->data->port_id);
2972                         } else {
2973                                 if (check_ret == IAVF_VECTOR_PATH)
2974                                         PMD_DRV_LOG(DEBUG,
2975                                                     "Using AVX512 Vector Scattered Rx (port %d).",
2976                                                     dev->data->port_id);
2977                                 else
2978                                         PMD_DRV_LOG(DEBUG,
2979                                                     "Using AVX512 OFFLOAD Vector Scattered Rx (port %d).",
2980                                                     dev->data->port_id);
2981                         }
2982                         if (use_flex) {
2983                                 dev->rx_pkt_burst = use_avx2 ?
2984                                         iavf_recv_scattered_pkts_vec_avx2_flex_rxd :
2985                                         iavf_recv_scattered_pkts_vec_flex_rxd;
2986 #ifdef CC_AVX512_SUPPORT
2987                                 if (use_avx512) {
2988                                         if (check_ret == IAVF_VECTOR_PATH)
2989                                                 dev->rx_pkt_burst =
2990                                                         iavf_recv_scattered_pkts_vec_avx512_flex_rxd;
2991                                         else
2992                                                 dev->rx_pkt_burst =
2993                                                         iavf_recv_scattered_pkts_vec_avx512_flex_rxd_offload;
2994                                 }
2995 #endif
2996                         } else {
2997                                 dev->rx_pkt_burst = use_avx2 ?
2998                                         iavf_recv_scattered_pkts_vec_avx2 :
2999                                         iavf_recv_scattered_pkts_vec;
3000 #ifdef CC_AVX512_SUPPORT
3001                                 if (use_avx512) {
3002                                         if (check_ret == IAVF_VECTOR_PATH)
3003                                                 dev->rx_pkt_burst =
3004                                                         iavf_recv_scattered_pkts_vec_avx512;
3005                                         else
3006                                                 dev->rx_pkt_burst =
3007                                                         iavf_recv_scattered_pkts_vec_avx512_offload;
3008                                 }
3009 #endif
3010                         }
3011                 } else {
3012                         if (!use_avx512) {
3013                                 PMD_DRV_LOG(DEBUG, "Using %sVector Rx (port %d).",
3014                                             use_avx2 ? "avx2 " : "",
3015                                             dev->data->port_id);
3016                         } else {
3017                                 if (check_ret == IAVF_VECTOR_PATH)
3018                                         PMD_DRV_LOG(DEBUG,
3019                                                     "Using AVX512 Vector Rx (port %d).",
3020                                                     dev->data->port_id);
3021                                 else
3022                                         PMD_DRV_LOG(DEBUG,
3023                                                     "Using AVX512 OFFLOAD Vector Rx (port %d).",
3024                                                     dev->data->port_id);
3025                         }
3026                         if (use_flex) {
3027                                 dev->rx_pkt_burst = use_avx2 ?
3028                                         iavf_recv_pkts_vec_avx2_flex_rxd :
3029                                         iavf_recv_pkts_vec_flex_rxd;
3030 #ifdef CC_AVX512_SUPPORT
3031                                 if (use_avx512) {
3032                                         if (check_ret == IAVF_VECTOR_PATH)
3033                                                 dev->rx_pkt_burst =
3034                                                         iavf_recv_pkts_vec_avx512_flex_rxd;
3035                                         else
3036                                                 dev->rx_pkt_burst =
3037                                                         iavf_recv_pkts_vec_avx512_flex_rxd_offload;
3038                                 }
3039 #endif
3040                         } else {
3041                                 dev->rx_pkt_burst = use_avx2 ?
3042                                         iavf_recv_pkts_vec_avx2 :
3043                                         iavf_recv_pkts_vec;
3044 #ifdef CC_AVX512_SUPPORT
3045                                 if (use_avx512) {
3046                                         if (check_ret == IAVF_VECTOR_PATH)
3047                                                 dev->rx_pkt_burst =
3048                                                         iavf_recv_pkts_vec_avx512;
3049                                         else
3050                                                 dev->rx_pkt_burst =
3051                                                         iavf_recv_pkts_vec_avx512_offload;
3052                                 }
3053 #endif
3054                         }
3055                 }
3056
3057                 return;
3058         }
3059
3060 #endif
3061         if (dev->data->scattered_rx) {
3062                 PMD_DRV_LOG(DEBUG, "Using a Scattered Rx callback (port=%d).",
3063                             dev->data->port_id);
3064                 if (use_flex)
3065                         dev->rx_pkt_burst = iavf_recv_scattered_pkts_flex_rxd;
3066                 else
3067                         dev->rx_pkt_burst = iavf_recv_scattered_pkts;
3068         } else if (adapter->rx_bulk_alloc_allowed) {
3069                 PMD_DRV_LOG(DEBUG, "Using bulk Rx callback (port=%d).",
3070                             dev->data->port_id);
3071                 dev->rx_pkt_burst = iavf_recv_pkts_bulk_alloc;
3072         } else {
3073                 PMD_DRV_LOG(DEBUG, "Using Basic Rx callback (port=%d).",
3074                             dev->data->port_id);
3075                 if (use_flex)
3076                         dev->rx_pkt_burst = iavf_recv_pkts_flex_rxd;
3077                 else
3078                         dev->rx_pkt_burst = iavf_recv_pkts;
3079         }
3080 }
3081
3082 /* choose tx function*/
3083 void
3084 iavf_set_tx_function(struct rte_eth_dev *dev)
3085 {
3086 #ifdef RTE_ARCH_X86
3087         struct iavf_tx_queue *txq;
3088         int i;
3089         int check_ret;
3090         bool use_sse = false;
3091         bool use_avx2 = false;
3092         bool use_avx512 = false;
3093
3094         check_ret = iavf_tx_vec_dev_check(dev);
3095
3096         if (check_ret >= 0 &&
3097             rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) {
3098                 /* SSE and AVX2 not support offload path yet. */
3099                 if (check_ret == IAVF_VECTOR_PATH) {
3100                         use_sse = true;
3101                         if ((rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) == 1 ||
3102                              rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1) &&
3103                             rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_256)
3104                                 use_avx2 = true;
3105                 }
3106 #ifdef CC_AVX512_SUPPORT
3107                 if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1 &&
3108                     rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) == 1 &&
3109                     rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_512)
3110                         use_avx512 = true;
3111 #endif
3112
3113                 if (!use_sse && !use_avx2 && !use_avx512)
3114                         goto normal;
3115
3116                 if (!use_avx512) {
3117                         PMD_DRV_LOG(DEBUG, "Using %sVector Tx (port %d).",
3118                                     use_avx2 ? "avx2 " : "",
3119                                     dev->data->port_id);
3120                         dev->tx_pkt_burst = use_avx2 ?
3121                                             iavf_xmit_pkts_vec_avx2 :
3122                                             iavf_xmit_pkts_vec;
3123                 }
3124                 dev->tx_pkt_prepare = NULL;
3125 #ifdef CC_AVX512_SUPPORT
3126                 if (use_avx512) {
3127                         if (check_ret == IAVF_VECTOR_PATH) {
3128                                 dev->tx_pkt_burst = iavf_xmit_pkts_vec_avx512;
3129                                 PMD_DRV_LOG(DEBUG, "Using AVX512 Vector Tx (port %d).",
3130                                             dev->data->port_id);
3131                         } else {
3132                                 dev->tx_pkt_burst = iavf_xmit_pkts_vec_avx512_offload;
3133                                 dev->tx_pkt_prepare = iavf_prep_pkts;
3134                                 PMD_DRV_LOG(DEBUG, "Using AVX512 OFFLOAD Vector Tx (port %d).",
3135                                             dev->data->port_id);
3136                         }
3137                 }
3138 #endif
3139
3140                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
3141                         txq = dev->data->tx_queues[i];
3142                         if (!txq)
3143                                 continue;
3144 #ifdef CC_AVX512_SUPPORT
3145                         if (use_avx512)
3146                                 iavf_txq_vec_setup_avx512(txq);
3147                         else
3148                                 iavf_txq_vec_setup(txq);
3149 #else
3150                         iavf_txq_vec_setup(txq);
3151 #endif
3152                 }
3153
3154                 return;
3155         }
3156
3157 normal:
3158 #endif
3159         PMD_DRV_LOG(DEBUG, "Using Basic Tx callback (port=%d).",
3160                     dev->data->port_id);
3161         dev->tx_pkt_burst = iavf_xmit_pkts;
3162         dev->tx_pkt_prepare = iavf_prep_pkts;
3163 }
3164
3165 static int
3166 iavf_tx_done_cleanup_full(struct iavf_tx_queue *txq,
3167                         uint32_t free_cnt)
3168 {
3169         struct iavf_tx_entry *swr_ring = txq->sw_ring;
3170         uint16_t i, tx_last, tx_id;
3171         uint16_t nb_tx_free_last;
3172         uint16_t nb_tx_to_clean;
3173         uint32_t pkt_cnt;
3174
3175         /* Start free mbuf from the next of tx_tail */
3176         tx_last = txq->tx_tail;
3177         tx_id  = swr_ring[tx_last].next_id;
3178
3179         if (txq->nb_free == 0 && iavf_xmit_cleanup(txq))
3180                 return 0;
3181
3182         nb_tx_to_clean = txq->nb_free;
3183         nb_tx_free_last = txq->nb_free;
3184         if (!free_cnt)
3185                 free_cnt = txq->nb_tx_desc;
3186
3187         /* Loop through swr_ring to count the amount of
3188          * freeable mubfs and packets.
3189          */
3190         for (pkt_cnt = 0; pkt_cnt < free_cnt; ) {
3191                 for (i = 0; i < nb_tx_to_clean &&
3192                         pkt_cnt < free_cnt &&
3193                         tx_id != tx_last; i++) {
3194                         if (swr_ring[tx_id].mbuf != NULL) {
3195                                 rte_pktmbuf_free_seg(swr_ring[tx_id].mbuf);
3196                                 swr_ring[tx_id].mbuf = NULL;
3197
3198                                 /*
3199                                  * last segment in the packet,
3200                                  * increment packet count
3201                                  */
3202                                 pkt_cnt += (swr_ring[tx_id].last_id == tx_id);
3203                         }
3204
3205                         tx_id = swr_ring[tx_id].next_id;
3206                 }
3207
3208                 if (txq->rs_thresh > txq->nb_tx_desc -
3209                         txq->nb_free || tx_id == tx_last)
3210                         break;
3211
3212                 if (pkt_cnt < free_cnt) {
3213                         if (iavf_xmit_cleanup(txq))
3214                                 break;
3215
3216                         nb_tx_to_clean = txq->nb_free - nb_tx_free_last;
3217                         nb_tx_free_last = txq->nb_free;
3218                 }
3219         }
3220
3221         return (int)pkt_cnt;
3222 }
3223
3224 int
3225 iavf_dev_tx_done_cleanup(void *txq, uint32_t free_cnt)
3226 {
3227         struct iavf_tx_queue *q = (struct iavf_tx_queue *)txq;
3228
3229         return iavf_tx_done_cleanup_full(q, free_cnt);
3230 }
3231
3232 void
3233 iavf_dev_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
3234                      struct rte_eth_rxq_info *qinfo)
3235 {
3236         struct iavf_rx_queue *rxq;
3237
3238         rxq = dev->data->rx_queues[queue_id];
3239
3240         qinfo->mp = rxq->mp;
3241         qinfo->scattered_rx = dev->data->scattered_rx;
3242         qinfo->nb_desc = rxq->nb_rx_desc;
3243
3244         qinfo->conf.rx_free_thresh = rxq->rx_free_thresh;
3245         qinfo->conf.rx_drop_en = true;
3246         qinfo->conf.rx_deferred_start = rxq->rx_deferred_start;
3247 }
3248
3249 void
3250 iavf_dev_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
3251                      struct rte_eth_txq_info *qinfo)
3252 {
3253         struct iavf_tx_queue *txq;
3254
3255         txq = dev->data->tx_queues[queue_id];
3256
3257         qinfo->nb_desc = txq->nb_tx_desc;
3258
3259         qinfo->conf.tx_free_thresh = txq->free_thresh;
3260         qinfo->conf.tx_rs_thresh = txq->rs_thresh;
3261         qinfo->conf.offloads = txq->offloads;
3262         qinfo->conf.tx_deferred_start = txq->tx_deferred_start;
3263 }
3264
3265 /* Get the number of used descriptors of a rx queue */
3266 uint32_t
3267 iavf_dev_rxq_count(void *rx_queue)
3268 {
3269 #define IAVF_RXQ_SCAN_INTERVAL 4
3270         volatile union iavf_rx_desc *rxdp;
3271         struct iavf_rx_queue *rxq;
3272         uint16_t desc = 0;
3273
3274         rxq = rx_queue;
3275         rxdp = &rxq->rx_ring[rxq->rx_tail];
3276
3277         while ((desc < rxq->nb_rx_desc) &&
3278                ((rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len) &
3279                  IAVF_RXD_QW1_STATUS_MASK) >> IAVF_RXD_QW1_STATUS_SHIFT) &
3280                (1 << IAVF_RX_DESC_STATUS_DD_SHIFT)) {
3281                 /* Check the DD bit of a rx descriptor of each 4 in a group,
3282                  * to avoid checking too frequently and downgrading performance
3283                  * too much.
3284                  */
3285                 desc += IAVF_RXQ_SCAN_INTERVAL;
3286                 rxdp += IAVF_RXQ_SCAN_INTERVAL;
3287                 if (rxq->rx_tail + desc >= rxq->nb_rx_desc)
3288                         rxdp = &(rxq->rx_ring[rxq->rx_tail +
3289                                         desc - rxq->nb_rx_desc]);
3290         }
3291
3292         return desc;
3293 }
3294
3295 int
3296 iavf_dev_rx_desc_status(void *rx_queue, uint16_t offset)
3297 {
3298         struct iavf_rx_queue *rxq = rx_queue;
3299         volatile uint64_t *status;
3300         uint64_t mask;
3301         uint32_t desc;
3302
3303         if (unlikely(offset >= rxq->nb_rx_desc))
3304                 return -EINVAL;
3305
3306         if (offset >= rxq->nb_rx_desc - rxq->nb_rx_hold)
3307                 return RTE_ETH_RX_DESC_UNAVAIL;
3308
3309         desc = rxq->rx_tail + offset;
3310         if (desc >= rxq->nb_rx_desc)
3311                 desc -= rxq->nb_rx_desc;
3312
3313         status = &rxq->rx_ring[desc].wb.qword1.status_error_len;
3314         mask = rte_le_to_cpu_64((1ULL << IAVF_RX_DESC_STATUS_DD_SHIFT)
3315                 << IAVF_RXD_QW1_STATUS_SHIFT);
3316         if (*status & mask)
3317                 return RTE_ETH_RX_DESC_DONE;
3318
3319         return RTE_ETH_RX_DESC_AVAIL;
3320 }
3321
3322 int
3323 iavf_dev_tx_desc_status(void *tx_queue, uint16_t offset)
3324 {
3325         struct iavf_tx_queue *txq = tx_queue;
3326         volatile uint64_t *status;
3327         uint64_t mask, expect;
3328         uint32_t desc;
3329
3330         if (unlikely(offset >= txq->nb_tx_desc))
3331                 return -EINVAL;
3332
3333         desc = txq->tx_tail + offset;
3334         /* go to next desc that has the RS bit */
3335         desc = ((desc + txq->rs_thresh - 1) / txq->rs_thresh) *
3336                 txq->rs_thresh;
3337         if (desc >= txq->nb_tx_desc) {
3338                 desc -= txq->nb_tx_desc;
3339                 if (desc >= txq->nb_tx_desc)
3340                         desc -= txq->nb_tx_desc;
3341         }
3342
3343         status = &txq->tx_ring[desc].cmd_type_offset_bsz;
3344         mask = rte_le_to_cpu_64(IAVF_TXD_QW1_DTYPE_MASK);
3345         expect = rte_cpu_to_le_64(
3346                  IAVF_TX_DESC_DTYPE_DESC_DONE << IAVF_TXD_QW1_DTYPE_SHIFT);
3347         if ((*status & mask) == expect)
3348                 return RTE_ETH_TX_DESC_DONE;
3349
3350         return RTE_ETH_TX_DESC_FULL;
3351 }
3352
3353 static inline uint32_t
3354 iavf_get_default_ptype(uint16_t ptype)
3355 {
3356         static const uint32_t ptype_tbl[IAVF_MAX_PKT_TYPE]
3357                 __rte_cache_aligned = {
3358                 /* L2 types */
3359                 /* [0] reserved */
3360                 [1] = RTE_PTYPE_L2_ETHER,
3361                 [2] = RTE_PTYPE_L2_ETHER_TIMESYNC,
3362                 /* [3] - [5] reserved */
3363                 [6] = RTE_PTYPE_L2_ETHER_LLDP,
3364                 /* [7] - [10] reserved */
3365                 [11] = RTE_PTYPE_L2_ETHER_ARP,
3366                 /* [12] - [21] reserved */
3367
3368                 /* Non tunneled IPv4 */
3369                 [22] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3370                        RTE_PTYPE_L4_FRAG,
3371                 [23] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3372                        RTE_PTYPE_L4_NONFRAG,
3373                 [24] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3374                        RTE_PTYPE_L4_UDP,
3375                 /* [25] reserved */
3376                 [26] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3377                        RTE_PTYPE_L4_TCP,
3378                 [27] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3379                        RTE_PTYPE_L4_SCTP,
3380                 [28] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3381                        RTE_PTYPE_L4_ICMP,
3382
3383                 /* IPv4 --> IPv4 */
3384                 [29] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3385                        RTE_PTYPE_TUNNEL_IP |
3386                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3387                        RTE_PTYPE_INNER_L4_FRAG,
3388                 [30] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3389                        RTE_PTYPE_TUNNEL_IP |
3390                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3391                        RTE_PTYPE_INNER_L4_NONFRAG,
3392                 [31] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3393                        RTE_PTYPE_TUNNEL_IP |
3394                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3395                        RTE_PTYPE_INNER_L4_UDP,
3396                 /* [32] reserved */
3397                 [33] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3398                        RTE_PTYPE_TUNNEL_IP |
3399                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3400                        RTE_PTYPE_INNER_L4_TCP,
3401                 [34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3402                        RTE_PTYPE_TUNNEL_IP |
3403                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3404                        RTE_PTYPE_INNER_L4_SCTP,
3405                 [35] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3406                        RTE_PTYPE_TUNNEL_IP |
3407                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3408                        RTE_PTYPE_INNER_L4_ICMP,
3409
3410                 /* IPv4 --> IPv6 */
3411                 [36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3412                        RTE_PTYPE_TUNNEL_IP |
3413                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3414                        RTE_PTYPE_INNER_L4_FRAG,
3415                 [37] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3416                        RTE_PTYPE_TUNNEL_IP |
3417                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3418                        RTE_PTYPE_INNER_L4_NONFRAG,
3419                 [38] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3420                        RTE_PTYPE_TUNNEL_IP |
3421                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3422                        RTE_PTYPE_INNER_L4_UDP,
3423                 /* [39] reserved */
3424                 [40] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3425                        RTE_PTYPE_TUNNEL_IP |
3426                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3427                        RTE_PTYPE_INNER_L4_TCP,
3428                 [41] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3429                        RTE_PTYPE_TUNNEL_IP |
3430                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3431                        RTE_PTYPE_INNER_L4_SCTP,
3432                 [42] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3433                        RTE_PTYPE_TUNNEL_IP |
3434                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3435                        RTE_PTYPE_INNER_L4_ICMP,
3436
3437                 /* IPv4 --> GRE/Teredo/VXLAN */
3438                 [43] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3439                        RTE_PTYPE_TUNNEL_GRENAT,
3440
3441                 /* IPv4 --> GRE/Teredo/VXLAN --> IPv4 */
3442                 [44] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3443                        RTE_PTYPE_TUNNEL_GRENAT |
3444                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3445                        RTE_PTYPE_INNER_L4_FRAG,
3446                 [45] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3447                        RTE_PTYPE_TUNNEL_GRENAT |
3448                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3449                        RTE_PTYPE_INNER_L4_NONFRAG,
3450                 [46] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3451                        RTE_PTYPE_TUNNEL_GRENAT |
3452                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3453                        RTE_PTYPE_INNER_L4_UDP,
3454                 /* [47] reserved */
3455                 [48] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3456                        RTE_PTYPE_TUNNEL_GRENAT |
3457                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3458                        RTE_PTYPE_INNER_L4_TCP,
3459                 [49] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3460                        RTE_PTYPE_TUNNEL_GRENAT |
3461                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3462                        RTE_PTYPE_INNER_L4_SCTP,
3463                 [50] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3464                        RTE_PTYPE_TUNNEL_GRENAT |
3465                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3466                        RTE_PTYPE_INNER_L4_ICMP,
3467
3468                 /* IPv4 --> GRE/Teredo/VXLAN --> IPv6 */
3469                 [51] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3470                        RTE_PTYPE_TUNNEL_GRENAT |
3471                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3472                        RTE_PTYPE_INNER_L4_FRAG,
3473                 [52] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3474                        RTE_PTYPE_TUNNEL_GRENAT |
3475                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3476                        RTE_PTYPE_INNER_L4_NONFRAG,
3477                 [53] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3478                        RTE_PTYPE_TUNNEL_GRENAT |
3479                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3480                        RTE_PTYPE_INNER_L4_UDP,
3481                 /* [54] reserved */
3482                 [55] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3483                        RTE_PTYPE_TUNNEL_GRENAT |
3484                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3485                        RTE_PTYPE_INNER_L4_TCP,
3486                 [56] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3487                        RTE_PTYPE_TUNNEL_GRENAT |
3488                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3489                        RTE_PTYPE_INNER_L4_SCTP,
3490                 [57] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3491                        RTE_PTYPE_TUNNEL_GRENAT |
3492                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3493                        RTE_PTYPE_INNER_L4_ICMP,
3494
3495                 /* IPv4 --> GRE/Teredo/VXLAN --> MAC */
3496                 [58] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3497                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER,
3498
3499                 /* IPv4 --> GRE/Teredo/VXLAN --> MAC --> IPv4 */
3500                 [59] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3501                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3502                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3503                        RTE_PTYPE_INNER_L4_FRAG,
3504                 [60] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3505                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3506                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3507                        RTE_PTYPE_INNER_L4_NONFRAG,
3508                 [61] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3509                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3510                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3511                        RTE_PTYPE_INNER_L4_UDP,
3512                 /* [62] reserved */
3513                 [63] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3514                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3515                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3516                        RTE_PTYPE_INNER_L4_TCP,
3517                 [64] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3518                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3519                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3520                        RTE_PTYPE_INNER_L4_SCTP,
3521                 [65] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3522                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3523                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3524                        RTE_PTYPE_INNER_L4_ICMP,
3525
3526                 /* IPv4 --> GRE/Teredo/VXLAN --> MAC --> IPv6 */
3527                 [66] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3528                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3529                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3530                        RTE_PTYPE_INNER_L4_FRAG,
3531                 [67] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3532                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3533                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3534                        RTE_PTYPE_INNER_L4_NONFRAG,
3535                 [68] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3536                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3537                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3538                        RTE_PTYPE_INNER_L4_UDP,
3539                 /* [69] reserved */
3540                 [70] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3541                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3542                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3543                        RTE_PTYPE_INNER_L4_TCP,
3544                 [71] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3545                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3546                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3547                        RTE_PTYPE_INNER_L4_SCTP,
3548                 [72] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3549                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3550                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3551                        RTE_PTYPE_INNER_L4_ICMP,
3552                 /* [73] - [87] reserved */
3553
3554                 /* Non tunneled IPv6 */
3555                 [88] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3556                        RTE_PTYPE_L4_FRAG,
3557                 [89] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3558                        RTE_PTYPE_L4_NONFRAG,
3559                 [90] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3560                        RTE_PTYPE_L4_UDP,
3561                 /* [91] reserved */
3562                 [92] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3563                        RTE_PTYPE_L4_TCP,
3564                 [93] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3565                        RTE_PTYPE_L4_SCTP,
3566                 [94] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3567                        RTE_PTYPE_L4_ICMP,
3568
3569                 /* IPv6 --> IPv4 */
3570                 [95] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3571                        RTE_PTYPE_TUNNEL_IP |
3572                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3573                        RTE_PTYPE_INNER_L4_FRAG,
3574                 [96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3575                        RTE_PTYPE_TUNNEL_IP |
3576                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3577                        RTE_PTYPE_INNER_L4_NONFRAG,
3578                 [97] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3579                        RTE_PTYPE_TUNNEL_IP |
3580                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3581                        RTE_PTYPE_INNER_L4_UDP,
3582                 /* [98] reserved */
3583                 [99] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3584                        RTE_PTYPE_TUNNEL_IP |
3585                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3586                        RTE_PTYPE_INNER_L4_TCP,
3587                 [100] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3588                         RTE_PTYPE_TUNNEL_IP |
3589                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3590                         RTE_PTYPE_INNER_L4_SCTP,
3591                 [101] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3592                         RTE_PTYPE_TUNNEL_IP |
3593                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3594                         RTE_PTYPE_INNER_L4_ICMP,
3595
3596                 /* IPv6 --> IPv6 */
3597                 [102] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3598                         RTE_PTYPE_TUNNEL_IP |
3599                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3600                         RTE_PTYPE_INNER_L4_FRAG,
3601                 [103] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3602                         RTE_PTYPE_TUNNEL_IP |
3603                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3604                         RTE_PTYPE_INNER_L4_NONFRAG,
3605                 [104] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3606                         RTE_PTYPE_TUNNEL_IP |
3607                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3608                         RTE_PTYPE_INNER_L4_UDP,
3609                 /* [105] reserved */
3610                 [106] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3611                         RTE_PTYPE_TUNNEL_IP |
3612                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3613                         RTE_PTYPE_INNER_L4_TCP,
3614                 [107] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3615                         RTE_PTYPE_TUNNEL_IP |
3616                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3617                         RTE_PTYPE_INNER_L4_SCTP,
3618                 [108] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3619                         RTE_PTYPE_TUNNEL_IP |
3620                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3621                         RTE_PTYPE_INNER_L4_ICMP,
3622
3623                 /* IPv6 --> GRE/Teredo/VXLAN */
3624                 [109] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3625                         RTE_PTYPE_TUNNEL_GRENAT,
3626
3627                 /* IPv6 --> GRE/Teredo/VXLAN --> IPv4 */
3628                 [110] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3629                         RTE_PTYPE_TUNNEL_GRENAT |
3630                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3631                         RTE_PTYPE_INNER_L4_FRAG,
3632                 [111] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3633                         RTE_PTYPE_TUNNEL_GRENAT |
3634                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3635                         RTE_PTYPE_INNER_L4_NONFRAG,
3636                 [112] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3637                         RTE_PTYPE_TUNNEL_GRENAT |
3638                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3639                         RTE_PTYPE_INNER_L4_UDP,
3640                 /* [113] reserved */
3641                 [114] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3642                         RTE_PTYPE_TUNNEL_GRENAT |
3643                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3644                         RTE_PTYPE_INNER_L4_TCP,
3645                 [115] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3646                         RTE_PTYPE_TUNNEL_GRENAT |
3647                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3648                         RTE_PTYPE_INNER_L4_SCTP,
3649                 [116] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3650                         RTE_PTYPE_TUNNEL_GRENAT |
3651                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3652                         RTE_PTYPE_INNER_L4_ICMP,
3653
3654                 /* IPv6 --> GRE/Teredo/VXLAN --> IPv6 */
3655                 [117] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3656                         RTE_PTYPE_TUNNEL_GRENAT |
3657                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3658                         RTE_PTYPE_INNER_L4_FRAG,
3659                 [118] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3660                         RTE_PTYPE_TUNNEL_GRENAT |
3661                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3662                         RTE_PTYPE_INNER_L4_NONFRAG,
3663                 [119] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3664                         RTE_PTYPE_TUNNEL_GRENAT |
3665                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3666                         RTE_PTYPE_INNER_L4_UDP,
3667                 /* [120] reserved */
3668                 [121] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3669                         RTE_PTYPE_TUNNEL_GRENAT |
3670                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3671                         RTE_PTYPE_INNER_L4_TCP,
3672                 [122] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3673                         RTE_PTYPE_TUNNEL_GRENAT |
3674                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3675                         RTE_PTYPE_INNER_L4_SCTP,
3676                 [123] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3677                         RTE_PTYPE_TUNNEL_GRENAT |
3678                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3679                         RTE_PTYPE_INNER_L4_ICMP,
3680
3681                 /* IPv6 --> GRE/Teredo/VXLAN --> MAC */
3682                 [124] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3683                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER,
3684
3685                 /* IPv6 --> GRE/Teredo/VXLAN --> MAC --> IPv4 */
3686                 [125] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3687                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3688                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3689                         RTE_PTYPE_INNER_L4_FRAG,
3690                 [126] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3691                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3692                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3693                         RTE_PTYPE_INNER_L4_NONFRAG,
3694                 [127] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3695                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3696                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3697                         RTE_PTYPE_INNER_L4_UDP,
3698                 /* [128] reserved */
3699                 [129] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3700                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3701                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3702                         RTE_PTYPE_INNER_L4_TCP,
3703                 [130] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3704                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3705                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3706                         RTE_PTYPE_INNER_L4_SCTP,
3707                 [131] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3708                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3709                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3710                         RTE_PTYPE_INNER_L4_ICMP,
3711
3712                 /* IPv6 --> GRE/Teredo/VXLAN --> MAC --> IPv6 */
3713                 [132] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3714                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3715                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3716                         RTE_PTYPE_INNER_L4_FRAG,
3717                 [133] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3718                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3719                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3720                         RTE_PTYPE_INNER_L4_NONFRAG,
3721                 [134] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3722                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3723                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3724                         RTE_PTYPE_INNER_L4_UDP,
3725                 /* [135] reserved */
3726                 [136] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3727                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3728                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3729                         RTE_PTYPE_INNER_L4_TCP,
3730                 [137] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3731                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3732                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3733                         RTE_PTYPE_INNER_L4_SCTP,
3734                 [138] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3735                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3736                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3737                         RTE_PTYPE_INNER_L4_ICMP,
3738                 /* [139] - [299] reserved */
3739
3740                 /* PPPoE */
3741                 [300] = RTE_PTYPE_L2_ETHER_PPPOE,
3742                 [301] = RTE_PTYPE_L2_ETHER_PPPOE,
3743
3744                 /* PPPoE --> IPv4 */
3745                 [302] = RTE_PTYPE_L2_ETHER_PPPOE |
3746                         RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3747                         RTE_PTYPE_L4_FRAG,
3748                 [303] = RTE_PTYPE_L2_ETHER_PPPOE |
3749                         RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3750                         RTE_PTYPE_L4_NONFRAG,
3751                 [304] = RTE_PTYPE_L2_ETHER_PPPOE |
3752                         RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3753                         RTE_PTYPE_L4_UDP,
3754                 [305] = RTE_PTYPE_L2_ETHER_PPPOE |
3755                         RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3756                         RTE_PTYPE_L4_TCP,
3757                 [306] = RTE_PTYPE_L2_ETHER_PPPOE |
3758                         RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3759                         RTE_PTYPE_L4_SCTP,
3760                 [307] = RTE_PTYPE_L2_ETHER_PPPOE |
3761                         RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3762                         RTE_PTYPE_L4_ICMP,
3763
3764                 /* PPPoE --> IPv6 */
3765                 [308] = RTE_PTYPE_L2_ETHER_PPPOE |
3766                         RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3767                         RTE_PTYPE_L4_FRAG,
3768                 [309] = RTE_PTYPE_L2_ETHER_PPPOE |
3769                         RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3770                         RTE_PTYPE_L4_NONFRAG,
3771                 [310] = RTE_PTYPE_L2_ETHER_PPPOE |
3772                         RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3773                         RTE_PTYPE_L4_UDP,
3774                 [311] = RTE_PTYPE_L2_ETHER_PPPOE |
3775                         RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3776                         RTE_PTYPE_L4_TCP,
3777                 [312] = RTE_PTYPE_L2_ETHER_PPPOE |
3778                         RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3779                         RTE_PTYPE_L4_SCTP,
3780                 [313] = RTE_PTYPE_L2_ETHER_PPPOE |
3781                         RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3782                         RTE_PTYPE_L4_ICMP,
3783                 /* [314] - [324] reserved */
3784
3785                 /* IPv4/IPv6 --> GTPC/GTPU */
3786                 [325] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3787                         RTE_PTYPE_TUNNEL_GTPC,
3788                 [326] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3789                         RTE_PTYPE_TUNNEL_GTPC,
3790                 [327] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3791                         RTE_PTYPE_TUNNEL_GTPC,
3792                 [328] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3793                         RTE_PTYPE_TUNNEL_GTPC,
3794                 [329] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3795                         RTE_PTYPE_TUNNEL_GTPU,
3796                 [330] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3797                         RTE_PTYPE_TUNNEL_GTPU,
3798
3799                 /* IPv4 --> GTPU --> IPv4 */
3800                 [331] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3801                         RTE_PTYPE_TUNNEL_GTPU |
3802                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3803                         RTE_PTYPE_INNER_L4_FRAG,
3804                 [332] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3805                         RTE_PTYPE_TUNNEL_GTPU |
3806                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3807                         RTE_PTYPE_INNER_L4_NONFRAG,
3808                 [333] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3809                         RTE_PTYPE_TUNNEL_GTPU |
3810                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3811                         RTE_PTYPE_INNER_L4_UDP,
3812                 [334] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3813                         RTE_PTYPE_TUNNEL_GTPU |
3814                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3815                         RTE_PTYPE_INNER_L4_TCP,
3816                 [335] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3817                         RTE_PTYPE_TUNNEL_GTPU |
3818                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3819                         RTE_PTYPE_INNER_L4_ICMP,
3820
3821                 /* IPv6 --> GTPU --> IPv4 */
3822                 [336] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3823                         RTE_PTYPE_TUNNEL_GTPU |
3824                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3825                         RTE_PTYPE_INNER_L4_FRAG,
3826                 [337] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3827                         RTE_PTYPE_TUNNEL_GTPU |
3828                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3829                         RTE_PTYPE_INNER_L4_NONFRAG,
3830                 [338] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3831                         RTE_PTYPE_TUNNEL_GTPU |
3832                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3833                         RTE_PTYPE_INNER_L4_UDP,
3834                 [339] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3835                         RTE_PTYPE_TUNNEL_GTPU |
3836                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3837                         RTE_PTYPE_INNER_L4_TCP,
3838                 [340] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3839                         RTE_PTYPE_TUNNEL_GTPU |
3840                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3841                         RTE_PTYPE_INNER_L4_ICMP,
3842
3843                 /* IPv4 --> GTPU --> IPv6 */
3844                 [341] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3845                         RTE_PTYPE_TUNNEL_GTPU |
3846                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3847                         RTE_PTYPE_INNER_L4_FRAG,
3848                 [342] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3849                         RTE_PTYPE_TUNNEL_GTPU |
3850                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3851                         RTE_PTYPE_INNER_L4_NONFRAG,
3852                 [343] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3853                         RTE_PTYPE_TUNNEL_GTPU |
3854                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3855                         RTE_PTYPE_INNER_L4_UDP,
3856                 [344] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3857                         RTE_PTYPE_TUNNEL_GTPU |
3858                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3859                         RTE_PTYPE_INNER_L4_TCP,
3860                 [345] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3861                         RTE_PTYPE_TUNNEL_GTPU |
3862                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3863                         RTE_PTYPE_INNER_L4_ICMP,
3864
3865                 /* IPv6 --> GTPU --> IPv6 */
3866                 [346] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3867                         RTE_PTYPE_TUNNEL_GTPU |
3868                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3869                         RTE_PTYPE_INNER_L4_FRAG,
3870                 [347] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3871                         RTE_PTYPE_TUNNEL_GTPU |
3872                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3873                         RTE_PTYPE_INNER_L4_NONFRAG,
3874                 [348] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3875                         RTE_PTYPE_TUNNEL_GTPU |
3876                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3877                         RTE_PTYPE_INNER_L4_UDP,
3878                 [349] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3879                         RTE_PTYPE_TUNNEL_GTPU |
3880                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3881                         RTE_PTYPE_INNER_L4_TCP,
3882                 [350] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3883                         RTE_PTYPE_TUNNEL_GTPU |
3884                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3885                         RTE_PTYPE_INNER_L4_ICMP,
3886
3887                 /* IPv4 --> UDP ECPRI */
3888                 [372] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3889                         RTE_PTYPE_L4_UDP,
3890                 [373] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3891                         RTE_PTYPE_L4_UDP,
3892                 [374] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3893                         RTE_PTYPE_L4_UDP,
3894                 [375] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3895                         RTE_PTYPE_L4_UDP,
3896                 [376] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3897                         RTE_PTYPE_L4_UDP,
3898                 [377] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3899                         RTE_PTYPE_L4_UDP,
3900                 [378] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3901                         RTE_PTYPE_L4_UDP,
3902                 [379] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3903                         RTE_PTYPE_L4_UDP,
3904                 [380] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3905                         RTE_PTYPE_L4_UDP,
3906                 [381] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3907                         RTE_PTYPE_L4_UDP,
3908
3909                 /* IPV6 --> UDP ECPRI */
3910                 [382] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3911                         RTE_PTYPE_L4_UDP,
3912                 [383] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3913                         RTE_PTYPE_L4_UDP,
3914                 [384] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3915                         RTE_PTYPE_L4_UDP,
3916                 [385] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3917                         RTE_PTYPE_L4_UDP,
3918                 [386] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3919                         RTE_PTYPE_L4_UDP,
3920                 [387] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3921                         RTE_PTYPE_L4_UDP,
3922                 [388] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3923                         RTE_PTYPE_L4_UDP,
3924                 [389] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3925                         RTE_PTYPE_L4_UDP,
3926                 [390] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3927                         RTE_PTYPE_L4_UDP,
3928                 [391] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3929                         RTE_PTYPE_L4_UDP,
3930                 /* All others reserved */
3931         };
3932
3933         return ptype_tbl[ptype];
3934 }
3935
3936 void __rte_cold
3937 iavf_set_default_ptype_table(struct rte_eth_dev *dev)
3938 {
3939         struct iavf_adapter *ad =
3940                 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
3941         int i;
3942
3943         for (i = 0; i < IAVF_MAX_PKT_TYPE; i++)
3944                 ad->ptype_tbl[i] = iavf_get_default_ptype(i);
3945 }