drivers/net/ice/ice_rxtx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4
   5 #include <ethdev_driver.h>
   6 #include <rte_net.h>
   7 #include <rte_vect.h>
   8
   9 #include "rte_pmd_ice.h"
  10 #include "ice_rxtx.h"
  11 #include "ice_rxtx_vec_common.h"
  12
  13 #define ICE_TX_CKSUM_OFFLOAD_MASK (              \
  14                 PKT_TX_IP_CKSUM |                \
  15                 PKT_TX_L4_MASK |                 \
  16                 PKT_TX_TCP_SEG |                 \
  17                 PKT_TX_OUTER_IP_CKSUM)
  18
  19 /* Offset of mbuf dynamic field for protocol extraction data */
  20 int rte_net_ice_dynfield_proto_xtr_metadata_offs = -1;
  21
  22 /* Mask of mbuf dynamic flags for protocol extraction type */
  23 uint64_t rte_net_ice_dynflag_proto_xtr_vlan_mask;
  24 uint64_t rte_net_ice_dynflag_proto_xtr_ipv4_mask;
  25 uint64_t rte_net_ice_dynflag_proto_xtr_ipv6_mask;
  26 uint64_t rte_net_ice_dynflag_proto_xtr_ipv6_flow_mask;
  27 uint64_t rte_net_ice_dynflag_proto_xtr_tcp_mask;
  28 uint64_t rte_net_ice_dynflag_proto_xtr_ip_offset_mask;
  29
  30 static int
  31 ice_monitor_callback(const uint64_t value,
  32                 const uint64_t arg[RTE_POWER_MONITOR_OPAQUE_SZ] __rte_unused)
  33 {
  34         const uint64_t m = rte_cpu_to_le_16(1 << ICE_RX_FLEX_DESC_STATUS0_DD_S);
  35         /*
  36          * we expect the DD bit to be set to 1 if this descriptor was already
  37          * written to.
  38          */
  39         return (value & m) == m ? -1 : 0;
  40 }
  41
  42 int
  43 ice_get_monitor_addr(void *rx_queue, struct rte_power_monitor_cond *pmc)
  44 {
  45         volatile union ice_rx_flex_desc *rxdp;
  46         struct ice_rx_queue *rxq = rx_queue;
  47         uint16_t desc;
  48
  49         desc = rxq->rx_tail;
  50         rxdp = &rxq->rx_ring[desc];
  51         /* watch for changes in status bit */
  52         pmc->addr = &rxdp->wb.status_error0;
  53
  54         /* comparison callback */
  55         pmc->fn = ice_monitor_callback;
  56
  57         /* register is 16-bit */
  58         pmc->size = sizeof(uint16_t);
  59
  60         return 0;
  61 }
  62
  63
  64 static inline uint8_t
  65 ice_proto_xtr_type_to_rxdid(uint8_t xtr_type)
  66 {
  67         static uint8_t rxdid_map[] = {
  68                 [PROTO_XTR_NONE]      = ICE_RXDID_COMMS_OVS,
  69                 [PROTO_XTR_VLAN]      = ICE_RXDID_COMMS_AUX_VLAN,
  70                 [PROTO_XTR_IPV4]      = ICE_RXDID_COMMS_AUX_IPV4,
  71                 [PROTO_XTR_IPV6]      = ICE_RXDID_COMMS_AUX_IPV6,
  72                 [PROTO_XTR_IPV6_FLOW] = ICE_RXDID_COMMS_AUX_IPV6_FLOW,
  73                 [PROTO_XTR_TCP]       = ICE_RXDID_COMMS_AUX_TCP,
  74                 [PROTO_XTR_IP_OFFSET] = ICE_RXDID_COMMS_AUX_IP_OFFSET,
  75         };
  76
  77         return xtr_type < RTE_DIM(rxdid_map) ?
  78                                 rxdid_map[xtr_type] : ICE_RXDID_COMMS_OVS;
  79 }
  80
  81 static inline void
  82 ice_rxd_to_pkt_fields_by_comms_generic(__rte_unused struct ice_rx_queue *rxq,
  83                                        struct rte_mbuf *mb,
  84                                        volatile union ice_rx_flex_desc *rxdp)
  85 {
  86         volatile struct ice_32b_rx_flex_desc_comms *desc =
  87                         (volatile struct ice_32b_rx_flex_desc_comms *)rxdp;
  88         uint16_t stat_err = rte_le_to_cpu_16(desc->status_error0);
  89
  90         if (likely(stat_err & (1 << ICE_RX_FLEX_DESC_STATUS0_RSS_VALID_S))) {
  91                 mb->ol_flags |= PKT_RX_RSS_HASH;
  92                 mb->hash.rss = rte_le_to_cpu_32(desc->rss_hash);
  93         }
  94
  95 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
  96         if (desc->flow_id != 0xFFFFFFFF) {
  97                 mb->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID;
  98                 mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id);
  99         }
 100 #endif
 101 }
 102
 103 static inline void
 104 ice_rxd_to_pkt_fields_by_comms_ovs(__rte_unused struct ice_rx_queue *rxq,
 105                                    struct rte_mbuf *mb,
 106                                    volatile union ice_rx_flex_desc *rxdp)
 107 {
 108         volatile struct ice_32b_rx_flex_desc_comms_ovs *desc =
 109                         (volatile struct ice_32b_rx_flex_desc_comms_ovs *)rxdp;
 110 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
 111         uint16_t stat_err;
 112 #endif
 113
 114         if (desc->flow_id != 0xFFFFFFFF) {
 115                 mb->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID;
 116                 mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id);
 117         }
 118
 119 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
 120         stat_err = rte_le_to_cpu_16(desc->status_error0);
 121         if (likely(stat_err & (1 << ICE_RX_FLEX_DESC_STATUS0_RSS_VALID_S))) {
 122                 mb->ol_flags |= PKT_RX_RSS_HASH;
 123                 mb->hash.rss = rte_le_to_cpu_32(desc->rss_hash);
 124         }
 125 #endif
 126 }
 127
 128 static inline void
 129 ice_rxd_to_pkt_fields_by_comms_aux_v1(struct ice_rx_queue *rxq,
 130                                       struct rte_mbuf *mb,
 131                                       volatile union ice_rx_flex_desc *rxdp)
 132 {
 133         volatile struct ice_32b_rx_flex_desc_comms *desc =
 134                         (volatile struct ice_32b_rx_flex_desc_comms *)rxdp;
 135         uint16_t stat_err;
 136
 137         stat_err = rte_le_to_cpu_16(desc->status_error0);
 138         if (likely(stat_err & (1 << ICE_RX_FLEX_DESC_STATUS0_RSS_VALID_S))) {
 139                 mb->ol_flags |= PKT_RX_RSS_HASH;
 140                 mb->hash.rss = rte_le_to_cpu_32(desc->rss_hash);
 141         }
 142
 143 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
 144         if (desc->flow_id != 0xFFFFFFFF) {
 145                 mb->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID;
 146                 mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id);
 147         }
 148
 149         if (rxq->xtr_ol_flag) {
 150                 uint32_t metadata = 0;
 151
 152                 stat_err = rte_le_to_cpu_16(desc->status_error1);
 153
 154                 if (stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S))
 155                         metadata = rte_le_to_cpu_16(desc->flex_ts.flex.aux0);
 156
 157                 if (stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S))
 158                         metadata |=
 159                                 rte_le_to_cpu_16(desc->flex_ts.flex.aux1) << 16;
 160
 161                 if (metadata) {
 162                         mb->ol_flags |= rxq->xtr_ol_flag;
 163
 164                         *RTE_NET_ICE_DYNF_PROTO_XTR_METADATA(mb) = metadata;
 165                 }
 166         }
 167 #endif
 168 }
 169
 170 static inline void
 171 ice_rxd_to_pkt_fields_by_comms_aux_v2(struct ice_rx_queue *rxq,
 172                                       struct rte_mbuf *mb,
 173                                       volatile union ice_rx_flex_desc *rxdp)
 174 {
 175         volatile struct ice_32b_rx_flex_desc_comms *desc =
 176                         (volatile struct ice_32b_rx_flex_desc_comms *)rxdp;
 177         uint16_t stat_err;
 178
 179         stat_err = rte_le_to_cpu_16(desc->status_error0);
 180         if (likely(stat_err & (1 << ICE_RX_FLEX_DESC_STATUS0_RSS_VALID_S))) {
 181                 mb->ol_flags |= PKT_RX_RSS_HASH;
 182                 mb->hash.rss = rte_le_to_cpu_32(desc->rss_hash);
 183         }
 184
 185 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
 186         if (desc->flow_id != 0xFFFFFFFF) {
 187                 mb->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID;
 188                 mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id);
 189         }
 190
 191         if (rxq->xtr_ol_flag) {
 192                 uint32_t metadata = 0;
 193
 194                 if (desc->flex_ts.flex.aux0 != 0xFFFF)
 195                         metadata = rte_le_to_cpu_16(desc->flex_ts.flex.aux0);
 196                 else if (desc->flex_ts.flex.aux1 != 0xFFFF)
 197                         metadata = rte_le_to_cpu_16(desc->flex_ts.flex.aux1);
 198
 199                 if (metadata) {
 200                         mb->ol_flags |= rxq->xtr_ol_flag;
 201
 202                         *RTE_NET_ICE_DYNF_PROTO_XTR_METADATA(mb) = metadata;
 203                 }
 204         }
 205 #endif
 206 }
 207
 208 void
 209 ice_select_rxd_to_pkt_fields_handler(struct ice_rx_queue *rxq, uint32_t rxdid)
 210 {
 211         switch (rxdid) {
 212         case ICE_RXDID_COMMS_AUX_VLAN:
 213                 rxq->xtr_ol_flag = rte_net_ice_dynflag_proto_xtr_vlan_mask;
 214                 rxq->rxd_to_pkt_fields = ice_rxd_to_pkt_fields_by_comms_aux_v1;
 215                 break;
 216
 217         case ICE_RXDID_COMMS_AUX_IPV4:
 218                 rxq->xtr_ol_flag = rte_net_ice_dynflag_proto_xtr_ipv4_mask;
 219                 rxq->rxd_to_pkt_fields = ice_rxd_to_pkt_fields_by_comms_aux_v1;
 220                 break;
 221
 222         case ICE_RXDID_COMMS_AUX_IPV6:
 223                 rxq->xtr_ol_flag = rte_net_ice_dynflag_proto_xtr_ipv6_mask;
 224                 rxq->rxd_to_pkt_fields = ice_rxd_to_pkt_fields_by_comms_aux_v1;
 225                 break;
 226
 227         case ICE_RXDID_COMMS_AUX_IPV6_FLOW:
 228                 rxq->xtr_ol_flag = rte_net_ice_dynflag_proto_xtr_ipv6_flow_mask;
 229                 rxq->rxd_to_pkt_fields = ice_rxd_to_pkt_fields_by_comms_aux_v1;
 230                 break;
 231
 232         case ICE_RXDID_COMMS_AUX_TCP:
 233                 rxq->xtr_ol_flag = rte_net_ice_dynflag_proto_xtr_tcp_mask;
 234                 rxq->rxd_to_pkt_fields = ice_rxd_to_pkt_fields_by_comms_aux_v1;
 235                 break;
 236
 237         case ICE_RXDID_COMMS_AUX_IP_OFFSET:
 238                 rxq->xtr_ol_flag = rte_net_ice_dynflag_proto_xtr_ip_offset_mask;
 239                 rxq->rxd_to_pkt_fields = ice_rxd_to_pkt_fields_by_comms_aux_v2;
 240                 break;
 241
 242         case ICE_RXDID_COMMS_GENERIC:
 243                 rxq->rxd_to_pkt_fields = ice_rxd_to_pkt_fields_by_comms_generic;
 244                 break;
 245
 246         case ICE_RXDID_COMMS_OVS:
 247                 rxq->rxd_to_pkt_fields = ice_rxd_to_pkt_fields_by_comms_ovs;
 248                 break;
 249
 250         default:
 251                 /* update this according to the RXDID for PROTO_XTR_NONE */
 252                 rxq->rxd_to_pkt_fields = ice_rxd_to_pkt_fields_by_comms_ovs;
 253                 break;
 254         }
 255
 256         if (!rte_net_ice_dynf_proto_xtr_metadata_avail())
 257                 rxq->xtr_ol_flag = 0;
 258 }
 259
 260 static enum ice_status
 261 ice_program_hw_rx_queue(struct ice_rx_queue *rxq)
 262 {
 263         struct ice_vsi *vsi = rxq->vsi;
 264         struct ice_hw *hw = ICE_VSI_TO_HW(vsi);
 265         struct ice_pf *pf = ICE_VSI_TO_PF(vsi);
 266         struct rte_eth_dev_data *dev_data = rxq->vsi->adapter->pf.dev_data;
 267         struct ice_rlan_ctx rx_ctx;
 268         enum ice_status err;
 269         uint16_t buf_size;
 270         uint32_t rxdid = ICE_RXDID_COMMS_OVS;
 271         uint32_t regval;
 272         struct ice_adapter *ad = rxq->vsi->adapter;
 273         uint32_t frame_size = dev_data->mtu + ICE_ETH_OVERHEAD;
 274
 275         /* Set buffer size as the head split is disabled. */
 276         buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 277                               RTE_PKTMBUF_HEADROOM);
 278         rxq->rx_hdr_len = 0;
 279         rxq->rx_buf_len = RTE_ALIGN(buf_size, (1 << ICE_RLAN_CTX_DBUF_S));
 280         rxq->max_pkt_len =
 281                 RTE_MIN((uint32_t)ICE_SUPPORT_CHAIN_NUM * rxq->rx_buf_len,
 282                         frame_size);
 283
 284         if (rxq->max_pkt_len <= RTE_ETHER_MIN_LEN ||
 285             rxq->max_pkt_len > ICE_FRAME_SIZE_MAX) {
 286                 PMD_DRV_LOG(ERR, "maximum packet length must "
 287                             "be larger than %u and smaller than %u",
 288                             (uint32_t)RTE_ETHER_MIN_LEN,
 289                             (uint32_t)ICE_FRAME_SIZE_MAX);
 290                 return -EINVAL;
 291         }
 292
 293         if (rxq->offloads & DEV_RX_OFFLOAD_TIMESTAMP) {
 294                 /* Register mbuf field and flag for Rx timestamp */
 295                 err = rte_mbuf_dyn_rx_timestamp_register(
 296                                 &ice_timestamp_dynfield_offset,
 297                                 &ice_timestamp_dynflag);
 298                 if (err) {
 299                         PMD_DRV_LOG(ERR,
 300                                 "Cannot register mbuf field/flag for timestamp");
 301                         return -EINVAL;
 302                 }
 303         }
 304
 305         memset(&rx_ctx, 0, sizeof(rx_ctx));
 306
 307         rx_ctx.base = rxq->rx_ring_dma / ICE_QUEUE_BASE_ADDR_UNIT;
 308         rx_ctx.qlen = rxq->nb_rx_desc;
 309         rx_ctx.dbuf = rxq->rx_buf_len >> ICE_RLAN_CTX_DBUF_S;
 310         rx_ctx.hbuf = rxq->rx_hdr_len >> ICE_RLAN_CTX_HBUF_S;
 311         rx_ctx.dtype = 0; /* No Header Split mode */
 312 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
 313         rx_ctx.dsize = 1; /* 32B descriptors */
 314 #endif
 315         rx_ctx.rxmax = rxq->max_pkt_len;
 316         /* TPH: Transaction Layer Packet (TLP) processing hints */
 317         rx_ctx.tphrdesc_ena = 1;
 318         rx_ctx.tphwdesc_ena = 1;
 319         rx_ctx.tphdata_ena = 1;
 320         rx_ctx.tphhead_ena = 1;
 321         /* Low Receive Queue Threshold defined in 64 descriptors units.
 322          * When the number of free descriptors goes below the lrxqthresh,
 323          * an immediate interrupt is triggered.
 324          */
 325         rx_ctx.lrxqthresh = 2;
 326         /*default use 32 byte descriptor, vlan tag extract to L2TAG2(1st)*/
 327         rx_ctx.l2tsel = 1;
 328         rx_ctx.showiv = 0;
 329         rx_ctx.crcstrip = (rxq->crc_len == 0) ? 1 : 0;
 330
 331         rxdid = ice_proto_xtr_type_to_rxdid(rxq->proto_xtr);
 332
 333         PMD_DRV_LOG(DEBUG, "Port (%u) - Rx queue (%u) is set with RXDID : %u",
 334                     rxq->port_id, rxq->queue_id, rxdid);
 335
 336         if (!(pf->supported_rxdid & BIT(rxdid))) {
 337                 PMD_DRV_LOG(ERR, "currently package doesn't support RXDID (%u)",
 338                             rxdid);
 339                 return -EINVAL;
 340         }
 341
 342         ice_select_rxd_to_pkt_fields_handler(rxq, rxdid);
 343
 344         /* Enable Flexible Descriptors in the queue context which
 345          * allows this driver to select a specific receive descriptor format
 346          */
 347         regval = (rxdid << QRXFLXP_CNTXT_RXDID_IDX_S) &
 348                 QRXFLXP_CNTXT_RXDID_IDX_M;
 349
 350         /* increasing context priority to pick up profile ID;
 351          * default is 0x01; setting to 0x03 to ensure profile
 352          * is programming if prev context is of same priority
 353          */
 354         regval |= (0x03 << QRXFLXP_CNTXT_RXDID_PRIO_S) &
 355                 QRXFLXP_CNTXT_RXDID_PRIO_M;
 356
 357         if (ad->ptp_ena || rxq->offloads & DEV_RX_OFFLOAD_TIMESTAMP)
 358                 regval |= QRXFLXP_CNTXT_TS_M;
 359
 360         ICE_WRITE_REG(hw, QRXFLXP_CNTXT(rxq->reg_idx), regval);
 361
 362         err = ice_clear_rxq_ctx(hw, rxq->reg_idx);
 363         if (err) {
 364                 PMD_DRV_LOG(ERR, "Failed to clear Lan Rx queue (%u) context",
 365                             rxq->queue_id);
 366                 return -EINVAL;
 367         }
 368         err = ice_write_rxq_ctx(hw, &rx_ctx, rxq->reg_idx);
 369         if (err) {
 370                 PMD_DRV_LOG(ERR, "Failed to write Lan Rx queue (%u) context",
 371                             rxq->queue_id);
 372                 return -EINVAL;
 373         }
 374
 375         /* Check if scattered RX needs to be used. */
 376         if (frame_size > buf_size)
 377                 dev_data->scattered_rx = 1;
 378
 379         rxq->qrx_tail = hw->hw_addr + QRX_TAIL(rxq->reg_idx);
 380
 381         /* Init the Rx tail register*/
 382         ICE_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
 383
 384         return 0;
 385 }
 386
 387 /* Allocate mbufs for all descriptors in rx queue */
 388 static int
 389 ice_alloc_rx_queue_mbufs(struct ice_rx_queue *rxq)
 390 {
 391         struct ice_rx_entry *rxe = rxq->sw_ring;
 392         uint64_t dma_addr;
 393         uint16_t i;
 394
 395         for (i = 0; i < rxq->nb_rx_desc; i++) {
 396                 volatile union ice_rx_flex_desc *rxd;
 397                 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mp);
 398
 399                 if (unlikely(!mbuf)) {
 400                         PMD_DRV_LOG(ERR, "Failed to allocate mbuf for RX");
 401                         return -ENOMEM;
 402                 }
 403
 404                 rte_mbuf_refcnt_set(mbuf, 1);
 405                 mbuf->next = NULL;
 406                 mbuf->data_off = RTE_PKTMBUF_HEADROOM;
 407                 mbuf->nb_segs = 1;
 408                 mbuf->port = rxq->port_id;
 409
 410                 dma_addr =
 411                         rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf));
 412
 413                 rxd = &rxq->rx_ring[i];
 414                 rxd->read.pkt_addr = dma_addr;
 415                 rxd->read.hdr_addr = 0;
 416 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
 417                 rxd->read.rsvd1 = 0;
 418                 rxd->read.rsvd2 = 0;
 419 #endif
 420                 rxe[i].mbuf = mbuf;
 421         }
 422
 423         return 0;
 424 }
 425
 426 /* Free all mbufs for descriptors in rx queue */
 427 static void
 428 _ice_rx_queue_release_mbufs(struct ice_rx_queue *rxq)
 429 {
 430         uint16_t i;
 431
 432         if (!rxq || !rxq->sw_ring) {
 433                 PMD_DRV_LOG(DEBUG, "Pointer to sw_ring is NULL");
 434                 return;
 435         }
 436
 437         for (i = 0; i < rxq->nb_rx_desc; i++) {
 438                 if (rxq->sw_ring[i].mbuf) {
 439                         rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
 440                         rxq->sw_ring[i].mbuf = NULL;
 441                 }
 442         }
 443         if (rxq->rx_nb_avail == 0)
 444                 return;
 445         for (i = 0; i < rxq->rx_nb_avail; i++)
 446                 rte_pktmbuf_free_seg(rxq->rx_stage[rxq->rx_next_avail + i]);
 447
 448         rxq->rx_nb_avail = 0;
 449 }
 450
 451 /* turn on or off rx queue
 452  * @q_idx: queue index in pf scope
 453  * @on: turn on or off the queue
 454  */
 455 static int
 456 ice_switch_rx_queue(struct ice_hw *hw, uint16_t q_idx, bool on)
 457 {
 458         uint32_t reg;
 459         uint16_t j;
 460
 461         /* QRX_CTRL = QRX_ENA */
 462         reg = ICE_READ_REG(hw, QRX_CTRL(q_idx));
 463
 464         if (on) {
 465                 if (reg & QRX_CTRL_QENA_STAT_M)
 466                         return 0; /* Already on, skip */
 467                 reg |= QRX_CTRL_QENA_REQ_M;
 468         } else {
 469                 if (!(reg & QRX_CTRL_QENA_STAT_M))
 470                         return 0; /* Already off, skip */
 471                 reg &= ~QRX_CTRL_QENA_REQ_M;
 472         }
 473
 474         /* Write the register */
 475         ICE_WRITE_REG(hw, QRX_CTRL(q_idx), reg);
 476         /* Check the result. It is said that QENA_STAT
 477          * follows the QENA_REQ not more than 10 use.
 478          * TODO: need to change the wait counter later
 479          */
 480         for (j = 0; j < ICE_CHK_Q_ENA_COUNT; j++) {
 481                 rte_delay_us(ICE_CHK_Q_ENA_INTERVAL_US);
 482                 reg = ICE_READ_REG(hw, QRX_CTRL(q_idx));
 483                 if (on) {
 484                         if ((reg & QRX_CTRL_QENA_REQ_M) &&
 485                             (reg & QRX_CTRL_QENA_STAT_M))
 486                                 break;
 487                 } else {
 488                         if (!(reg & QRX_CTRL_QENA_REQ_M) &&
 489                             !(reg & QRX_CTRL_QENA_STAT_M))
 490                                 break;
 491                 }
 492         }
 493
 494         /* Check if it is timeout */
 495         if (j >= ICE_CHK_Q_ENA_COUNT) {
 496                 PMD_DRV_LOG(ERR, "Failed to %s rx queue[%u]",
 497                             (on ? "enable" : "disable"), q_idx);
 498                 return -ETIMEDOUT;
 499         }
 500
 501         return 0;
 502 }
 503
 504 static inline int
 505 ice_check_rx_burst_bulk_alloc_preconditions(struct ice_rx_queue *rxq)
 506 {
 507         int ret = 0;
 508
 509         if (!(rxq->rx_free_thresh >= ICE_RX_MAX_BURST)) {
 510                 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
 511                              "rxq->rx_free_thresh=%d, "
 512                              "ICE_RX_MAX_BURST=%d",
 513                              rxq->rx_free_thresh, ICE_RX_MAX_BURST);
 514                 ret = -EINVAL;
 515         } else if (!(rxq->rx_free_thresh < rxq->nb_rx_desc)) {
 516                 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
 517                              "rxq->rx_free_thresh=%d, "
 518                              "rxq->nb_rx_desc=%d",
 519                              rxq->rx_free_thresh, rxq->nb_rx_desc);
 520                 ret = -EINVAL;
 521         } else if (rxq->nb_rx_desc % rxq->rx_free_thresh != 0) {
 522                 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
 523                              "rxq->nb_rx_desc=%d, "
 524                              "rxq->rx_free_thresh=%d",
 525                              rxq->nb_rx_desc, rxq->rx_free_thresh);
 526                 ret = -EINVAL;
 527         }
 528
 529         return ret;
 530 }
 531
 532 /* reset fields in ice_rx_queue back to default */
 533 static void
 534 ice_reset_rx_queue(struct ice_rx_queue *rxq)
 535 {
 536         unsigned int i;
 537         uint16_t len;
 538
 539         if (!rxq) {
 540                 PMD_DRV_LOG(DEBUG, "Pointer to rxq is NULL");
 541                 return;
 542         }
 543
 544         len = (uint16_t)(rxq->nb_rx_desc + ICE_RX_MAX_BURST);
 545
 546         for (i = 0; i < len * sizeof(union ice_rx_flex_desc); i++)
 547                 ((volatile char *)rxq->rx_ring)[i] = 0;
 548
 549         memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf));
 550         for (i = 0; i < ICE_RX_MAX_BURST; ++i)
 551                 rxq->sw_ring[rxq->nb_rx_desc + i].mbuf = &rxq->fake_mbuf;
 552
 553         rxq->rx_nb_avail = 0;
 554         rxq->rx_next_avail = 0;
 555         rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1);
 556
 557         rxq->rx_tail = 0;
 558         rxq->nb_rx_hold = 0;
 559         rxq->pkt_first_seg = NULL;
 560         rxq->pkt_last_seg = NULL;
 561
 562         rxq->rxrearm_start = 0;
 563         rxq->rxrearm_nb = 0;
 564 }
 565
 566 int
 567 ice_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 568 {
 569         struct ice_rx_queue *rxq;
 570         int err;
 571         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 572
 573         PMD_INIT_FUNC_TRACE();
 574
 575         if (rx_queue_id >= dev->data->nb_rx_queues) {
 576                 PMD_DRV_LOG(ERR, "RX queue %u is out of range %u",
 577                             rx_queue_id, dev->data->nb_rx_queues);
 578                 return -EINVAL;
 579         }
 580
 581         rxq = dev->data->rx_queues[rx_queue_id];
 582         if (!rxq || !rxq->q_set) {
 583                 PMD_DRV_LOG(ERR, "RX queue %u not available or setup",
 584                             rx_queue_id);
 585                 return -EINVAL;
 586         }
 587
 588         err = ice_program_hw_rx_queue(rxq);
 589         if (err) {
 590                 PMD_DRV_LOG(ERR, "fail to program RX queue %u",
 591                             rx_queue_id);
 592                 return -EIO;
 593         }
 594
 595         err = ice_alloc_rx_queue_mbufs(rxq);
 596         if (err) {
 597                 PMD_DRV_LOG(ERR, "Failed to allocate RX queue mbuf");
 598                 return -ENOMEM;
 599         }
 600
 601         /* Init the RX tail register. */
 602         ICE_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
 603
 604         err = ice_switch_rx_queue(hw, rxq->reg_idx, true);
 605         if (err) {
 606                 PMD_DRV_LOG(ERR, "Failed to switch RX queue %u on",
 607                             rx_queue_id);
 608
 609                 rxq->rx_rel_mbufs(rxq);
 610                 ice_reset_rx_queue(rxq);
 611                 return -EINVAL;
 612         }
 613
 614         dev->data->rx_queue_state[rx_queue_id] =
 615                 RTE_ETH_QUEUE_STATE_STARTED;
 616
 617         return 0;
 618 }
 619
 620 int
 621 ice_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 622 {
 623         struct ice_rx_queue *rxq;
 624         int err;
 625         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 626
 627         if (rx_queue_id < dev->data->nb_rx_queues) {
 628                 rxq = dev->data->rx_queues[rx_queue_id];
 629
 630                 err = ice_switch_rx_queue(hw, rxq->reg_idx, false);
 631                 if (err) {
 632                         PMD_DRV_LOG(ERR, "Failed to switch RX queue %u off",
 633                                     rx_queue_id);
 634                         return -EINVAL;
 635                 }
 636                 rxq->rx_rel_mbufs(rxq);
 637                 ice_reset_rx_queue(rxq);
 638                 dev->data->rx_queue_state[rx_queue_id] =
 639                         RTE_ETH_QUEUE_STATE_STOPPED;
 640         }
 641
 642         return 0;
 643 }
 644
 645 int
 646 ice_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 647 {
 648         struct ice_tx_queue *txq;
 649         int err;
 650         struct ice_vsi *vsi;
 651         struct ice_hw *hw;
 652         struct ice_aqc_add_tx_qgrp *txq_elem;
 653         struct ice_tlan_ctx tx_ctx;
 654         int buf_len;
 655
 656         PMD_INIT_FUNC_TRACE();
 657
 658         if (tx_queue_id >= dev->data->nb_tx_queues) {
 659                 PMD_DRV_LOG(ERR, "TX queue %u is out of range %u",
 660                             tx_queue_id, dev->data->nb_tx_queues);
 661                 return -EINVAL;
 662         }
 663
 664         txq = dev->data->tx_queues[tx_queue_id];
 665         if (!txq || !txq->q_set) {
 666                 PMD_DRV_LOG(ERR, "TX queue %u is not available or setup",
 667                             tx_queue_id);
 668                 return -EINVAL;
 669         }
 670
 671         buf_len = ice_struct_size(txq_elem, txqs, 1);
 672         txq_elem = ice_malloc(hw, buf_len);
 673         if (!txq_elem)
 674                 return -ENOMEM;
 675
 676         vsi = txq->vsi;
 677         hw = ICE_VSI_TO_HW(vsi);
 678
 679         memset(&tx_ctx, 0, sizeof(tx_ctx));
 680         txq_elem->num_txqs = 1;
 681         txq_elem->txqs[0].txq_id = rte_cpu_to_le_16(txq->reg_idx);
 682
 683         tx_ctx.base = txq->tx_ring_dma / ICE_QUEUE_BASE_ADDR_UNIT;
 684         tx_ctx.qlen = txq->nb_tx_desc;
 685         tx_ctx.pf_num = hw->pf_id;
 686         tx_ctx.vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_PF;
 687         tx_ctx.src_vsi = vsi->vsi_id;
 688         tx_ctx.port_num = hw->port_info->lport;
 689         tx_ctx.tso_ena = 1; /* tso enable */
 690         tx_ctx.tso_qnum = txq->reg_idx; /* index for tso state structure */
 691         tx_ctx.legacy_int = 1; /* Legacy or Advanced Host Interface */
 692         tx_ctx.tsyn_ena = 1;
 693
 694         ice_set_ctx(hw, (uint8_t *)&tx_ctx, txq_elem->txqs[0].txq_ctx,
 695                     ice_tlan_ctx_info);
 696
 697         txq->qtx_tail = hw->hw_addr + QTX_COMM_DBELL(txq->reg_idx);
 698
 699         /* Init the Tx tail register*/
 700         ICE_PCI_REG_WRITE(txq->qtx_tail, 0);
 701
 702         /* Fix me, we assume TC always 0 here */
 703         err = ice_ena_vsi_txq(hw->port_info, vsi->idx, 0, tx_queue_id, 1,
 704                         txq_elem, buf_len, NULL);
 705         if (err) {
 706                 PMD_DRV_LOG(ERR, "Failed to add lan txq");
 707                 rte_free(txq_elem);
 708                 return -EIO;
 709         }
 710         /* store the schedule node id */
 711         txq->q_teid = txq_elem->txqs[0].q_teid;
 712
 713         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 714
 715         rte_free(txq_elem);
 716         return 0;
 717 }
 718
 719 static enum ice_status
 720 ice_fdir_program_hw_rx_queue(struct ice_rx_queue *rxq)
 721 {
 722         struct ice_vsi *vsi = rxq->vsi;
 723         struct ice_hw *hw = ICE_VSI_TO_HW(vsi);
 724         uint32_t rxdid = ICE_RXDID_LEGACY_1;
 725         struct ice_rlan_ctx rx_ctx;
 726         enum ice_status err;
 727         uint32_t regval;
 728
 729         rxq->rx_hdr_len = 0;
 730         rxq->rx_buf_len = 1024;
 731
 732         memset(&rx_ctx, 0, sizeof(rx_ctx));
 733
 734         rx_ctx.base = rxq->rx_ring_dma / ICE_QUEUE_BASE_ADDR_UNIT;
 735         rx_ctx.qlen = rxq->nb_rx_desc;
 736         rx_ctx.dbuf = rxq->rx_buf_len >> ICE_RLAN_CTX_DBUF_S;
 737         rx_ctx.hbuf = rxq->rx_hdr_len >> ICE_RLAN_CTX_HBUF_S;
 738         rx_ctx.dtype = 0; /* No Header Split mode */
 739         rx_ctx.dsize = 1; /* 32B descriptors */
 740         rx_ctx.rxmax = ICE_ETH_MAX_LEN;
 741         /* TPH: Transaction Layer Packet (TLP) processing hints */
 742         rx_ctx.tphrdesc_ena = 1;
 743         rx_ctx.tphwdesc_ena = 1;
 744         rx_ctx.tphdata_ena = 1;
 745         rx_ctx.tphhead_ena = 1;
 746         /* Low Receive Queue Threshold defined in 64 descriptors units.
 747          * When the number of free descriptors goes below the lrxqthresh,
 748          * an immediate interrupt is triggered.
 749          */
 750         rx_ctx.lrxqthresh = 2;
 751         /*default use 32 byte descriptor, vlan tag extract to L2TAG2(1st)*/
 752         rx_ctx.l2tsel = 1;
 753         rx_ctx.showiv = 0;
 754         rx_ctx.crcstrip = (rxq->crc_len == 0) ? 1 : 0;
 755
 756         /* Enable Flexible Descriptors in the queue context which
 757          * allows this driver to select a specific receive descriptor format
 758          */
 759         regval = (rxdid << QRXFLXP_CNTXT_RXDID_IDX_S) &
 760                 QRXFLXP_CNTXT_RXDID_IDX_M;
 761
 762         /* increasing context priority to pick up profile ID;
 763          * default is 0x01; setting to 0x03 to ensure profile
 764          * is programming if prev context is of same priority
 765          */
 766         regval |= (0x03 << QRXFLXP_CNTXT_RXDID_PRIO_S) &
 767                 QRXFLXP_CNTXT_RXDID_PRIO_M;
 768
 769         ICE_WRITE_REG(hw, QRXFLXP_CNTXT(rxq->reg_idx), regval);
 770
 771         err = ice_clear_rxq_ctx(hw, rxq->reg_idx);
 772         if (err) {
 773                 PMD_DRV_LOG(ERR, "Failed to clear Lan Rx queue (%u) context",
 774                             rxq->queue_id);
 775                 return -EINVAL;
 776         }
 777         err = ice_write_rxq_ctx(hw, &rx_ctx, rxq->reg_idx);
 778         if (err) {
 779                 PMD_DRV_LOG(ERR, "Failed to write Lan Rx queue (%u) context",
 780                             rxq->queue_id);
 781                 return -EINVAL;
 782         }
 783
 784         rxq->qrx_tail = hw->hw_addr + QRX_TAIL(rxq->reg_idx);
 785
 786         /* Init the Rx tail register*/
 787         ICE_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
 788
 789         return 0;
 790 }
 791
 792 int
 793 ice_fdir_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 794 {
 795         struct ice_rx_queue *rxq;
 796         int err;
 797         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 798         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 799
 800         PMD_INIT_FUNC_TRACE();
 801
 802         rxq = pf->fdir.rxq;
 803         if (!rxq || !rxq->q_set) {
 804                 PMD_DRV_LOG(ERR, "FDIR RX queue %u not available or setup",
 805                             rx_queue_id);
 806                 return -EINVAL;
 807         }
 808
 809         err = ice_fdir_program_hw_rx_queue(rxq);
 810         if (err) {
 811                 PMD_DRV_LOG(ERR, "fail to program FDIR RX queue %u",
 812                             rx_queue_id);
 813                 return -EIO;
 814         }
 815
 816         /* Init the RX tail register. */
 817         ICE_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
 818
 819         err = ice_switch_rx_queue(hw, rxq->reg_idx, true);
 820         if (err) {
 821                 PMD_DRV_LOG(ERR, "Failed to switch FDIR RX queue %u on",
 822                             rx_queue_id);
 823
 824                 ice_reset_rx_queue(rxq);
 825                 return -EINVAL;
 826         }
 827
 828         return 0;
 829 }
 830
 831 int
 832 ice_fdir_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 833 {
 834         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 835         struct ice_tx_queue *txq;
 836         int err;
 837         struct ice_vsi *vsi;
 838         struct ice_hw *hw;
 839         struct ice_aqc_add_tx_qgrp *txq_elem;
 840         struct ice_tlan_ctx tx_ctx;
 841         int buf_len;
 842
 843         PMD_INIT_FUNC_TRACE();
 844
 845         txq = pf->fdir.txq;
 846         if (!txq || !txq->q_set) {
 847                 PMD_DRV_LOG(ERR, "FDIR TX queue %u is not available or setup",
 848                             tx_queue_id);
 849                 return -EINVAL;
 850         }
 851
 852         buf_len = ice_struct_size(txq_elem, txqs, 1);
 853         txq_elem = ice_malloc(hw, buf_len);
 854         if (!txq_elem)
 855                 return -ENOMEM;
 856
 857         vsi = txq->vsi;
 858         hw = ICE_VSI_TO_HW(vsi);
 859
 860         memset(&tx_ctx, 0, sizeof(tx_ctx));
 861         txq_elem->num_txqs = 1;
 862         txq_elem->txqs[0].txq_id = rte_cpu_to_le_16(txq->reg_idx);
 863
 864         tx_ctx.base = txq->tx_ring_dma / ICE_QUEUE_BASE_ADDR_UNIT;
 865         tx_ctx.qlen = txq->nb_tx_desc;
 866         tx_ctx.pf_num = hw->pf_id;
 867         tx_ctx.vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_PF;
 868         tx_ctx.src_vsi = vsi->vsi_id;
 869         tx_ctx.port_num = hw->port_info->lport;
 870         tx_ctx.tso_ena = 1; /* tso enable */
 871         tx_ctx.tso_qnum = txq->reg_idx; /* index for tso state structure */
 872         tx_ctx.legacy_int = 1; /* Legacy or Advanced Host Interface */
 873
 874         ice_set_ctx(hw, (uint8_t *)&tx_ctx, txq_elem->txqs[0].txq_ctx,
 875                     ice_tlan_ctx_info);
 876
 877         txq->qtx_tail = hw->hw_addr + QTX_COMM_DBELL(txq->reg_idx);
 878
 879         /* Init the Tx tail register*/
 880         ICE_PCI_REG_WRITE(txq->qtx_tail, 0);
 881
 882         /* Fix me, we assume TC always 0 here */
 883         err = ice_ena_vsi_txq(hw->port_info, vsi->idx, 0, tx_queue_id, 1,
 884                               txq_elem, buf_len, NULL);
 885         if (err) {
 886                 PMD_DRV_LOG(ERR, "Failed to add FDIR txq");
 887                 rte_free(txq_elem);
 888                 return -EIO;
 889         }
 890         /* store the schedule node id */
 891         txq->q_teid = txq_elem->txqs[0].q_teid;
 892
 893         rte_free(txq_elem);
 894         return 0;
 895 }
 896
 897 /* Free all mbufs for descriptors in tx queue */
 898 static void
 899 _ice_tx_queue_release_mbufs(struct ice_tx_queue *txq)
 900 {
 901         uint16_t i;
 902
 903         if (!txq || !txq->sw_ring) {
 904                 PMD_DRV_LOG(DEBUG, "Pointer to txq or sw_ring is NULL");
 905                 return;
 906         }
 907
 908         for (i = 0; i < txq->nb_tx_desc; i++) {
 909                 if (txq->sw_ring[i].mbuf) {
 910                         rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
 911                         txq->sw_ring[i].mbuf = NULL;
 912                 }
 913         }
 914 }
 915
 916 static void
 917 ice_reset_tx_queue(struct ice_tx_queue *txq)
 918 {
 919         struct ice_tx_entry *txe;
 920         uint16_t i, prev, size;
 921
 922         if (!txq) {
 923                 PMD_DRV_LOG(DEBUG, "Pointer to txq is NULL");
 924                 return;
 925         }
 926
 927         txe = txq->sw_ring;
 928         size = sizeof(struct ice_tx_desc) * txq->nb_tx_desc;
 929         for (i = 0; i < size; i++)
 930                 ((volatile char *)txq->tx_ring)[i] = 0;
 931
 932         prev = (uint16_t)(txq->nb_tx_desc - 1);
 933         for (i = 0; i < txq->nb_tx_desc; i++) {
 934                 volatile struct ice_tx_desc *txd = &txq->tx_ring[i];
 935
 936                 txd->cmd_type_offset_bsz =
 937                         rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DESC_DONE);
 938                 txe[i].mbuf =  NULL;
 939                 txe[i].last_id = i;
 940                 txe[prev].next_id = i;
 941                 prev = i;
 942         }
 943
 944         txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
 945         txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
 946
 947         txq->tx_tail = 0;
 948         txq->nb_tx_used = 0;
 949
 950         txq->last_desc_cleaned = (uint16_t)(txq->nb_tx_desc - 1);
 951         txq->nb_tx_free = (uint16_t)(txq->nb_tx_desc - 1);
 952 }
 953
 954 int
 955 ice_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 956 {
 957         struct ice_tx_queue *txq;
 958         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 959         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 960         struct ice_vsi *vsi = pf->main_vsi;
 961         enum ice_status status;
 962         uint16_t q_ids[1];
 963         uint32_t q_teids[1];
 964         uint16_t q_handle = tx_queue_id;
 965
 966         if (tx_queue_id >= dev->data->nb_tx_queues) {
 967                 PMD_DRV_LOG(ERR, "TX queue %u is out of range %u",
 968                             tx_queue_id, dev->data->nb_tx_queues);
 969                 return -EINVAL;
 970         }
 971
 972         txq = dev->data->tx_queues[tx_queue_id];
 973         if (!txq) {
 974                 PMD_DRV_LOG(ERR, "TX queue %u is not available",
 975                             tx_queue_id);
 976                 return -EINVAL;
 977         }
 978
 979         q_ids[0] = txq->reg_idx;
 980         q_teids[0] = txq->q_teid;
 981
 982         /* Fix me, we assume TC always 0 here */
 983         status = ice_dis_vsi_txq(hw->port_info, vsi->idx, 0, 1, &q_handle,
 984                                 q_ids, q_teids, ICE_NO_RESET, 0, NULL);
 985         if (status != ICE_SUCCESS) {
 986                 PMD_DRV_LOG(DEBUG, "Failed to disable Lan Tx queue");
 987                 return -EINVAL;
 988         }
 989
 990         txq->tx_rel_mbufs(txq);
 991         ice_reset_tx_queue(txq);
 992         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 993
 994         return 0;
 995 }
 996
 997 int
 998 ice_fdir_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 999 {
1000         struct ice_rx_queue *rxq;
1001         int err;
1002         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1003         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1004
1005         rxq = pf->fdir.rxq;
1006
1007         err = ice_switch_rx_queue(hw, rxq->reg_idx, false);
1008         if (err) {
1009                 PMD_DRV_LOG(ERR, "Failed to switch FDIR RX queue %u off",
1010                             rx_queue_id);
1011                 return -EINVAL;
1012         }
1013         rxq->rx_rel_mbufs(rxq);
1014
1015         return 0;
1016 }
1017
1018 int
1019 ice_fdir_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
1020 {
1021         struct ice_tx_queue *txq;
1022         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1023         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1024         struct ice_vsi *vsi = pf->main_vsi;
1025         enum ice_status status;
1026         uint16_t q_ids[1];
1027         uint32_t q_teids[1];
1028         uint16_t q_handle = tx_queue_id;
1029
1030         txq = pf->fdir.txq;
1031         if (!txq) {
1032                 PMD_DRV_LOG(ERR, "TX queue %u is not available",
1033                             tx_queue_id);
1034                 return -EINVAL;
1035         }
1036         vsi = txq->vsi;
1037
1038         q_ids[0] = txq->reg_idx;
1039         q_teids[0] = txq->q_teid;
1040
1041         /* Fix me, we assume TC always 0 here */
1042         status = ice_dis_vsi_txq(hw->port_info, vsi->idx, 0, 1, &q_handle,
1043                                  q_ids, q_teids, ICE_NO_RESET, 0, NULL);
1044         if (status != ICE_SUCCESS) {
1045                 PMD_DRV_LOG(DEBUG, "Failed to disable Lan Tx queue");
1046                 return -EINVAL;
1047         }
1048
1049         txq->tx_rel_mbufs(txq);
1050
1051         return 0;
1052 }
1053
1054 int
1055 ice_rx_queue_setup(struct rte_eth_dev *dev,
1056                    uint16_t queue_idx,
1057                    uint16_t nb_desc,
1058                    unsigned int socket_id,
1059                    const struct rte_eth_rxconf *rx_conf,
1060                    struct rte_mempool *mp)
1061 {
1062         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1063         struct ice_adapter *ad =
1064                 ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
1065         struct ice_vsi *vsi = pf->main_vsi;
1066         struct ice_rx_queue *rxq;
1067         const struct rte_memzone *rz;
1068         uint32_t ring_size;
1069         uint16_t len;
1070         int use_def_burst_func = 1;
1071         uint64_t offloads;
1072
1073         if (nb_desc % ICE_ALIGN_RING_DESC != 0 ||
1074             nb_desc > ICE_MAX_RING_DESC ||
1075             nb_desc < ICE_MIN_RING_DESC) {
1076                 PMD_INIT_LOG(ERR, "Number (%u) of receive descriptors is "
1077                              "invalid", nb_desc);
1078                 return -EINVAL;
1079         }
1080
1081         offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads;
1082
1083         /* Free memory if needed */
1084         if (dev->data->rx_queues[queue_idx]) {
1085                 ice_rx_queue_release(dev->data->rx_queues[queue_idx]);
1086                 dev->data->rx_queues[queue_idx] = NULL;
1087         }
1088
1089         /* Allocate the rx queue data structure */
1090         rxq = rte_zmalloc_socket(NULL,
1091                                  sizeof(struct ice_rx_queue),
1092                                  RTE_CACHE_LINE_SIZE,
1093                                  socket_id);
1094         if (!rxq) {
1095                 PMD_INIT_LOG(ERR, "Failed to allocate memory for "
1096                              "rx queue data structure");
1097                 return -ENOMEM;
1098         }
1099         rxq->mp = mp;
1100         rxq->nb_rx_desc = nb_desc;
1101         rxq->rx_free_thresh = rx_conf->rx_free_thresh;
1102         rxq->queue_id = queue_idx;
1103         rxq->offloads = offloads;
1104
1105         rxq->reg_idx = vsi->base_queue + queue_idx;
1106         rxq->port_id = dev->data->port_id;
1107         if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC)
1108                 rxq->crc_len = RTE_ETHER_CRC_LEN;
1109         else
1110                 rxq->crc_len = 0;
1111
1112         rxq->drop_en = rx_conf->rx_drop_en;
1113         rxq->vsi = vsi;
1114         rxq->rx_deferred_start = rx_conf->rx_deferred_start;
1115         rxq->proto_xtr = pf->proto_xtr != NULL ?
1116                          pf->proto_xtr[queue_idx] : PROTO_XTR_NONE;
1117
1118         /* Allocate the maximun number of RX ring hardware descriptor. */
1119         len = ICE_MAX_RING_DESC;
1120
1121         /**
1122          * Allocating a little more memory because vectorized/bulk_alloc Rx
1123          * functions doesn't check boundaries each time.
1124          */
1125         len += ICE_RX_MAX_BURST;
1126
1127         /* Allocate the maximum number of RX ring hardware descriptor. */
1128         ring_size = sizeof(union ice_rx_flex_desc) * len;
1129         ring_size = RTE_ALIGN(ring_size, ICE_DMA_MEM_ALIGN);
1130         rz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx,
1131                                       ring_size, ICE_RING_BASE_ALIGN,
1132                                       socket_id);
1133         if (!rz) {
1134                 ice_rx_queue_release(rxq);
1135                 PMD_INIT_LOG(ERR, "Failed to reserve DMA memory for RX");
1136                 return -ENOMEM;
1137         }
1138
1139         rxq->mz = rz;
1140         /* Zero all the descriptors in the ring. */
1141         memset(rz->addr, 0, ring_size);
1142
1143         rxq->rx_ring_dma = rz->iova;
1144         rxq->rx_ring = rz->addr;
1145
1146         /* always reserve more for bulk alloc */
1147         len = (uint16_t)(nb_desc + ICE_RX_MAX_BURST);
1148
1149         /* Allocate the software ring. */
1150         rxq->sw_ring = rte_zmalloc_socket(NULL,
1151                                           sizeof(struct ice_rx_entry) * len,
1152                                           RTE_CACHE_LINE_SIZE,
1153                                           socket_id);
1154         if (!rxq->sw_ring) {
1155                 ice_rx_queue_release(rxq);
1156                 PMD_INIT_LOG(ERR, "Failed to allocate memory for SW ring");
1157                 return -ENOMEM;
1158         }
1159
1160         ice_reset_rx_queue(rxq);
1161         rxq->q_set = true;
1162         dev->data->rx_queues[queue_idx] = rxq;
1163         rxq->rx_rel_mbufs = _ice_rx_queue_release_mbufs;
1164
1165         use_def_burst_func = ice_check_rx_burst_bulk_alloc_preconditions(rxq);
1166
1167         if (!use_def_burst_func) {
1168                 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are "
1169                              "satisfied. Rx Burst Bulk Alloc function will be "
1170                              "used on port=%d, queue=%d.",
1171                              rxq->port_id, rxq->queue_id);
1172         } else {
1173                 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are "
1174                              "not satisfied, Scattered Rx is requested. "
1175                              "on port=%d, queue=%d.",
1176                              rxq->port_id, rxq->queue_id);
1177                 ad->rx_bulk_alloc_allowed = false;
1178         }
1179
1180         return 0;
1181 }
1182
1183 void
1184 ice_rx_queue_release(void *rxq)
1185 {
1186         struct ice_rx_queue *q = (struct ice_rx_queue *)rxq;
1187
1188         if (!q) {
1189                 PMD_DRV_LOG(DEBUG, "Pointer to rxq is NULL");
1190                 return;
1191         }
1192
1193         q->rx_rel_mbufs(q);
1194         rte_free(q->sw_ring);
1195         rte_memzone_free(q->mz);
1196         rte_free(q);
1197 }
1198
1199 int
1200 ice_tx_queue_setup(struct rte_eth_dev *dev,
1201                    uint16_t queue_idx,
1202                    uint16_t nb_desc,
1203                    unsigned int socket_id,
1204                    const struct rte_eth_txconf *tx_conf)
1205 {
1206         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1207         struct ice_vsi *vsi = pf->main_vsi;
1208         struct ice_tx_queue *txq;
1209         const struct rte_memzone *tz;
1210         uint32_t ring_size;
1211         uint16_t tx_rs_thresh, tx_free_thresh;
1212         uint64_t offloads;
1213
1214         offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads;
1215
1216         if (nb_desc % ICE_ALIGN_RING_DESC != 0 ||
1217             nb_desc > ICE_MAX_RING_DESC ||
1218             nb_desc < ICE_MIN_RING_DESC) {
1219                 PMD_INIT_LOG(ERR, "Number (%u) of transmit descriptors is "
1220                              "invalid", nb_desc);
1221                 return -EINVAL;
1222         }
1223
1224         /**
1225          * The following two parameters control the setting of the RS bit on
1226          * transmit descriptors. TX descriptors will have their RS bit set
1227          * after txq->tx_rs_thresh descriptors have been used. The TX
1228          * descriptor ring will be cleaned after txq->tx_free_thresh
1229          * descriptors are used or if the number of descriptors required to
1230          * transmit a packet is greater than the number of free TX descriptors.
1231          *
1232          * The following constraints must be satisfied:
1233          *  - tx_rs_thresh must be greater than 0.
1234          *  - tx_rs_thresh must be less than the size of the ring minus 2.
1235          *  - tx_rs_thresh must be less than or equal to tx_free_thresh.
1236          *  - tx_rs_thresh must be a divisor of the ring size.
1237          *  - tx_free_thresh must be greater than 0.
1238          *  - tx_free_thresh must be less than the size of the ring minus 3.
1239          *  - tx_free_thresh + tx_rs_thresh must not exceed nb_desc.
1240          *
1241          * One descriptor in the TX ring is used as a sentinel to avoid a H/W
1242          * race condition, hence the maximum threshold constraints. When set
1243          * to zero use default values.
1244          */
1245         tx_free_thresh = (uint16_t)(tx_conf->tx_free_thresh ?
1246                                     tx_conf->tx_free_thresh :
1247                                     ICE_DEFAULT_TX_FREE_THRESH);
1248         /* force tx_rs_thresh to adapt an aggresive tx_free_thresh */
1249         tx_rs_thresh =
1250                 (ICE_DEFAULT_TX_RSBIT_THRESH + tx_free_thresh > nb_desc) ?
1251                         nb_desc - tx_free_thresh : ICE_DEFAULT_TX_RSBIT_THRESH;
1252         if (tx_conf->tx_rs_thresh)
1253                 tx_rs_thresh = tx_conf->tx_rs_thresh;
1254         if (tx_rs_thresh + tx_free_thresh > nb_desc) {
1255                 PMD_INIT_LOG(ERR, "tx_rs_thresh + tx_free_thresh must not "
1256                                 "exceed nb_desc. (tx_rs_thresh=%u "
1257                                 "tx_free_thresh=%u nb_desc=%u port = %d queue=%d)",
1258                                 (unsigned int)tx_rs_thresh,
1259                                 (unsigned int)tx_free_thresh,
1260                                 (unsigned int)nb_desc,
1261                                 (int)dev->data->port_id,
1262                                 (int)queue_idx);
1263                 return -EINVAL;
1264         }
1265         if (tx_rs_thresh >= (nb_desc - 2)) {
1266                 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than the "
1267                              "number of TX descriptors minus 2. "
1268                              "(tx_rs_thresh=%u port=%d queue=%d)",
1269                              (unsigned int)tx_rs_thresh,
1270                              (int)dev->data->port_id,
1271                              (int)queue_idx);
1272                 return -EINVAL;
1273         }
1274         if (tx_free_thresh >= (nb_desc - 3)) {
1275                 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than the "
1276                              "tx_free_thresh must be less than the "
1277                              "number of TX descriptors minus 3. "
1278                              "(tx_free_thresh=%u port=%d queue=%d)",
1279                              (unsigned int)tx_free_thresh,
1280                              (int)dev->data->port_id,
1281                              (int)queue_idx);
1282                 return -EINVAL;
1283         }
1284         if (tx_rs_thresh > tx_free_thresh) {
1285                 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than or "
1286                              "equal to tx_free_thresh. (tx_free_thresh=%u"
1287                              " tx_rs_thresh=%u port=%d queue=%d)",
1288                              (unsigned int)tx_free_thresh,
1289                              (unsigned int)tx_rs_thresh,
1290                              (int)dev->data->port_id,
1291                              (int)queue_idx);
1292                 return -EINVAL;
1293         }
1294         if ((nb_desc % tx_rs_thresh) != 0) {
1295                 PMD_INIT_LOG(ERR, "tx_rs_thresh must be a divisor of the "
1296                              "number of TX descriptors. (tx_rs_thresh=%u"
1297                              " port=%d queue=%d)",
1298                              (unsigned int)tx_rs_thresh,
1299                              (int)dev->data->port_id,
1300                              (int)queue_idx);
1301                 return -EINVAL;
1302         }
1303         if (tx_rs_thresh > 1 && tx_conf->tx_thresh.wthresh != 0) {
1304                 PMD_INIT_LOG(ERR, "TX WTHRESH must be set to 0 if "
1305                              "tx_rs_thresh is greater than 1. "
1306                              "(tx_rs_thresh=%u port=%d queue=%d)",
1307                              (unsigned int)tx_rs_thresh,
1308                              (int)dev->data->port_id,
1309                              (int)queue_idx);
1310                 return -EINVAL;
1311         }
1312
1313         /* Free memory if needed. */
1314         if (dev->data->tx_queues[queue_idx]) {
1315                 ice_tx_queue_release(dev->data->tx_queues[queue_idx]);
1316                 dev->data->tx_queues[queue_idx] = NULL;
1317         }
1318
1319         /* Allocate the TX queue data structure. */
1320         txq = rte_zmalloc_socket(NULL,
1321                                  sizeof(struct ice_tx_queue),
1322                                  RTE_CACHE_LINE_SIZE,
1323                                  socket_id);
1324         if (!txq) {
1325                 PMD_INIT_LOG(ERR, "Failed to allocate memory for "
1326                              "tx queue structure");
1327                 return -ENOMEM;
1328         }
1329
1330         /* Allocate TX hardware ring descriptors. */
1331         ring_size = sizeof(struct ice_tx_desc) * ICE_MAX_RING_DESC;
1332         ring_size = RTE_ALIGN(ring_size, ICE_DMA_MEM_ALIGN);
1333         tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx,
1334                                       ring_size, ICE_RING_BASE_ALIGN,
1335                                       socket_id);
1336         if (!tz) {
1337                 ice_tx_queue_release(txq);
1338                 PMD_INIT_LOG(ERR, "Failed to reserve DMA memory for TX");
1339                 return -ENOMEM;
1340         }
1341
1342         txq->mz = tz;
1343         txq->nb_tx_desc = nb_desc;
1344         txq->tx_rs_thresh = tx_rs_thresh;
1345         txq->tx_free_thresh = tx_free_thresh;
1346         txq->pthresh = tx_conf->tx_thresh.pthresh;
1347         txq->hthresh = tx_conf->tx_thresh.hthresh;
1348         txq->wthresh = tx_conf->tx_thresh.wthresh;
1349         txq->queue_id = queue_idx;
1350
1351         txq->reg_idx = vsi->base_queue + queue_idx;
1352         txq->port_id = dev->data->port_id;
1353         txq->offloads = offloads;
1354         txq->vsi = vsi;
1355         txq->tx_deferred_start = tx_conf->tx_deferred_start;
1356
1357         txq->tx_ring_dma = tz->iova;
1358         txq->tx_ring = tz->addr;
1359
1360         /* Allocate software ring */
1361         txq->sw_ring =
1362                 rte_zmalloc_socket(NULL,
1363                                    sizeof(struct ice_tx_entry) * nb_desc,
1364                                    RTE_CACHE_LINE_SIZE,
1365                                    socket_id);
1366         if (!txq->sw_ring) {
1367                 ice_tx_queue_release(txq);
1368                 PMD_INIT_LOG(ERR, "Failed to allocate memory for SW TX ring");
1369                 return -ENOMEM;
1370         }
1371
1372         ice_reset_tx_queue(txq);
1373         txq->q_set = true;
1374         dev->data->tx_queues[queue_idx] = txq;
1375         txq->tx_rel_mbufs = _ice_tx_queue_release_mbufs;
1376         ice_set_tx_function_flag(dev, txq);
1377
1378         return 0;
1379 }
1380
1381 void
1382 ice_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
1383 {
1384         ice_rx_queue_release(dev->data->rx_queues[qid]);
1385 }
1386
1387 void
1388 ice_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
1389 {
1390         ice_tx_queue_release(dev->data->tx_queues[qid]);
1391 }
1392
1393 void
1394 ice_tx_queue_release(void *txq)
1395 {
1396         struct ice_tx_queue *q = (struct ice_tx_queue *)txq;
1397
1398         if (!q) {
1399                 PMD_DRV_LOG(DEBUG, "Pointer to TX queue is NULL");
1400                 return;
1401         }
1402
1403         q->tx_rel_mbufs(q);
1404         rte_free(q->sw_ring);
1405         rte_memzone_free(q->mz);
1406         rte_free(q);
1407 }
1408
1409 void
1410 ice_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
1411                  struct rte_eth_rxq_info *qinfo)
1412 {
1413         struct ice_rx_queue *rxq;
1414
1415         rxq = dev->data->rx_queues[queue_id];
1416
1417         qinfo->mp = rxq->mp;
1418         qinfo->scattered_rx = dev->data->scattered_rx;
1419         qinfo->nb_desc = rxq->nb_rx_desc;
1420
1421         qinfo->conf.rx_free_thresh = rxq->rx_free_thresh;
1422         qinfo->conf.rx_drop_en = rxq->drop_en;
1423         qinfo->conf.rx_deferred_start = rxq->rx_deferred_start;
1424 }
1425
1426 void
1427 ice_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
1428                  struct rte_eth_txq_info *qinfo)
1429 {
1430         struct ice_tx_queue *txq;
1431
1432         txq = dev->data->tx_queues[queue_id];
1433
1434         qinfo->nb_desc = txq->nb_tx_desc;
1435
1436         qinfo->conf.tx_thresh.pthresh = txq->pthresh;
1437         qinfo->conf.tx_thresh.hthresh = txq->hthresh;
1438         qinfo->conf.tx_thresh.wthresh = txq->wthresh;
1439
1440         qinfo->conf.tx_free_thresh = txq->tx_free_thresh;
1441         qinfo->conf.tx_rs_thresh = txq->tx_rs_thresh;
1442         qinfo->conf.offloads = txq->offloads;
1443         qinfo->conf.tx_deferred_start = txq->tx_deferred_start;
1444 }
1445
1446 uint32_t
1447 ice_rx_queue_count(void *rx_queue)
1448 {
1449 #define ICE_RXQ_SCAN_INTERVAL 4
1450         volatile union ice_rx_flex_desc *rxdp;
1451         struct ice_rx_queue *rxq;
1452         uint16_t desc = 0;
1453
1454         rxq = rx_queue;
1455         rxdp = &rxq->rx_ring[rxq->rx_tail];
1456         while ((desc < rxq->nb_rx_desc) &&
1457                rte_le_to_cpu_16(rxdp->wb.status_error0) &
1458                (1 << ICE_RX_FLEX_DESC_STATUS0_DD_S)) {
1459                 /**
1460                  * Check the DD bit of a rx descriptor of each 4 in a group,
1461                  * to avoid checking too frequently and downgrading performance
1462                  * too much.
1463                  */
1464                 desc += ICE_RXQ_SCAN_INTERVAL;
1465                 rxdp += ICE_RXQ_SCAN_INTERVAL;
1466                 if (rxq->rx_tail + desc >= rxq->nb_rx_desc)
1467                         rxdp = &(rxq->rx_ring[rxq->rx_tail +
1468                                  desc - rxq->nb_rx_desc]);
1469         }
1470
1471         return desc;
1472 }
1473
1474 #define ICE_RX_FLEX_ERR0_BITS   \
1475         ((1 << ICE_RX_FLEX_DESC_STATUS0_HBO_S) |        \
1476          (1 << ICE_RX_FLEX_DESC_STATUS0_XSUM_IPE_S) |   \
1477          (1 << ICE_RX_FLEX_DESC_STATUS0_XSUM_L4E_S) |   \
1478          (1 << ICE_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S) |  \
1479          (1 << ICE_RX_FLEX_DESC_STATUS0_XSUM_EUDPE_S) | \
1480          (1 << ICE_RX_FLEX_DESC_STATUS0_RXE_S))
1481
1482 /* Rx L3/L4 checksum */
1483 static inline uint64_t
1484 ice_rxd_error_to_pkt_flags(uint16_t stat_err0)
1485 {
1486         uint64_t flags = 0;
1487
1488         /* check if HW has decoded the packet and checksum */
1489         if (unlikely(!(stat_err0 & (1 << ICE_RX_FLEX_DESC_STATUS0_L3L4P_S))))
1490                 return 0;
1491
1492         if (likely(!(stat_err0 & ICE_RX_FLEX_ERR0_BITS))) {
1493                 flags |= (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD);
1494                 return flags;
1495         }
1496
1497         if (unlikely(stat_err0 & (1 << ICE_RX_FLEX_DESC_STATUS0_XSUM_IPE_S)))
1498                 flags |= PKT_RX_IP_CKSUM_BAD;
1499         else
1500                 flags |= PKT_RX_IP_CKSUM_GOOD;
1501
1502         if (unlikely(stat_err0 & (1 << ICE_RX_FLEX_DESC_STATUS0_XSUM_L4E_S)))
1503                 flags |= PKT_RX_L4_CKSUM_BAD;
1504         else
1505                 flags |= PKT_RX_L4_CKSUM_GOOD;
1506
1507         if (unlikely(stat_err0 & (1 << ICE_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S)))
1508                 flags |= PKT_RX_OUTER_IP_CKSUM_BAD;
1509
1510         if (unlikely(stat_err0 & (1 << ICE_RX_FLEX_DESC_STATUS0_XSUM_EUDPE_S)))
1511                 flags |= PKT_RX_OUTER_L4_CKSUM_BAD;
1512         else
1513                 flags |= PKT_RX_OUTER_L4_CKSUM_GOOD;
1514
1515         return flags;
1516 }
1517
1518 static inline void
1519 ice_rxd_to_vlan_tci(struct rte_mbuf *mb, volatile union ice_rx_flex_desc *rxdp)
1520 {
1521         if (rte_le_to_cpu_16(rxdp->wb.status_error0) &
1522             (1 << ICE_RX_FLEX_DESC_STATUS0_L2TAG1P_S)) {
1523                 mb->ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED;
1524                 mb->vlan_tci =
1525                         rte_le_to_cpu_16(rxdp->wb.l2tag1);
1526                 PMD_RX_LOG(DEBUG, "Descriptor l2tag1: %u",
1527                            rte_le_to_cpu_16(rxdp->wb.l2tag1));
1528         } else {
1529                 mb->vlan_tci = 0;
1530         }
1531
1532 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
1533         if (rte_le_to_cpu_16(rxdp->wb.status_error1) &
1534             (1 << ICE_RX_FLEX_DESC_STATUS1_L2TAG2P_S)) {
1535                 mb->ol_flags |= PKT_RX_QINQ_STRIPPED | PKT_RX_QINQ |
1536                                 PKT_RX_VLAN_STRIPPED | PKT_RX_VLAN;
1537                 mb->vlan_tci_outer = mb->vlan_tci;
1538                 mb->vlan_tci = rte_le_to_cpu_16(rxdp->wb.l2tag2_2nd);
1539                 PMD_RX_LOG(DEBUG, "Descriptor l2tag2_1: %u, l2tag2_2: %u",
1540                            rte_le_to_cpu_16(rxdp->wb.l2tag2_1st),
1541                            rte_le_to_cpu_16(rxdp->wb.l2tag2_2nd));
1542         } else {
1543                 mb->vlan_tci_outer = 0;
1544         }
1545 #endif
1546         PMD_RX_LOG(DEBUG, "Mbuf vlan_tci: %u, vlan_tci_outer: %u",
1547                    mb->vlan_tci, mb->vlan_tci_outer);
1548 }
1549
1550 #define ICE_LOOK_AHEAD 8
1551 #if (ICE_LOOK_AHEAD != 8)
1552 #error "PMD ICE: ICE_LOOK_AHEAD must be 8\n"
1553 #endif
1554 static inline int
1555 ice_rx_scan_hw_ring(struct ice_rx_queue *rxq)
1556 {
1557         volatile union ice_rx_flex_desc *rxdp;
1558         struct ice_rx_entry *rxep;
1559         struct rte_mbuf *mb;
1560         uint16_t stat_err0;
1561         uint16_t pkt_len;
1562         int32_t s[ICE_LOOK_AHEAD], nb_dd;
1563         int32_t i, j, nb_rx = 0;
1564         uint64_t pkt_flags = 0;
1565         uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl;
1566 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
1567         struct ice_vsi *vsi = rxq->vsi;
1568         struct ice_hw *hw = ICE_VSI_TO_HW(vsi);
1569         uint64_t ts_ns;
1570         struct ice_adapter *ad = rxq->vsi->adapter;
1571 #endif
1572         rxdp = &rxq->rx_ring[rxq->rx_tail];
1573         rxep = &rxq->sw_ring[rxq->rx_tail];
1574
1575         stat_err0 = rte_le_to_cpu_16(rxdp->wb.status_error0);
1576
1577         /* Make sure there is at least 1 packet to receive */
1578         if (!(stat_err0 & (1 << ICE_RX_FLEX_DESC_STATUS0_DD_S)))
1579                 return 0;
1580
1581         /**
1582          * Scan LOOK_AHEAD descriptors at a time to determine which
1583          * descriptors reference packets that are ready to be received.
1584          */
1585         for (i = 0; i < ICE_RX_MAX_BURST; i += ICE_LOOK_AHEAD,
1586              rxdp += ICE_LOOK_AHEAD, rxep += ICE_LOOK_AHEAD) {
1587                 /* Read desc statuses backwards to avoid race condition */
1588                 for (j = ICE_LOOK_AHEAD - 1; j >= 0; j--)
1589                         s[j] = rte_le_to_cpu_16(rxdp[j].wb.status_error0);
1590
1591                 rte_smp_rmb();
1592
1593                 /* Compute how many status bits were set */
1594                 for (j = 0, nb_dd = 0; j < ICE_LOOK_AHEAD; j++)
1595                         nb_dd += s[j] & (1 << ICE_RX_FLEX_DESC_STATUS0_DD_S);
1596
1597                 nb_rx += nb_dd;
1598
1599                 /* Translate descriptor info to mbuf parameters */
1600                 for (j = 0; j < nb_dd; j++) {
1601                         mb = rxep[j].mbuf;
1602                         pkt_len = (rte_le_to_cpu_16(rxdp[j].wb.pkt_len) &
1603                                    ICE_RX_FLX_DESC_PKT_LEN_M) - rxq->crc_len;
1604                         mb->data_len = pkt_len;
1605                         mb->pkt_len = pkt_len;
1606                         mb->ol_flags = 0;
1607                         stat_err0 = rte_le_to_cpu_16(rxdp[j].wb.status_error0);
1608                         pkt_flags = ice_rxd_error_to_pkt_flags(stat_err0);
1609                         mb->packet_type = ptype_tbl[ICE_RX_FLEX_DESC_PTYPE_M &
1610                                 rte_le_to_cpu_16(rxdp[j].wb.ptype_flex_flags0)];
1611                         ice_rxd_to_vlan_tci(mb, &rxdp[j]);
1612                         rxq->rxd_to_pkt_fields(rxq, mb, &rxdp[j]);
1613 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
1614                         if (rxq->offloads & DEV_RX_OFFLOAD_TIMESTAMP) {
1615                                 ts_ns = ice_tstamp_convert_32b_64b(hw,
1616                                         rte_le_to_cpu_32(rxdp[j].wb.flex_ts.ts_high));
1617                                 if (ice_timestamp_dynflag > 0) {
1618                                         *RTE_MBUF_DYNFIELD(mb,
1619                                                 ice_timestamp_dynfield_offset,
1620                                                 rte_mbuf_timestamp_t *) = ts_ns;
1621                                         mb->ol_flags |= ice_timestamp_dynflag;
1622                                 }
1623                         }
1624
1625                         if (ad->ptp_ena && ((mb->packet_type &
1626                             RTE_PTYPE_L2_MASK) == RTE_PTYPE_L2_ETHER_TIMESYNC)) {
1627                                 rxq->time_high =
1628                                    rte_le_to_cpu_32(rxdp[j].wb.flex_ts.ts_high);
1629                                 mb->timesync = rxq->queue_id;
1630                                 pkt_flags |= PKT_RX_IEEE1588_PTP;
1631                         }
1632 #endif
1633                         mb->ol_flags |= pkt_flags;
1634                 }
1635
1636                 for (j = 0; j < ICE_LOOK_AHEAD; j++)
1637                         rxq->rx_stage[i + j] = rxep[j].mbuf;
1638
1639                 if (nb_dd != ICE_LOOK_AHEAD)
1640                         break;
1641         }
1642
1643         /* Clear software ring entries */
1644         for (i = 0; i < nb_rx; i++)
1645                 rxq->sw_ring[rxq->rx_tail + i].mbuf = NULL;
1646
1647         PMD_RX_LOG(DEBUG, "ice_rx_scan_hw_ring: "
1648                    "port_id=%u, queue_id=%u, nb_rx=%d",
1649                    rxq->port_id, rxq->queue_id, nb_rx);
1650
1651         return nb_rx;
1652 }
1653
1654 static inline uint16_t
1655 ice_rx_fill_from_stage(struct ice_rx_queue *rxq,
1656                        struct rte_mbuf **rx_pkts,
1657                        uint16_t nb_pkts)
1658 {
1659         uint16_t i;
1660         struct rte_mbuf **stage = &rxq->rx_stage[rxq->rx_next_avail];
1661
1662         nb_pkts = (uint16_t)RTE_MIN(nb_pkts, rxq->rx_nb_avail);
1663
1664         for (i = 0; i < nb_pkts; i++)
1665                 rx_pkts[i] = stage[i];
1666
1667         rxq->rx_nb_avail = (uint16_t)(rxq->rx_nb_avail - nb_pkts);
1668         rxq->rx_next_avail = (uint16_t)(rxq->rx_next_avail + nb_pkts);
1669
1670         return nb_pkts;
1671 }
1672
1673 static inline int
1674 ice_rx_alloc_bufs(struct ice_rx_queue *rxq)
1675 {
1676         volatile union ice_rx_flex_desc *rxdp;
1677         struct ice_rx_entry *rxep;
1678         struct rte_mbuf *mb;
1679         uint16_t alloc_idx, i;
1680         uint64_t dma_addr;
1681         int diag;
1682
1683         /* Allocate buffers in bulk */
1684         alloc_idx = (uint16_t)(rxq->rx_free_trigger -
1685                                (rxq->rx_free_thresh - 1));
1686         rxep = &rxq->sw_ring[alloc_idx];
1687         diag = rte_mempool_get_bulk(rxq->mp, (void *)rxep,
1688                                     rxq->rx_free_thresh);
1689         if (unlikely(diag != 0)) {
1690                 PMD_RX_LOG(ERR, "Failed to get mbufs in bulk");
1691                 return -ENOMEM;
1692         }
1693
1694         rxdp = &rxq->rx_ring[alloc_idx];
1695         for (i = 0; i < rxq->rx_free_thresh; i++) {
1696                 if (likely(i < (rxq->rx_free_thresh - 1)))
1697                         /* Prefetch next mbuf */
1698                         rte_prefetch0(rxep[i + 1].mbuf);
1699
1700                 mb = rxep[i].mbuf;
1701                 rte_mbuf_refcnt_set(mb, 1);
1702                 mb->next = NULL;
1703                 mb->data_off = RTE_PKTMBUF_HEADROOM;
1704                 mb->nb_segs = 1;
1705                 mb->port = rxq->port_id;
1706                 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mb));
1707                 rxdp[i].read.hdr_addr = 0;
1708                 rxdp[i].read.pkt_addr = dma_addr;
1709         }
1710
1711         /* Update rx tail regsiter */
1712         ICE_PCI_REG_WRITE(rxq->qrx_tail, rxq->rx_free_trigger);
1713
1714         rxq->rx_free_trigger =
1715                 (uint16_t)(rxq->rx_free_trigger + rxq->rx_free_thresh);
1716         if (rxq->rx_free_trigger >= rxq->nb_rx_desc)
1717                 rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1);
1718
1719         return 0;
1720 }
1721
1722 static inline uint16_t
1723 rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
1724 {
1725         struct ice_rx_queue *rxq = (struct ice_rx_queue *)rx_queue;
1726         uint16_t nb_rx = 0;
1727
1728         if (!nb_pkts)
1729                 return 0;
1730
1731         if (rxq->rx_nb_avail)
1732                 return ice_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
1733
1734         nb_rx = (uint16_t)ice_rx_scan_hw_ring(rxq);
1735         rxq->rx_next_avail = 0;
1736         rxq->rx_nb_avail = nb_rx;
1737         rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_rx);
1738
1739         if (rxq->rx_tail > rxq->rx_free_trigger) {
1740                 if (ice_rx_alloc_bufs(rxq) != 0) {
1741                         uint16_t i, j;
1742
1743                         rxq->vsi->adapter->pf.dev_data->rx_mbuf_alloc_failed +=
1744                                 rxq->rx_free_thresh;
1745                         PMD_RX_LOG(DEBUG, "Rx mbuf alloc failed for "
1746                                    "port_id=%u, queue_id=%u",
1747                                    rxq->port_id, rxq->queue_id);
1748                         rxq->rx_nb_avail = 0;
1749                         rxq->rx_tail = (uint16_t)(rxq->rx_tail - nb_rx);
1750                         for (i = 0, j = rxq->rx_tail; i < nb_rx; i++, j++)
1751                                 rxq->sw_ring[j].mbuf = rxq->rx_stage[i];
1752
1753                         return 0;
1754                 }
1755         }
1756
1757         if (rxq->rx_tail >= rxq->nb_rx_desc)
1758                 rxq->rx_tail = 0;
1759
1760         if (rxq->rx_nb_avail)
1761                 return ice_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
1762
1763         return 0;
1764 }
1765
1766 static uint16_t
1767 ice_recv_pkts_bulk_alloc(void *rx_queue,
1768                          struct rte_mbuf **rx_pkts,
1769                          uint16_t nb_pkts)
1770 {
1771         uint16_t nb_rx = 0;
1772         uint16_t n;
1773         uint16_t count;
1774
1775         if (unlikely(nb_pkts == 0))
1776                 return nb_rx;
1777
1778         if (likely(nb_pkts <= ICE_RX_MAX_BURST))
1779                 return rx_recv_pkts(rx_queue, rx_pkts, nb_pkts);
1780
1781         while (nb_pkts) {
1782                 n = RTE_MIN(nb_pkts, ICE_RX_MAX_BURST);
1783                 count = rx_recv_pkts(rx_queue, &rx_pkts[nb_rx], n);
1784                 nb_rx = (uint16_t)(nb_rx + count);
1785                 nb_pkts = (uint16_t)(nb_pkts - count);
1786                 if (count < n)
1787                         break;
1788         }
1789
1790         return nb_rx;
1791 }
1792
1793 static uint16_t
1794 ice_recv_scattered_pkts(void *rx_queue,
1795                         struct rte_mbuf **rx_pkts,
1796                         uint16_t nb_pkts)
1797 {
1798         struct ice_rx_queue *rxq = rx_queue;
1799         volatile union ice_rx_flex_desc *rx_ring = rxq->rx_ring;
1800         volatile union ice_rx_flex_desc *rxdp;
1801         union ice_rx_flex_desc rxd;
1802         struct ice_rx_entry *sw_ring = rxq->sw_ring;
1803         struct ice_rx_entry *rxe;
1804         struct rte_mbuf *first_seg = rxq->pkt_first_seg;
1805         struct rte_mbuf *last_seg = rxq->pkt_last_seg;
1806         struct rte_mbuf *nmb; /* new allocated mbuf */
1807         struct rte_mbuf *rxm; /* pointer to store old mbuf in SW ring */
1808         uint16_t rx_id = rxq->rx_tail;
1809         uint16_t nb_rx = 0;
1810         uint16_t nb_hold = 0;
1811         uint16_t rx_packet_len;
1812         uint16_t rx_stat_err0;
1813         uint64_t dma_addr;
1814         uint64_t pkt_flags;
1815         uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl;
1816 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
1817         struct ice_vsi *vsi = rxq->vsi;
1818         struct ice_hw *hw = ICE_VSI_TO_HW(vsi);
1819         uint64_t ts_ns;
1820         struct ice_adapter *ad = rxq->vsi->adapter;
1821 #endif
1822         while (nb_rx < nb_pkts) {
1823                 rxdp = &rx_ring[rx_id];
1824                 rx_stat_err0 = rte_le_to_cpu_16(rxdp->wb.status_error0);
1825
1826                 /* Check the DD bit first */
1827                 if (!(rx_stat_err0 & (1 << ICE_RX_FLEX_DESC_STATUS0_DD_S)))
1828                         break;
1829
1830                 /* allocate mbuf */
1831                 nmb = rte_mbuf_raw_alloc(rxq->mp);
1832                 if (unlikely(!nmb)) {
1833                         rxq->vsi->adapter->pf.dev_data->rx_mbuf_alloc_failed++;
1834                         break;
1835                 }
1836                 rxd = *rxdp; /* copy descriptor in ring to temp variable*/
1837
1838                 nb_hold++;
1839                 rxe = &sw_ring[rx_id]; /* get corresponding mbuf in SW ring */
1840                 rx_id++;
1841                 if (unlikely(rx_id == rxq->nb_rx_desc))
1842                         rx_id = 0;
1843
1844                 /* Prefetch next mbuf */
1845                 rte_prefetch0(sw_ring[rx_id].mbuf);
1846
1847                 /**
1848                  * When next RX descriptor is on a cache line boundary,
1849                  * prefetch the next 4 RX descriptors and next 8 pointers
1850                  * to mbufs.
1851                  */
1852                 if ((rx_id & 0x3) == 0) {
1853                         rte_prefetch0(&rx_ring[rx_id]);
1854                         rte_prefetch0(&sw_ring[rx_id]);
1855                 }
1856
1857                 rxm = rxe->mbuf;
1858                 rxe->mbuf = nmb;
1859                 dma_addr =
1860                         rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
1861
1862                 /* Set data buffer address and data length of the mbuf */
1863                 rxdp->read.hdr_addr = 0;
1864                 rxdp->read.pkt_addr = dma_addr;
1865                 rx_packet_len = rte_le_to_cpu_16(rxd.wb.pkt_len) &
1866                                 ICE_RX_FLX_DESC_PKT_LEN_M;
1867                 rxm->data_len = rx_packet_len;
1868                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1869
1870                 /**
1871                  * If this is the first buffer of the received packet, set the
1872                  * pointer to the first mbuf of the packet and initialize its
1873                  * context. Otherwise, update the total length and the number
1874                  * of segments of the current scattered packet, and update the
1875                  * pointer to the last mbuf of the current packet.
1876                  */
1877                 if (!first_seg) {
1878                         first_seg = rxm;
1879                         first_seg->nb_segs = 1;
1880                         first_seg->pkt_len = rx_packet_len;
1881                 } else {
1882                         first_seg->pkt_len =
1883                                 (uint16_t)(first_seg->pkt_len +
1884                                            rx_packet_len);
1885                         first_seg->nb_segs++;
1886                         last_seg->next = rxm;
1887                 }
1888
1889                 /**
1890                  * If this is not the last buffer of the received packet,
1891                  * update the pointer to the last mbuf of the current scattered
1892                  * packet and continue to parse the RX ring.
1893                  */
1894                 if (!(rx_stat_err0 & (1 << ICE_RX_FLEX_DESC_STATUS0_EOF_S))) {
1895                         last_seg = rxm;
1896                         continue;
1897                 }
1898
1899                 /**
1900                  * This is the last buffer of the received packet. If the CRC
1901                  * is not stripped by the hardware:
1902                  *  - Subtract the CRC length from the total packet length.
1903                  *  - If the last buffer only contains the whole CRC or a part
1904                  *  of it, free the mbuf associated to the last buffer. If part
1905                  *  of the CRC is also contained in the previous mbuf, subtract
1906                  *  the length of that CRC part from the data length of the
1907                  *  previous mbuf.
1908                  */
1909                 rxm->next = NULL;
1910                 if (unlikely(rxq->crc_len > 0)) {
1911                         first_seg->pkt_len -= RTE_ETHER_CRC_LEN;
1912                         if (rx_packet_len <= RTE_ETHER_CRC_LEN) {
1913                                 rte_pktmbuf_free_seg(rxm);
1914                                 first_seg->nb_segs--;
1915                                 last_seg->data_len =
1916                                         (uint16_t)(last_seg->data_len -
1917                                         (RTE_ETHER_CRC_LEN - rx_packet_len));
1918                                 last_seg->next = NULL;
1919                         } else
1920                                 rxm->data_len = (uint16_t)(rx_packet_len -
1921                                                            RTE_ETHER_CRC_LEN);
1922                 }
1923
1924                 first_seg->port = rxq->port_id;
1925                 first_seg->ol_flags = 0;
1926                 first_seg->packet_type = ptype_tbl[ICE_RX_FLEX_DESC_PTYPE_M &
1927                         rte_le_to_cpu_16(rxd.wb.ptype_flex_flags0)];
1928                 ice_rxd_to_vlan_tci(first_seg, &rxd);
1929                 rxq->rxd_to_pkt_fields(rxq, first_seg, &rxd);
1930                 pkt_flags = ice_rxd_error_to_pkt_flags(rx_stat_err0);
1931 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
1932                 if (rxq->offloads & DEV_RX_OFFLOAD_TIMESTAMP) {
1933                         ts_ns = ice_tstamp_convert_32b_64b(hw,
1934                                 rte_le_to_cpu_32(rxd.wb.flex_ts.ts_high));
1935                         if (ice_timestamp_dynflag > 0) {
1936                                 *RTE_MBUF_DYNFIELD(first_seg,
1937                                         ice_timestamp_dynfield_offset,
1938                                         rte_mbuf_timestamp_t *) = ts_ns;
1939                                 first_seg->ol_flags |= ice_timestamp_dynflag;
1940                         }
1941                 }
1942
1943                 if (ad->ptp_ena && ((first_seg->packet_type & RTE_PTYPE_L2_MASK)
1944                     == RTE_PTYPE_L2_ETHER_TIMESYNC)) {
1945                         rxq->time_high =
1946                            rte_le_to_cpu_32(rxd.wb.flex_ts.ts_high);
1947                         first_seg->timesync = rxq->queue_id;
1948                         pkt_flags |= PKT_RX_IEEE1588_PTP;
1949                 }
1950 #endif
1951                 first_seg->ol_flags |= pkt_flags;
1952                 /* Prefetch data of first segment, if configured to do so. */
1953                 rte_prefetch0(RTE_PTR_ADD(first_seg->buf_addr,
1954                                           first_seg->data_off));
1955                 rx_pkts[nb_rx++] = first_seg;
1956                 first_seg = NULL;
1957         }
1958
1959         /* Record index of the next RX descriptor to probe. */
1960         rxq->rx_tail = rx_id;
1961         rxq->pkt_first_seg = first_seg;
1962         rxq->pkt_last_seg = last_seg;
1963
1964         /**
1965          * If the number of free RX descriptors is greater than the RX free
1966          * threshold of the queue, advance the Receive Descriptor Tail (RDT)
1967          * register. Update the RDT with the value of the last processed RX
1968          * descriptor minus 1, to guarantee that the RDT register is never
1969          * equal to the RDH register, which creates a "full" ring situtation
1970          * from the hardware point of view.
1971          */
1972         nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold);
1973         if (nb_hold > rxq->rx_free_thresh) {
1974                 rx_id = (uint16_t)(rx_id == 0 ?
1975                                    (rxq->nb_rx_desc - 1) : (rx_id - 1));
1976                 /* write TAIL register */
1977                 ICE_PCI_REG_WC_WRITE(rxq->qrx_tail, rx_id);
1978                 nb_hold = 0;
1979         }
1980         rxq->nb_rx_hold = nb_hold;
1981
1982         /* return received packet in the burst */
1983         return nb_rx;
1984 }
1985
1986 const uint32_t *
1987 ice_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1988 {
1989         struct ice_adapter *ad =
1990                 ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
1991         const uint32_t *ptypes;
1992
1993         static const uint32_t ptypes_os[] = {
1994                 /* refers to ice_get_default_pkt_type() */
1995                 RTE_PTYPE_L2_ETHER,
1996                 RTE_PTYPE_L2_ETHER_TIMESYNC,
1997                 RTE_PTYPE_L2_ETHER_LLDP,
1998                 RTE_PTYPE_L2_ETHER_ARP,
1999                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
2000                 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
2001                 RTE_PTYPE_L4_FRAG,
2002                 RTE_PTYPE_L4_ICMP,
2003                 RTE_PTYPE_L4_NONFRAG,
2004                 RTE_PTYPE_L4_SCTP,
2005                 RTE_PTYPE_L4_TCP,
2006                 RTE_PTYPE_L4_UDP,
2007                 RTE_PTYPE_TUNNEL_GRENAT,
2008                 RTE_PTYPE_TUNNEL_IP,
2009                 RTE_PTYPE_INNER_L2_ETHER,
2010                 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN,
2011                 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN,
2012                 RTE_PTYPE_INNER_L4_FRAG,
2013                 RTE_PTYPE_INNER_L4_ICMP,
2014                 RTE_PTYPE_INNER_L4_NONFRAG,
2015                 RTE_PTYPE_INNER_L4_SCTP,
2016                 RTE_PTYPE_INNER_L4_TCP,
2017                 RTE_PTYPE_INNER_L4_UDP,
2018                 RTE_PTYPE_UNKNOWN
2019         };
2020
2021         static const uint32_t ptypes_comms[] = {
2022                 /* refers to ice_get_default_pkt_type() */
2023                 RTE_PTYPE_L2_ETHER,
2024                 RTE_PTYPE_L2_ETHER_TIMESYNC,
2025                 RTE_PTYPE_L2_ETHER_LLDP,
2026                 RTE_PTYPE_L2_ETHER_ARP,
2027                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
2028                 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
2029                 RTE_PTYPE_L4_FRAG,
2030                 RTE_PTYPE_L4_ICMP,
2031                 RTE_PTYPE_L4_NONFRAG,
2032                 RTE_PTYPE_L4_SCTP,
2033                 RTE_PTYPE_L4_TCP,
2034                 RTE_PTYPE_L4_UDP,
2035                 RTE_PTYPE_TUNNEL_GRENAT,
2036                 RTE_PTYPE_TUNNEL_IP,
2037                 RTE_PTYPE_INNER_L2_ETHER,
2038                 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN,
2039                 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN,
2040                 RTE_PTYPE_INNER_L4_FRAG,
2041                 RTE_PTYPE_INNER_L4_ICMP,
2042                 RTE_PTYPE_INNER_L4_NONFRAG,
2043                 RTE_PTYPE_INNER_L4_SCTP,
2044                 RTE_PTYPE_INNER_L4_TCP,
2045                 RTE_PTYPE_INNER_L4_UDP,
2046                 RTE_PTYPE_TUNNEL_GTPC,
2047                 RTE_PTYPE_TUNNEL_GTPU,
2048                 RTE_PTYPE_L2_ETHER_PPPOE,
2049                 RTE_PTYPE_UNKNOWN
2050         };
2051
2052         if (ad->active_pkg_type == ICE_PKG_TYPE_COMMS)
2053                 ptypes = ptypes_comms;
2054         else
2055                 ptypes = ptypes_os;
2056
2057         if (dev->rx_pkt_burst == ice_recv_pkts ||
2058             dev->rx_pkt_burst == ice_recv_pkts_bulk_alloc ||
2059             dev->rx_pkt_burst == ice_recv_scattered_pkts)
2060                 return ptypes;
2061
2062 #ifdef RTE_ARCH_X86
2063         if (dev->rx_pkt_burst == ice_recv_pkts_vec ||
2064             dev->rx_pkt_burst == ice_recv_scattered_pkts_vec ||
2065 #ifdef CC_AVX512_SUPPORT
2066             dev->rx_pkt_burst == ice_recv_pkts_vec_avx512 ||
2067             dev->rx_pkt_burst == ice_recv_pkts_vec_avx512_offload ||
2068             dev->rx_pkt_burst == ice_recv_scattered_pkts_vec_avx512 ||
2069             dev->rx_pkt_burst == ice_recv_scattered_pkts_vec_avx512_offload ||
2070 #endif
2071             dev->rx_pkt_burst == ice_recv_pkts_vec_avx2 ||
2072             dev->rx_pkt_burst == ice_recv_pkts_vec_avx2_offload ||
2073             dev->rx_pkt_burst == ice_recv_scattered_pkts_vec_avx2 ||
2074             dev->rx_pkt_burst == ice_recv_scattered_pkts_vec_avx2_offload)
2075                 return ptypes;
2076 #endif
2077
2078         return NULL;
2079 }
2080
2081 int
2082 ice_rx_descriptor_status(void *rx_queue, uint16_t offset)
2083 {
2084         volatile union ice_rx_flex_desc *rxdp;
2085         struct ice_rx_queue *rxq = rx_queue;
2086         uint32_t desc;
2087
2088         if (unlikely(offset >= rxq->nb_rx_desc))
2089                 return -EINVAL;
2090
2091         if (offset >= rxq->nb_rx_desc - rxq->nb_rx_hold)
2092                 return RTE_ETH_RX_DESC_UNAVAIL;
2093
2094         desc = rxq->rx_tail + offset;
2095         if (desc >= rxq->nb_rx_desc)
2096                 desc -= rxq->nb_rx_desc;
2097
2098         rxdp = &rxq->rx_ring[desc];
2099         if (rte_le_to_cpu_16(rxdp->wb.status_error0) &
2100             (1 << ICE_RX_FLEX_DESC_STATUS0_DD_S))
2101                 return RTE_ETH_RX_DESC_DONE;
2102
2103         return RTE_ETH_RX_DESC_AVAIL;
2104 }
2105
2106 int
2107 ice_tx_descriptor_status(void *tx_queue, uint16_t offset)
2108 {
2109         struct ice_tx_queue *txq = tx_queue;
2110         volatile uint64_t *status;
2111         uint64_t mask, expect;
2112         uint32_t desc;
2113
2114         if (unlikely(offset >= txq->nb_tx_desc))
2115                 return -EINVAL;
2116
2117         desc = txq->tx_tail + offset;
2118         /* go to next desc that has the RS bit */
2119         desc = ((desc + txq->tx_rs_thresh - 1) / txq->tx_rs_thresh) *
2120                 txq->tx_rs_thresh;
2121         if (desc >= txq->nb_tx_desc) {
2122                 desc -= txq->nb_tx_desc;
2123                 if (desc >= txq->nb_tx_desc)
2124                         desc -= txq->nb_tx_desc;
2125         }
2126
2127         status = &txq->tx_ring[desc].cmd_type_offset_bsz;
2128         mask = rte_cpu_to_le_64(ICE_TXD_QW1_DTYPE_M);
2129         expect = rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DESC_DONE <<
2130                                   ICE_TXD_QW1_DTYPE_S);
2131         if ((*status & mask) == expect)
2132                 return RTE_ETH_TX_DESC_DONE;
2133
2134         return RTE_ETH_TX_DESC_FULL;
2135 }
2136
2137 void
2138 ice_free_queues(struct rte_eth_dev *dev)
2139 {
2140         uint16_t i;
2141
2142         PMD_INIT_FUNC_TRACE();
2143
2144         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2145                 if (!dev->data->rx_queues[i])
2146                         continue;
2147                 ice_rx_queue_release(dev->data->rx_queues[i]);
2148                 dev->data->rx_queues[i] = NULL;
2149         }
2150         dev->data->nb_rx_queues = 0;
2151
2152         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2153                 if (!dev->data->tx_queues[i])
2154                         continue;
2155                 ice_tx_queue_release(dev->data->tx_queues[i]);
2156                 dev->data->tx_queues[i] = NULL;
2157         }
2158         dev->data->nb_tx_queues = 0;
2159 }
2160
2161 #define ICE_FDIR_NUM_TX_DESC  ICE_MIN_RING_DESC
2162 #define ICE_FDIR_NUM_RX_DESC  ICE_MIN_RING_DESC
2163
2164 int
2165 ice_fdir_setup_tx_resources(struct ice_pf *pf)
2166 {
2167         struct ice_tx_queue *txq;
2168         const struct rte_memzone *tz = NULL;
2169         uint32_t ring_size;
2170         struct rte_eth_dev *dev;
2171
2172         if (!pf) {
2173                 PMD_DRV_LOG(ERR, "PF is not available");
2174                 return -EINVAL;
2175         }
2176
2177         dev = &rte_eth_devices[pf->adapter->pf.dev_data->port_id];
2178
2179         /* Allocate the TX queue data structure. */
2180         txq = rte_zmalloc_socket("ice fdir tx queue",
2181                                  sizeof(struct ice_tx_queue),
2182                                  RTE_CACHE_LINE_SIZE,
2183                                  SOCKET_ID_ANY);
2184         if (!txq) {
2185                 PMD_DRV_LOG(ERR, "Failed to allocate memory for "
2186                             "tx queue structure.");
2187                 return -ENOMEM;
2188         }
2189
2190         /* Allocate TX hardware ring descriptors. */
2191         ring_size = sizeof(struct ice_tx_desc) * ICE_FDIR_NUM_TX_DESC;
2192         ring_size = RTE_ALIGN(ring_size, ICE_DMA_MEM_ALIGN);
2193
2194         tz = rte_eth_dma_zone_reserve(dev, "fdir_tx_ring",
2195                                       ICE_FDIR_QUEUE_ID, ring_size,
2196                                       ICE_RING_BASE_ALIGN, SOCKET_ID_ANY);
2197         if (!tz) {
2198                 ice_tx_queue_release(txq);
2199                 PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for TX.");
2200                 return -ENOMEM;
2201         }
2202
2203         txq->mz = tz;
2204         txq->nb_tx_desc = ICE_FDIR_NUM_TX_DESC;
2205         txq->queue_id = ICE_FDIR_QUEUE_ID;
2206         txq->reg_idx = pf->fdir.fdir_vsi->base_queue;
2207         txq->vsi = pf->fdir.fdir_vsi;
2208
2209         txq->tx_ring_dma = tz->iova;
2210         txq->tx_ring = (struct ice_tx_desc *)tz->addr;
2211         /*
2212          * don't need to allocate software ring and reset for the fdir
2213          * program queue just set the queue has been configured.
2214          */
2215         txq->q_set = true;
2216         pf->fdir.txq = txq;
2217
2218         txq->tx_rel_mbufs = _ice_tx_queue_release_mbufs;
2219
2220         return ICE_SUCCESS;
2221 }
2222
2223 int
2224 ice_fdir_setup_rx_resources(struct ice_pf *pf)
2225 {
2226         struct ice_rx_queue *rxq;
2227         const struct rte_memzone *rz = NULL;
2228         uint32_t ring_size;
2229         struct rte_eth_dev *dev;
2230
2231         if (!pf) {
2232                 PMD_DRV_LOG(ERR, "PF is not available");
2233                 return -EINVAL;
2234         }
2235
2236         dev = &rte_eth_devices[pf->adapter->pf.dev_data->port_id];
2237
2238         /* Allocate the RX queue data structure. */
2239         rxq = rte_zmalloc_socket("ice fdir rx queue",
2240                                  sizeof(struct ice_rx_queue),
2241                                  RTE_CACHE_LINE_SIZE,
2242                                  SOCKET_ID_ANY);
2243         if (!rxq) {
2244                 PMD_DRV_LOG(ERR, "Failed to allocate memory for "
2245                             "rx queue structure.");
2246                 return -ENOMEM;
2247         }
2248
2249         /* Allocate RX hardware ring descriptors. */
2250         ring_size = sizeof(union ice_32byte_rx_desc) * ICE_FDIR_NUM_RX_DESC;
2251         ring_size = RTE_ALIGN(ring_size, ICE_DMA_MEM_ALIGN);
2252
2253         rz = rte_eth_dma_zone_reserve(dev, "fdir_rx_ring",
2254                                       ICE_FDIR_QUEUE_ID, ring_size,
2255                                       ICE_RING_BASE_ALIGN, SOCKET_ID_ANY);
2256         if (!rz) {
2257                 ice_rx_queue_release(rxq);
2258                 PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for RX.");
2259                 return -ENOMEM;
2260         }
2261
2262         rxq->mz = rz;
2263         rxq->nb_rx_desc = ICE_FDIR_NUM_RX_DESC;
2264         rxq->queue_id = ICE_FDIR_QUEUE_ID;
2265         rxq->reg_idx = pf->fdir.fdir_vsi->base_queue;
2266         rxq->vsi = pf->fdir.fdir_vsi;
2267
2268         rxq->rx_ring_dma = rz->iova;
2269         memset(rz->addr, 0, ICE_FDIR_NUM_RX_DESC *
2270                sizeof(union ice_32byte_rx_desc));
2271         rxq->rx_ring = (union ice_rx_flex_desc *)rz->addr;
2272
2273         /*
2274          * Don't need to allocate software ring and reset for the fdir
2275          * rx queue, just set the queue has been configured.
2276          */
2277         rxq->q_set = true;
2278         pf->fdir.rxq = rxq;
2279
2280         rxq->rx_rel_mbufs = _ice_rx_queue_release_mbufs;
2281
2282         return ICE_SUCCESS;
2283 }
2284
2285 uint16_t
2286 ice_recv_pkts(void *rx_queue,
2287               struct rte_mbuf **rx_pkts,
2288               uint16_t nb_pkts)
2289 {
2290         struct ice_rx_queue *rxq = rx_queue;
2291         volatile union ice_rx_flex_desc *rx_ring = rxq->rx_ring;
2292         volatile union ice_rx_flex_desc *rxdp;
2293         union ice_rx_flex_desc rxd;
2294         struct ice_rx_entry *sw_ring = rxq->sw_ring;
2295         struct ice_rx_entry *rxe;
2296         struct rte_mbuf *nmb; /* new allocated mbuf */
2297         struct rte_mbuf *rxm; /* pointer to store old mbuf in SW ring */
2298         uint16_t rx_id = rxq->rx_tail;
2299         uint16_t nb_rx = 0;
2300         uint16_t nb_hold = 0;
2301         uint16_t rx_packet_len;
2302         uint16_t rx_stat_err0;
2303         uint64_t dma_addr;
2304         uint64_t pkt_flags;
2305         uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl;
2306 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
2307         struct ice_vsi *vsi = rxq->vsi;
2308         struct ice_hw *hw = ICE_VSI_TO_HW(vsi);
2309         uint64_t ts_ns;
2310         struct ice_adapter *ad = rxq->vsi->adapter;
2311 #endif
2312         while (nb_rx < nb_pkts) {
2313                 rxdp = &rx_ring[rx_id];
2314                 rx_stat_err0 = rte_le_to_cpu_16(rxdp->wb.status_error0);
2315
2316                 /* Check the DD bit first */
2317                 if (!(rx_stat_err0 & (1 << ICE_RX_FLEX_DESC_STATUS0_DD_S)))
2318                         break;
2319
2320                 /* allocate mbuf */
2321                 nmb = rte_mbuf_raw_alloc(rxq->mp);
2322                 if (unlikely(!nmb)) {
2323                         rxq->vsi->adapter->pf.dev_data->rx_mbuf_alloc_failed++;
2324                         break;
2325                 }
2326                 rxd = *rxdp; /* copy descriptor in ring to temp variable*/
2327
2328                 nb_hold++;
2329                 rxe = &sw_ring[rx_id]; /* get corresponding mbuf in SW ring */
2330                 rx_id++;
2331                 if (unlikely(rx_id == rxq->nb_rx_desc))
2332                         rx_id = 0;
2333                 rxm = rxe->mbuf;
2334                 rxe->mbuf = nmb;
2335                 dma_addr =
2336                         rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
2337
2338                 /**
2339                  * fill the read format of descriptor with physic address in
2340                  * new allocated mbuf: nmb
2341                  */
2342                 rxdp->read.hdr_addr = 0;
2343                 rxdp->read.pkt_addr = dma_addr;
2344
2345                 /* calculate rx_packet_len of the received pkt */
2346                 rx_packet_len = (rte_le_to_cpu_16(rxd.wb.pkt_len) &
2347                                  ICE_RX_FLX_DESC_PKT_LEN_M) - rxq->crc_len;
2348
2349                 /* fill old mbuf with received descriptor: rxd */
2350                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
2351                 rte_prefetch0(RTE_PTR_ADD(rxm->buf_addr, RTE_PKTMBUF_HEADROOM));
2352                 rxm->nb_segs = 1;
2353                 rxm->next = NULL;
2354                 rxm->pkt_len = rx_packet_len;
2355                 rxm->data_len = rx_packet_len;
2356                 rxm->port = rxq->port_id;
2357                 rxm->packet_type = ptype_tbl[ICE_RX_FLEX_DESC_PTYPE_M &
2358                         rte_le_to_cpu_16(rxd.wb.ptype_flex_flags0)];
2359                 ice_rxd_to_vlan_tci(rxm, &rxd);
2360                 rxq->rxd_to_pkt_fields(rxq, rxm, &rxd);
2361                 pkt_flags = ice_rxd_error_to_pkt_flags(rx_stat_err0);
2362 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
2363                 if (rxq->offloads & DEV_RX_OFFLOAD_TIMESTAMP) {
2364                         ts_ns = ice_tstamp_convert_32b_64b(hw,
2365                                 rte_le_to_cpu_32(rxd.wb.flex_ts.ts_high));
2366                         if (ice_timestamp_dynflag > 0) {
2367                                 *RTE_MBUF_DYNFIELD(rxm,
2368                                         ice_timestamp_dynfield_offset,
2369                                         rte_mbuf_timestamp_t *) = ts_ns;
2370                                 rxm->ol_flags |= ice_timestamp_dynflag;
2371                         }
2372                 }
2373
2374                 if (ad->ptp_ena && ((rxm->packet_type & RTE_PTYPE_L2_MASK) ==
2375                     RTE_PTYPE_L2_ETHER_TIMESYNC)) {
2376                         rxq->time_high =
2377                            rte_le_to_cpu_32(rxd.wb.flex_ts.ts_high);
2378                         rxm->timesync = rxq->queue_id;
2379                         pkt_flags |= PKT_RX_IEEE1588_PTP;
2380                 }
2381 #endif
2382                 rxm->ol_flags |= pkt_flags;
2383                 /* copy old mbuf to rx_pkts */
2384                 rx_pkts[nb_rx++] = rxm;
2385         }
2386         rxq->rx_tail = rx_id;
2387         /**
2388          * If the number of free RX descriptors is greater than the RX free
2389          * threshold of the queue, advance the receive tail register of queue.
2390          * Update that register with the value of the last processed RX
2391          * descriptor minus 1.
2392          */
2393         nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold);
2394         if (nb_hold > rxq->rx_free_thresh) {
2395                 rx_id = (uint16_t)(rx_id == 0 ?
2396                                    (rxq->nb_rx_desc - 1) : (rx_id - 1));
2397                 /* write TAIL register */
2398                 ICE_PCI_REG_WC_WRITE(rxq->qrx_tail, rx_id);
2399                 nb_hold = 0;
2400         }
2401         rxq->nb_rx_hold = nb_hold;
2402
2403         /* return received packet in the burst */
2404         return nb_rx;
2405 }
2406
2407 static inline void
2408 ice_parse_tunneling_params(uint64_t ol_flags,
2409                             union ice_tx_offload tx_offload,
2410                             uint32_t *cd_tunneling)
2411 {
2412         /* EIPT: External (outer) IP header type */
2413         if (ol_flags & PKT_TX_OUTER_IP_CKSUM)
2414                 *cd_tunneling |= ICE_TX_CTX_EIPT_IPV4;
2415         else if (ol_flags & PKT_TX_OUTER_IPV4)
2416                 *cd_tunneling |= ICE_TX_CTX_EIPT_IPV4_NO_CSUM;
2417         else if (ol_flags & PKT_TX_OUTER_IPV6)
2418                 *cd_tunneling |= ICE_TX_CTX_EIPT_IPV6;
2419
2420         /* EIPLEN: External (outer) IP header length, in DWords */
2421         *cd_tunneling |= (tx_offload.outer_l3_len >> 2) <<
2422                 ICE_TXD_CTX_QW0_EIPLEN_S;
2423
2424         /* L4TUNT: L4 Tunneling Type */
2425         switch (ol_flags & PKT_TX_TUNNEL_MASK) {
2426         case PKT_TX_TUNNEL_IPIP:
2427                 /* for non UDP / GRE tunneling, set to 00b */
2428                 break;
2429         case PKT_TX_TUNNEL_VXLAN:
2430         case PKT_TX_TUNNEL_GTP:
2431         case PKT_TX_TUNNEL_GENEVE:
2432                 *cd_tunneling |= ICE_TXD_CTX_UDP_TUNNELING;
2433                 break;
2434         case PKT_TX_TUNNEL_GRE:
2435                 *cd_tunneling |= ICE_TXD_CTX_GRE_TUNNELING;
2436                 break;
2437         default:
2438                 PMD_TX_LOG(ERR, "Tunnel type not supported");
2439                 return;
2440         }
2441
2442         /* L4TUNLEN: L4 Tunneling Length, in Words
2443          *
2444          * We depend on app to set rte_mbuf.l2_len correctly.
2445          * For IP in GRE it should be set to the length of the GRE
2446          * header;
2447          * For MAC in GRE or MAC in UDP it should be set to the length
2448          * of the GRE or UDP headers plus the inner MAC up to including
2449          * its last Ethertype.
2450          * If MPLS labels exists, it should include them as well.
2451          */
2452         *cd_tunneling |= (tx_offload.l2_len >> 1) <<
2453                 ICE_TXD_CTX_QW0_NATLEN_S;
2454
2455         /**
2456          * Calculate the tunneling UDP checksum.
2457          * Shall be set only if L4TUNT = 01b and EIPT is not zero
2458          */
2459         if (!(*cd_tunneling & ICE_TX_CTX_EIPT_NONE) &&
2460             (*cd_tunneling & ICE_TXD_CTX_UDP_TUNNELING))
2461                 *cd_tunneling |= ICE_TXD_CTX_QW0_L4T_CS_M;
2462 }
2463
2464 static inline void
2465 ice_txd_enable_checksum(uint64_t ol_flags,
2466                         uint32_t *td_cmd,
2467                         uint32_t *td_offset,
2468                         union ice_tx_offload tx_offload)
2469 {
2470         /* Set MACLEN */
2471         if (ol_flags & PKT_TX_TUNNEL_MASK)
2472                 *td_offset |= (tx_offload.outer_l2_len >> 1)
2473                         << ICE_TX_DESC_LEN_MACLEN_S;
2474         else
2475                 *td_offset |= (tx_offload.l2_len >> 1)
2476                         << ICE_TX_DESC_LEN_MACLEN_S;
2477
2478         /* Enable L3 checksum offloads */
2479         if (ol_flags & PKT_TX_IP_CKSUM) {
2480                 *td_cmd |= ICE_TX_DESC_CMD_IIPT_IPV4_CSUM;
2481                 *td_offset |= (tx_offload.l3_len >> 2) <<
2482                               ICE_TX_DESC_LEN_IPLEN_S;
2483         } else if (ol_flags & PKT_TX_IPV4) {
2484                 *td_cmd |= ICE_TX_DESC_CMD_IIPT_IPV4;
2485                 *td_offset |= (tx_offload.l3_len >> 2) <<
2486                               ICE_TX_DESC_LEN_IPLEN_S;
2487         } else if (ol_flags & PKT_TX_IPV6) {
2488                 *td_cmd |= ICE_TX_DESC_CMD_IIPT_IPV6;
2489                 *td_offset |= (tx_offload.l3_len >> 2) <<
2490                               ICE_TX_DESC_LEN_IPLEN_S;
2491         }
2492
2493         if (ol_flags & PKT_TX_TCP_SEG) {
2494                 *td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_TCP;
2495                 *td_offset |= (tx_offload.l4_len >> 2) <<
2496                               ICE_TX_DESC_LEN_L4_LEN_S;
2497                 return;
2498         }
2499
2500         /* Enable L4 checksum offloads */
2501         switch (ol_flags & PKT_TX_L4_MASK) {
2502         case PKT_TX_TCP_CKSUM:
2503                 *td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_TCP;
2504                 *td_offset |= (sizeof(struct rte_tcp_hdr) >> 2) <<
2505                               ICE_TX_DESC_LEN_L4_LEN_S;
2506                 break;
2507         case PKT_TX_SCTP_CKSUM:
2508                 *td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_SCTP;
2509                 *td_offset |= (sizeof(struct rte_sctp_hdr) >> 2) <<
2510                               ICE_TX_DESC_LEN_L4_LEN_S;
2511                 break;
2512         case PKT_TX_UDP_CKSUM:
2513                 *td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_UDP;
2514                 *td_offset |= (sizeof(struct rte_udp_hdr) >> 2) <<
2515                               ICE_TX_DESC_LEN_L4_LEN_S;
2516                 break;
2517         default:
2518                 break;
2519         }
2520 }
2521
2522 static inline int
2523 ice_xmit_cleanup(struct ice_tx_queue *txq)
2524 {
2525         struct ice_tx_entry *sw_ring = txq->sw_ring;
2526         volatile struct ice_tx_desc *txd = txq->tx_ring;
2527         uint16_t last_desc_cleaned = txq->last_desc_cleaned;
2528         uint16_t nb_tx_desc = txq->nb_tx_desc;
2529         uint16_t desc_to_clean_to;
2530         uint16_t nb_tx_to_clean;
2531
2532         /* Determine the last descriptor needing to be cleaned */
2533         desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->tx_rs_thresh);
2534         if (desc_to_clean_to >= nb_tx_desc)
2535                 desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc);
2536
2537         /* Check to make sure the last descriptor to clean is done */
2538         desc_to_clean_to = sw_ring[desc_to_clean_to].last_id;
2539         if (!(txd[desc_to_clean_to].cmd_type_offset_bsz &
2540             rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DESC_DONE))) {
2541                 PMD_TX_LOG(DEBUG, "TX descriptor %4u is not done "
2542                            "(port=%d queue=%d) value=0x%"PRIx64"\n",
2543                            desc_to_clean_to,
2544                            txq->port_id, txq->queue_id,
2545                            txd[desc_to_clean_to].cmd_type_offset_bsz);
2546                 /* Failed to clean any descriptors */
2547                 return -1;
2548         }
2549
2550         /* Figure out how many descriptors will be cleaned */
2551         if (last_desc_cleaned > desc_to_clean_to)
2552                 nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) +
2553                                             desc_to_clean_to);
2554         else
2555                 nb_tx_to_clean = (uint16_t)(desc_to_clean_to -
2556                                             last_desc_cleaned);
2557
2558         /* The last descriptor to clean is done, so that means all the
2559          * descriptors from the last descriptor that was cleaned
2560          * up to the last descriptor with the RS bit set
2561          * are done. Only reset the threshold descriptor.
2562          */
2563         txd[desc_to_clean_to].cmd_type_offset_bsz = 0;
2564
2565         /* Update the txq to reflect the last descriptor that was cleaned */
2566         txq->last_desc_cleaned = desc_to_clean_to;
2567         txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + nb_tx_to_clean);
2568
2569         return 0;
2570 }
2571
2572 /* Construct the tx flags */
2573 static inline uint64_t
2574 ice_build_ctob(uint32_t td_cmd,
2575                uint32_t td_offset,
2576                uint16_t size,
2577                uint32_t td_tag)
2578 {
2579         return rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DATA |
2580                                 ((uint64_t)td_cmd << ICE_TXD_QW1_CMD_S) |
2581                                 ((uint64_t)td_offset << ICE_TXD_QW1_OFFSET_S) |
2582                                 ((uint64_t)size << ICE_TXD_QW1_TX_BUF_SZ_S) |
2583                                 ((uint64_t)td_tag << ICE_TXD_QW1_L2TAG1_S));
2584 }
2585
2586 /* Check if the context descriptor is needed for TX offloading */
2587 static inline uint16_t
2588 ice_calc_context_desc(uint64_t flags)
2589 {
2590         static uint64_t mask = PKT_TX_TCP_SEG |
2591                 PKT_TX_QINQ |
2592                 PKT_TX_OUTER_IP_CKSUM |
2593                 PKT_TX_TUNNEL_MASK |
2594                 PKT_TX_IEEE1588_TMST;
2595
2596         return (flags & mask) ? 1 : 0;
2597 }
2598
2599 /* set ice TSO context descriptor */
2600 static inline uint64_t
2601 ice_set_tso_ctx(struct rte_mbuf *mbuf, union ice_tx_offload tx_offload)
2602 {
2603         uint64_t ctx_desc = 0;
2604         uint32_t cd_cmd, hdr_len, cd_tso_len;
2605
2606         if (!tx_offload.l4_len) {
2607                 PMD_TX_LOG(DEBUG, "L4 length set to 0");
2608                 return ctx_desc;
2609         }
2610
2611         hdr_len = tx_offload.l2_len + tx_offload.l3_len + tx_offload.l4_len;
2612         hdr_len += (mbuf->ol_flags & PKT_TX_TUNNEL_MASK) ?
2613                    tx_offload.outer_l2_len + tx_offload.outer_l3_len : 0;
2614
2615         cd_cmd = ICE_TX_CTX_DESC_TSO;
2616         cd_tso_len = mbuf->pkt_len - hdr_len;
2617         ctx_desc |= ((uint64_t)cd_cmd << ICE_TXD_CTX_QW1_CMD_S) |
2618                     ((uint64_t)cd_tso_len << ICE_TXD_CTX_QW1_TSO_LEN_S) |
2619                     ((uint64_t)mbuf->tso_segsz << ICE_TXD_CTX_QW1_MSS_S);
2620
2621         return ctx_desc;
2622 }
2623
2624 /* HW requires that TX buffer size ranges from 1B up to (16K-1)B. */
2625 #define ICE_MAX_DATA_PER_TXD \
2626         (ICE_TXD_QW1_TX_BUF_SZ_M >> ICE_TXD_QW1_TX_BUF_SZ_S)
2627 /* Calculate the number of TX descriptors needed for each pkt */
2628 static inline uint16_t
2629 ice_calc_pkt_desc(struct rte_mbuf *tx_pkt)
2630 {
2631         struct rte_mbuf *txd = tx_pkt;
2632         uint16_t count = 0;
2633
2634         while (txd != NULL) {
2635                 count += DIV_ROUND_UP(txd->data_len, ICE_MAX_DATA_PER_TXD);
2636                 txd = txd->next;
2637         }
2638
2639         return count;
2640 }
2641
2642 uint16_t
2643 ice_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
2644 {
2645         struct ice_tx_queue *txq;
2646         volatile struct ice_tx_desc *tx_ring;
2647         volatile struct ice_tx_desc *txd;
2648         struct ice_tx_entry *sw_ring;
2649         struct ice_tx_entry *txe, *txn;
2650         struct rte_mbuf *tx_pkt;
2651         struct rte_mbuf *m_seg;
2652         uint32_t cd_tunneling_params;
2653         uint16_t tx_id;
2654         uint16_t nb_tx;
2655         uint16_t nb_used;
2656         uint16_t nb_ctx;
2657         uint32_t td_cmd = 0;
2658         uint32_t td_offset = 0;
2659         uint32_t td_tag = 0;
2660         uint16_t tx_last;
2661         uint16_t slen;
2662         uint64_t buf_dma_addr;
2663         uint64_t ol_flags;
2664         union ice_tx_offload tx_offload = {0};
2665
2666         txq = tx_queue;
2667         sw_ring = txq->sw_ring;
2668         tx_ring = txq->tx_ring;
2669         tx_id = txq->tx_tail;
2670         txe = &sw_ring[tx_id];
2671
2672         /* Check if the descriptor ring needs to be cleaned. */
2673         if (txq->nb_tx_free < txq->tx_free_thresh)
2674                 (void)ice_xmit_cleanup(txq);
2675
2676         for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
2677                 tx_pkt = *tx_pkts++;
2678
2679                 td_cmd = 0;
2680                 td_tag = 0;
2681                 td_offset = 0;
2682                 ol_flags = tx_pkt->ol_flags;
2683                 tx_offload.l2_len = tx_pkt->l2_len;
2684                 tx_offload.l3_len = tx_pkt->l3_len;
2685                 tx_offload.outer_l2_len = tx_pkt->outer_l2_len;
2686                 tx_offload.outer_l3_len = tx_pkt->outer_l3_len;
2687                 tx_offload.l4_len = tx_pkt->l4_len;
2688                 tx_offload.tso_segsz = tx_pkt->tso_segsz;
2689                 /* Calculate the number of context descriptors needed. */
2690                 nb_ctx = ice_calc_context_desc(ol_flags);
2691
2692                 /* The number of descriptors that must be allocated for
2693                  * a packet equals to the number of the segments of that
2694                  * packet plus the number of context descriptor if needed.
2695                  * Recalculate the needed tx descs when TSO enabled in case
2696                  * the mbuf data size exceeds max data size that hw allows
2697                  * per tx desc.
2698                  */
2699                 if (ol_flags & PKT_TX_TCP_SEG)
2700                         nb_used = (uint16_t)(ice_calc_pkt_desc(tx_pkt) +
2701                                              nb_ctx);
2702                 else
2703                         nb_used = (uint16_t)(tx_pkt->nb_segs + nb_ctx);
2704                 tx_last = (uint16_t)(tx_id + nb_used - 1);
2705
2706                 /* Circular ring */
2707                 if (tx_last >= txq->nb_tx_desc)
2708                         tx_last = (uint16_t)(tx_last - txq->nb_tx_desc);
2709
2710                 if (nb_used > txq->nb_tx_free) {
2711                         if (ice_xmit_cleanup(txq) != 0) {
2712                                 if (nb_tx == 0)
2713                                         return 0;
2714                                 goto end_of_tx;
2715                         }
2716                         if (unlikely(nb_used > txq->tx_rs_thresh)) {
2717                                 while (nb_used > txq->nb_tx_free) {
2718                                         if (ice_xmit_cleanup(txq) != 0) {
2719                                                 if (nb_tx == 0)
2720                                                         return 0;
2721                                                 goto end_of_tx;
2722                                         }
2723                                 }
2724                         }
2725                 }
2726
2727                 /* Descriptor based VLAN insertion */
2728                 if (ol_flags & (PKT_TX_VLAN | PKT_TX_QINQ)) {
2729                         td_cmd |= ICE_TX_DESC_CMD_IL2TAG1;
2730                         td_tag = tx_pkt->vlan_tci;
2731                 }
2732
2733                 /* Fill in tunneling parameters if necessary */
2734                 cd_tunneling_params = 0;
2735                 if (ol_flags & PKT_TX_TUNNEL_MASK)
2736                         ice_parse_tunneling_params(ol_flags, tx_offload,
2737                                                    &cd_tunneling_params);
2738
2739                 /* Enable checksum offloading */
2740                 if (ol_flags & ICE_TX_CKSUM_OFFLOAD_MASK)
2741                         ice_txd_enable_checksum(ol_flags, &td_cmd,
2742                                                 &td_offset, tx_offload);
2743
2744                 if (nb_ctx) {
2745                         /* Setup TX context descriptor if required */
2746                         volatile struct ice_tx_ctx_desc *ctx_txd =
2747                                 (volatile struct ice_tx_ctx_desc *)
2748                                         &tx_ring[tx_id];
2749                         uint16_t cd_l2tag2 = 0;
2750                         uint64_t cd_type_cmd_tso_mss = ICE_TX_DESC_DTYPE_CTX;
2751
2752                         txn = &sw_ring[txe->next_id];
2753                         RTE_MBUF_PREFETCH_TO_FREE(txn->mbuf);
2754                         if (txe->mbuf) {
2755                                 rte_pktmbuf_free_seg(txe->mbuf);
2756                                 txe->mbuf = NULL;
2757                         }
2758
2759                         if (ol_flags & PKT_TX_TCP_SEG)
2760                                 cd_type_cmd_tso_mss |=
2761                                         ice_set_tso_ctx(tx_pkt, tx_offload);
2762                         else if (ol_flags & PKT_TX_IEEE1588_TMST)
2763                                 cd_type_cmd_tso_mss |=
2764                                         ((uint64_t)ICE_TX_CTX_DESC_TSYN <<
2765                                         ICE_TXD_CTX_QW1_CMD_S);
2766
2767                         ctx_txd->tunneling_params =
2768                                 rte_cpu_to_le_32(cd_tunneling_params);
2769
2770                         /* TX context descriptor based double VLAN insert */
2771                         if (ol_flags & PKT_TX_QINQ) {
2772                                 cd_l2tag2 = tx_pkt->vlan_tci_outer;
2773                                 cd_type_cmd_tso_mss |=
2774                                         ((uint64_t)ICE_TX_CTX_DESC_IL2TAG2 <<
2775                                          ICE_TXD_CTX_QW1_CMD_S);
2776                         }
2777                         ctx_txd->l2tag2 = rte_cpu_to_le_16(cd_l2tag2);
2778                         ctx_txd->qw1 =
2779                                 rte_cpu_to_le_64(cd_type_cmd_tso_mss);
2780
2781                         txe->last_id = tx_last;
2782                         tx_id = txe->next_id;
2783                         txe = txn;
2784                 }
2785                 m_seg = tx_pkt;
2786
2787                 do {
2788                         txd = &tx_ring[tx_id];
2789                         txn = &sw_ring[txe->next_id];
2790
2791                         if (txe->mbuf)
2792                                 rte_pktmbuf_free_seg(txe->mbuf);
2793                         txe->mbuf = m_seg;
2794
2795                         /* Setup TX Descriptor */
2796                         slen = m_seg->data_len;
2797                         buf_dma_addr = rte_mbuf_data_iova(m_seg);
2798
2799                         while ((ol_flags & PKT_TX_TCP_SEG) &&
2800                                 unlikely(slen > ICE_MAX_DATA_PER_TXD)) {
2801                                 txd->buf_addr = rte_cpu_to_le_64(buf_dma_addr);
2802                                 txd->cmd_type_offset_bsz =
2803                                 rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DATA |
2804                                 ((uint64_t)td_cmd << ICE_TXD_QW1_CMD_S) |
2805                                 ((uint64_t)td_offset << ICE_TXD_QW1_OFFSET_S) |
2806                                 ((uint64_t)ICE_MAX_DATA_PER_TXD <<
2807                                  ICE_TXD_QW1_TX_BUF_SZ_S) |
2808                                 ((uint64_t)td_tag << ICE_TXD_QW1_L2TAG1_S));
2809
2810                                 buf_dma_addr += ICE_MAX_DATA_PER_TXD;
2811                                 slen -= ICE_MAX_DATA_PER_TXD;
2812
2813                                 txe->last_id = tx_last;
2814                                 tx_id = txe->next_id;
2815                                 txe = txn;
2816                                 txd = &tx_ring[tx_id];
2817                                 txn = &sw_ring[txe->next_id];
2818                         }
2819
2820                         txd->buf_addr = rte_cpu_to_le_64(buf_dma_addr);
2821                         txd->cmd_type_offset_bsz =
2822                                 rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DATA |
2823                                 ((uint64_t)td_cmd << ICE_TXD_QW1_CMD_S) |
2824                                 ((uint64_t)td_offset << ICE_TXD_QW1_OFFSET_S) |
2825                                 ((uint64_t)slen << ICE_TXD_QW1_TX_BUF_SZ_S) |
2826                                 ((uint64_t)td_tag << ICE_TXD_QW1_L2TAG1_S));
2827
2828                         txe->last_id = tx_last;
2829                         tx_id = txe->next_id;
2830                         txe = txn;
2831                         m_seg = m_seg->next;
2832                 } while (m_seg);
2833
2834                 /* fill the last descriptor with End of Packet (EOP) bit */
2835                 td_cmd |= ICE_TX_DESC_CMD_EOP;
2836                 txq->nb_tx_used = (uint16_t)(txq->nb_tx_used + nb_used);
2837                 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_used);
2838
2839                 /* set RS bit on the last descriptor of one packet */
2840                 if (txq->nb_tx_used >= txq->tx_rs_thresh) {
2841                         PMD_TX_LOG(DEBUG,
2842                                    "Setting RS bit on TXD id="
2843                                    "%4u (port=%d queue=%d)",
2844                                    tx_last, txq->port_id, txq->queue_id);
2845
2846                         td_cmd |= ICE_TX_DESC_CMD_RS;
2847
2848                         /* Update txq RS bit counters */
2849                         txq->nb_tx_used = 0;
2850                 }
2851                 txd->cmd_type_offset_bsz |=
2852                         rte_cpu_to_le_64(((uint64_t)td_cmd) <<
2853                                          ICE_TXD_QW1_CMD_S);
2854         }
2855 end_of_tx:
2856         /* update Tail register */
2857         ICE_PCI_REG_WRITE(txq->qtx_tail, tx_id);
2858         txq->tx_tail = tx_id;
2859
2860         return nb_tx;
2861 }
2862
2863 static __rte_always_inline int
2864 ice_tx_free_bufs(struct ice_tx_queue *txq)
2865 {
2866         struct ice_tx_entry *txep;
2867         uint16_t i;
2868
2869         if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &
2870              rte_cpu_to_le_64(ICE_TXD_QW1_DTYPE_M)) !=
2871             rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DESC_DONE))
2872                 return 0;
2873
2874         txep = &txq->sw_ring[txq->tx_next_dd - (txq->tx_rs_thresh - 1)];
2875
2876         for (i = 0; i < txq->tx_rs_thresh; i++)
2877                 rte_prefetch0((txep + i)->mbuf);
2878
2879         if (txq->offloads & DEV_TX_OFFLOAD_MBUF_FAST_FREE) {
2880                 for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) {
2881                         rte_mempool_put(txep->mbuf->pool, txep->mbuf);
2882                         txep->mbuf = NULL;
2883                 }
2884         } else {
2885                 for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) {
2886                         rte_pktmbuf_free_seg(txep->mbuf);
2887                         txep->mbuf = NULL;
2888                 }
2889         }
2890
2891         txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh);
2892         txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh);
2893         if (txq->tx_next_dd >= txq->nb_tx_desc)
2894                 txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
2895
2896         return txq->tx_rs_thresh;
2897 }
2898
2899 static int
2900 ice_tx_done_cleanup_full(struct ice_tx_queue *txq,
2901                         uint32_t free_cnt)
2902 {
2903         struct ice_tx_entry *swr_ring = txq->sw_ring;
2904         uint16_t i, tx_last, tx_id;
2905         uint16_t nb_tx_free_last;
2906         uint16_t nb_tx_to_clean;
2907         uint32_t pkt_cnt;
2908
2909         /* Start free mbuf from the next of tx_tail */
2910         tx_last = txq->tx_tail;
2911         tx_id  = swr_ring[tx_last].next_id;
2912
2913         if (txq->nb_tx_free == 0 && ice_xmit_cleanup(txq))
2914                 return 0;
2915
2916         nb_tx_to_clean = txq->nb_tx_free;
2917         nb_tx_free_last = txq->nb_tx_free;
2918         if (!free_cnt)
2919                 free_cnt = txq->nb_tx_desc;
2920
2921         /* Loop through swr_ring to count the amount of
2922          * freeable mubfs and packets.
2923          */
2924         for (pkt_cnt = 0; pkt_cnt < free_cnt; ) {
2925                 for (i = 0; i < nb_tx_to_clean &&
2926                         pkt_cnt < free_cnt &&
2927                         tx_id != tx_last; i++) {
2928                         if (swr_ring[tx_id].mbuf != NULL) {
2929                                 rte_pktmbuf_free_seg(swr_ring[tx_id].mbuf);
2930                                 swr_ring[tx_id].mbuf = NULL;
2931
2932                                 /*
2933                                  * last segment in the packet,
2934                                  * increment packet count
2935                                  */
2936                                 pkt_cnt += (swr_ring[tx_id].last_id == tx_id);
2937                         }
2938
2939                         tx_id = swr_ring[tx_id].next_id;
2940                 }
2941
2942                 if (txq->tx_rs_thresh > txq->nb_tx_desc -
2943                         txq->nb_tx_free || tx_id == tx_last)
2944                         break;
2945
2946                 if (pkt_cnt < free_cnt) {
2947                         if (ice_xmit_cleanup(txq))
2948                                 break;
2949
2950                         nb_tx_to_clean = txq->nb_tx_free - nb_tx_free_last;
2951                         nb_tx_free_last = txq->nb_tx_free;
2952                 }
2953         }
2954
2955         return (int)pkt_cnt;
2956 }
2957
2958 #ifdef RTE_ARCH_X86
2959 static int
2960 ice_tx_done_cleanup_vec(struct ice_tx_queue *txq __rte_unused,
2961                         uint32_t free_cnt __rte_unused)
2962 {
2963         return -ENOTSUP;
2964 }
2965 #endif
2966
2967 static int
2968 ice_tx_done_cleanup_simple(struct ice_tx_queue *txq,
2969                         uint32_t free_cnt)
2970 {
2971         int i, n, cnt;
2972
2973         if (free_cnt == 0 || free_cnt > txq->nb_tx_desc)
2974                 free_cnt = txq->nb_tx_desc;
2975
2976         cnt = free_cnt - free_cnt % txq->tx_rs_thresh;
2977
2978         for (i = 0; i < cnt; i += n) {
2979                 if (txq->nb_tx_desc - txq->nb_tx_free < txq->tx_rs_thresh)
2980                         break;
2981
2982                 n = ice_tx_free_bufs(txq);
2983
2984                 if (n == 0)
2985                         break;
2986         }
2987
2988         return i;
2989 }
2990
2991 int
2992 ice_tx_done_cleanup(void *txq, uint32_t free_cnt)
2993 {
2994         struct ice_tx_queue *q = (struct ice_tx_queue *)txq;
2995         struct rte_eth_dev *dev = &rte_eth_devices[q->port_id];
2996         struct ice_adapter *ad =
2997                 ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
2998
2999 #ifdef RTE_ARCH_X86
3000         if (ad->tx_vec_allowed)
3001                 return ice_tx_done_cleanup_vec(q, free_cnt);
3002 #endif
3003         if (ad->tx_simple_allowed)
3004                 return ice_tx_done_cleanup_simple(q, free_cnt);
3005         else
3006                 return ice_tx_done_cleanup_full(q, free_cnt);
3007 }
3008
3009 /* Populate 4 descriptors with data from 4 mbufs */
3010 static inline void
3011 tx4(volatile struct ice_tx_desc *txdp, struct rte_mbuf **pkts)
3012 {
3013         uint64_t dma_addr;
3014         uint32_t i;
3015
3016         for (i = 0; i < 4; i++, txdp++, pkts++) {
3017                 dma_addr = rte_mbuf_data_iova(*pkts);
3018                 txdp->buf_addr = rte_cpu_to_le_64(dma_addr);
3019                 txdp->cmd_type_offset_bsz =
3020                         ice_build_ctob((uint32_t)ICE_TD_CMD, 0,
3021                                        (*pkts)->data_len, 0);
3022         }
3023 }
3024
3025 /* Populate 1 descriptor with data from 1 mbuf */
3026 static inline void
3027 tx1(volatile struct ice_tx_desc *txdp, struct rte_mbuf **pkts)
3028 {
3029         uint64_t dma_addr;
3030
3031         dma_addr = rte_mbuf_data_iova(*pkts);
3032         txdp->buf_addr = rte_cpu_to_le_64(dma_addr);
3033         txdp->cmd_type_offset_bsz =
3034                 ice_build_ctob((uint32_t)ICE_TD_CMD, 0,
3035                                (*pkts)->data_len, 0);
3036 }
3037
3038 static inline void
3039 ice_tx_fill_hw_ring(struct ice_tx_queue *txq, struct rte_mbuf **pkts,
3040                     uint16_t nb_pkts)
3041 {
3042         volatile struct ice_tx_desc *txdp = &txq->tx_ring[txq->tx_tail];
3043         struct ice_tx_entry *txep = &txq->sw_ring[txq->tx_tail];
3044         const int N_PER_LOOP = 4;
3045         const int N_PER_LOOP_MASK = N_PER_LOOP - 1;
3046         int mainpart, leftover;
3047         int i, j;
3048
3049         /**
3050          * Process most of the packets in chunks of N pkts.  Any
3051          * leftover packets will get processed one at a time.
3052          */
3053         mainpart = nb_pkts & ((uint32_t)~N_PER_LOOP_MASK);
3054         leftover = nb_pkts & ((uint32_t)N_PER_LOOP_MASK);
3055         for (i = 0; i < mainpart; i += N_PER_LOOP) {
3056                 /* Copy N mbuf pointers to the S/W ring */
3057                 for (j = 0; j < N_PER_LOOP; ++j)
3058                         (txep + i + j)->mbuf = *(pkts + i + j);
3059                 tx4(txdp + i, pkts + i);
3060         }
3061
3062         if (unlikely(leftover > 0)) {
3063                 for (i = 0; i < leftover; ++i) {
3064                         (txep + mainpart + i)->mbuf = *(pkts + mainpart + i);
3065                         tx1(txdp + mainpart + i, pkts + mainpart + i);
3066                 }
3067         }
3068 }
3069
3070 static inline uint16_t
3071 tx_xmit_pkts(struct ice_tx_queue *txq,
3072              struct rte_mbuf **tx_pkts,
3073              uint16_t nb_pkts)
3074 {
3075         volatile struct ice_tx_desc *txr = txq->tx_ring;
3076         uint16_t n = 0;
3077
3078         /**
3079          * Begin scanning the H/W ring for done descriptors when the number
3080          * of available descriptors drops below tx_free_thresh. For each done
3081          * descriptor, free the associated buffer.
3082          */
3083         if (txq->nb_tx_free < txq->tx_free_thresh)
3084                 ice_tx_free_bufs(txq);
3085
3086         /* Use available descriptor only */
3087         nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts);
3088         if (unlikely(!nb_pkts))
3089                 return 0;
3090
3091         txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts);
3092         if ((txq->tx_tail + nb_pkts) > txq->nb_tx_desc) {
3093                 n = (uint16_t)(txq->nb_tx_desc - txq->tx_tail);
3094                 ice_tx_fill_hw_ring(txq, tx_pkts, n);
3095                 txr[txq->tx_next_rs].cmd_type_offset_bsz |=
3096                         rte_cpu_to_le_64(((uint64_t)ICE_TX_DESC_CMD_RS) <<
3097                                          ICE_TXD_QW1_CMD_S);
3098                 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
3099                 txq->tx_tail = 0;
3100         }
3101
3102         /* Fill hardware descriptor ring with mbuf data */
3103         ice_tx_fill_hw_ring(txq, tx_pkts + n, (uint16_t)(nb_pkts - n));
3104         txq->tx_tail = (uint16_t)(txq->tx_tail + (nb_pkts - n));
3105
3106         /* Determin if RS bit needs to be set */
3107         if (txq->tx_tail > txq->tx_next_rs) {
3108                 txr[txq->tx_next_rs].cmd_type_offset_bsz |=
3109                         rte_cpu_to_le_64(((uint64_t)ICE_TX_DESC_CMD_RS) <<
3110                                          ICE_TXD_QW1_CMD_S);
3111                 txq->tx_next_rs =
3112                         (uint16_t)(txq->tx_next_rs + txq->tx_rs_thresh);
3113                 if (txq->tx_next_rs >= txq->nb_tx_desc)
3114                         txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
3115         }
3116
3117         if (txq->tx_tail >= txq->nb_tx_desc)
3118                 txq->tx_tail = 0;
3119
3120         /* Update the tx tail register */
3121         ICE_PCI_REG_WC_WRITE(txq->qtx_tail, txq->tx_tail);
3122
3123         return nb_pkts;
3124 }
3125
3126 static uint16_t
3127 ice_xmit_pkts_simple(void *tx_queue,
3128                      struct rte_mbuf **tx_pkts,
3129                      uint16_t nb_pkts)
3130 {
3131         uint16_t nb_tx = 0;
3132
3133         if (likely(nb_pkts <= ICE_TX_MAX_BURST))
3134                 return tx_xmit_pkts((struct ice_tx_queue *)tx_queue,
3135                                     tx_pkts, nb_pkts);
3136
3137         while (nb_pkts) {
3138                 uint16_t ret, num = (uint16_t)RTE_MIN(nb_pkts,
3139                                                       ICE_TX_MAX_BURST);
3140
3141                 ret = tx_xmit_pkts((struct ice_tx_queue *)tx_queue,
3142                                    &tx_pkts[nb_tx], num);
3143                 nb_tx = (uint16_t)(nb_tx + ret);
3144                 nb_pkts = (uint16_t)(nb_pkts - ret);
3145                 if (ret < num)
3146                         break;
3147         }
3148
3149         return nb_tx;
3150 }
3151
3152 void __rte_cold
3153 ice_set_rx_function(struct rte_eth_dev *dev)
3154 {
3155         PMD_INIT_FUNC_TRACE();
3156         struct ice_adapter *ad =
3157                 ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
3158 #ifdef RTE_ARCH_X86
3159         struct ice_rx_queue *rxq;
3160         int i;
3161         int rx_check_ret = -1;
3162
3163         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
3164                 ad->rx_use_avx512 = false;
3165                 ad->rx_use_avx2 = false;
3166                 rx_check_ret = ice_rx_vec_dev_check(dev);
3167                 if (ad->ptp_ena)
3168                         rx_check_ret = -1;
3169                 if (rx_check_ret >= 0 && ad->rx_bulk_alloc_allowed &&
3170                     rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) {
3171                         ad->rx_vec_allowed = true;
3172                         for (i = 0; i < dev->data->nb_rx_queues; i++) {
3173                                 rxq = dev->data->rx_queues[i];
3174                                 if (rxq && ice_rxq_vec_setup(rxq)) {
3175                                         ad->rx_vec_allowed = false;
3176                                         break;
3177                                 }
3178                         }
3179
3180                         if (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_512 &&
3181                         rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1 &&
3182                         rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) == 1)
3183 #ifdef CC_AVX512_SUPPORT
3184                                 ad->rx_use_avx512 = true;
3185 #else
3186                         PMD_DRV_LOG(NOTICE,
3187                                 "AVX512 is not supported in build env");
3188 #endif
3189                         if (!ad->rx_use_avx512 &&
3190                         (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) == 1 ||
3191                         rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1) &&
3192                         rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_256)
3193                                 ad->rx_use_avx2 = true;
3194
3195                 } else {
3196                         ad->rx_vec_allowed = false;
3197                 }
3198         }
3199
3200         if (ad->rx_vec_allowed) {
3201                 if (dev->data->scattered_rx) {
3202                         if (ad->rx_use_avx512) {
3203 #ifdef CC_AVX512_SUPPORT
3204                                 if (rx_check_ret == ICE_VECTOR_OFFLOAD_PATH) {
3205                                         PMD_DRV_LOG(NOTICE,
3206                                                 "Using AVX512 OFFLOAD Vector Scattered Rx (port %d).",
3207                                                 dev->data->port_id);
3208                                         dev->rx_pkt_burst =
3209                                                 ice_recv_scattered_pkts_vec_avx512_offload;
3210                                 } else {
3211                                         PMD_DRV_LOG(NOTICE,
3212                                                 "Using AVX512 Vector Scattered Rx (port %d).",
3213                                                 dev->data->port_id);
3214                                         dev->rx_pkt_burst =
3215                                                 ice_recv_scattered_pkts_vec_avx512;
3216                                 }
3217 #endif
3218                         } else if (ad->rx_use_avx2) {
3219                                 if (rx_check_ret == ICE_VECTOR_OFFLOAD_PATH) {
3220                                         PMD_DRV_LOG(NOTICE,
3221                                                     "Using AVX2 OFFLOAD Vector Scattered Rx (port %d).",
3222                                                     dev->data->port_id);
3223                                         dev->rx_pkt_burst =
3224                                                 ice_recv_scattered_pkts_vec_avx2_offload;
3225                                 } else {
3226                                         PMD_DRV_LOG(NOTICE,
3227                                                     "Using AVX2 Vector Scattered Rx (port %d).",
3228                                                     dev->data->port_id);
3229                                         dev->rx_pkt_burst =
3230                                                 ice_recv_scattered_pkts_vec_avx2;
3231                                 }
3232                         } else {
3233                                 PMD_DRV_LOG(DEBUG,
3234                                         "Using Vector Scattered Rx (port %d).",
3235                                         dev->data->port_id);
3236                                 dev->rx_pkt_burst = ice_recv_scattered_pkts_vec;
3237                         }
3238                 } else {
3239                         if (ad->rx_use_avx512) {
3240 #ifdef CC_AVX512_SUPPORT
3241                                 if (rx_check_ret == ICE_VECTOR_OFFLOAD_PATH) {
3242                                         PMD_DRV_LOG(NOTICE,
3243                                                 "Using AVX512 OFFLOAD Vector Rx (port %d).",
3244                                                 dev->data->port_id);
3245                                         dev->rx_pkt_burst =
3246                                                 ice_recv_pkts_vec_avx512_offload;
3247                                 } else {
3248                                         PMD_DRV_LOG(NOTICE,
3249                                                 "Using AVX512 Vector Rx (port %d).",
3250                                                 dev->data->port_id);
3251                                         dev->rx_pkt_burst =
3252                                                 ice_recv_pkts_vec_avx512;
3253                                 }
3254 #endif
3255                         } else if (ad->rx_use_avx2) {
3256                                 if (rx_check_ret == ICE_VECTOR_OFFLOAD_PATH) {
3257                                         PMD_DRV_LOG(NOTICE,
3258                                                     "Using AVX2 OFFLOAD Vector Rx (port %d).",
3259                                                     dev->data->port_id);
3260                                         dev->rx_pkt_burst =
3261                                                 ice_recv_pkts_vec_avx2_offload;
3262                                 } else {
3263                                         PMD_DRV_LOG(NOTICE,
3264                                                     "Using AVX2 Vector Rx (port %d).",
3265                                                     dev->data->port_id);
3266                                         dev->rx_pkt_burst =
3267                                                 ice_recv_pkts_vec_avx2;
3268                                 }
3269                         } else {
3270                                 PMD_DRV_LOG(DEBUG,
3271                                         "Using Vector Rx (port %d).",
3272                                         dev->data->port_id);
3273                                 dev->rx_pkt_burst = ice_recv_pkts_vec;
3274                         }
3275                 }
3276                 return;
3277         }
3278
3279 #endif
3280
3281         if (dev->data->scattered_rx) {
3282                 /* Set the non-LRO scattered function */
3283                 PMD_INIT_LOG(DEBUG,
3284                              "Using a Scattered function on port %d.",
3285                              dev->data->port_id);
3286                 dev->rx_pkt_burst = ice_recv_scattered_pkts;
3287         } else if (ad->rx_bulk_alloc_allowed) {
3288                 PMD_INIT_LOG(DEBUG,
3289                              "Rx Burst Bulk Alloc Preconditions are "
3290                              "satisfied. Rx Burst Bulk Alloc function "
3291                              "will be used on port %d.",
3292                              dev->data->port_id);
3293                 dev->rx_pkt_burst = ice_recv_pkts_bulk_alloc;
3294         } else {
3295                 PMD_INIT_LOG(DEBUG,
3296                              "Rx Burst Bulk Alloc Preconditions are not "
3297                              "satisfied, Normal Rx will be used on port %d.",
3298                              dev->data->port_id);
3299                 dev->rx_pkt_burst = ice_recv_pkts;
3300         }
3301 }
3302
3303 static const struct {
3304         eth_rx_burst_t pkt_burst;
3305         const char *info;
3306 } ice_rx_burst_infos[] = {
3307         { ice_recv_scattered_pkts,          "Scalar Scattered" },
3308         { ice_recv_pkts_bulk_alloc,         "Scalar Bulk Alloc" },
3309         { ice_recv_pkts,                    "Scalar" },
3310 #ifdef RTE_ARCH_X86
3311 #ifdef CC_AVX512_SUPPORT
3312         { ice_recv_scattered_pkts_vec_avx512, "Vector AVX512 Scattered" },
3313         { ice_recv_scattered_pkts_vec_avx512_offload, "Offload Vector AVX512 Scattered" },
3314         { ice_recv_pkts_vec_avx512,           "Vector AVX512" },
3315         { ice_recv_pkts_vec_avx512_offload,   "Offload Vector AVX512" },
3316 #endif
3317         { ice_recv_scattered_pkts_vec_avx2, "Vector AVX2 Scattered" },
3318         { ice_recv_scattered_pkts_vec_avx2_offload, "Offload Vector AVX2 Scattered" },
3319         { ice_recv_pkts_vec_avx2,           "Vector AVX2" },
3320         { ice_recv_pkts_vec_avx2_offload,   "Offload Vector AVX2" },
3321         { ice_recv_scattered_pkts_vec,      "Vector SSE Scattered" },
3322         { ice_recv_pkts_vec,                "Vector SSE" },
3323 #endif
3324 };
3325
3326 int
3327 ice_rx_burst_mode_get(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id,
3328                       struct rte_eth_burst_mode *mode)
3329 {
3330         eth_rx_burst_t pkt_burst = dev->rx_pkt_burst;
3331         int ret = -EINVAL;
3332         unsigned int i;
3333
3334         for (i = 0; i < RTE_DIM(ice_rx_burst_infos); ++i) {
3335                 if (pkt_burst == ice_rx_burst_infos[i].pkt_burst) {
3336                         snprintf(mode->info, sizeof(mode->info), "%s",
3337                                  ice_rx_burst_infos[i].info);
3338                         ret = 0;
3339                         break;
3340                 }
3341         }
3342
3343         return ret;
3344 }
3345
3346 void __rte_cold
3347 ice_set_tx_function_flag(struct rte_eth_dev *dev, struct ice_tx_queue *txq)
3348 {
3349         struct ice_adapter *ad =
3350                 ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
3351
3352         /* Use a simple Tx queue if possible (only fast free is allowed) */
3353         ad->tx_simple_allowed =
3354                 (txq->offloads ==
3355                 (txq->offloads & DEV_TX_OFFLOAD_MBUF_FAST_FREE) &&
3356                 txq->tx_rs_thresh >= ICE_TX_MAX_BURST);
3357
3358         if (ad->tx_simple_allowed)
3359                 PMD_INIT_LOG(DEBUG, "Simple Tx can be enabled on Tx queue %u.",
3360                              txq->queue_id);
3361         else
3362                 PMD_INIT_LOG(DEBUG,
3363                              "Simple Tx can NOT be enabled on Tx queue %u.",
3364                              txq->queue_id);
3365 }
3366
3367 /*********************************************************************
3368  *
3369  *  TX prep functions
3370  *
3371  **********************************************************************/
3372 /* The default values of TSO MSS */
3373 #define ICE_MIN_TSO_MSS            64
3374 #define ICE_MAX_TSO_MSS            9728
3375 #define ICE_MAX_TSO_FRAME_SIZE     262144
3376 uint16_t
3377 ice_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts,
3378               uint16_t nb_pkts)
3379 {
3380         int i, ret;
3381         uint64_t ol_flags;
3382         struct rte_mbuf *m;
3383
3384         for (i = 0; i < nb_pkts; i++) {
3385                 m = tx_pkts[i];
3386                 ol_flags = m->ol_flags;
3387
3388                 if (ol_flags & PKT_TX_TCP_SEG &&
3389                     (m->tso_segsz < ICE_MIN_TSO_MSS ||
3390                      m->tso_segsz > ICE_MAX_TSO_MSS ||
3391                      m->pkt_len > ICE_MAX_TSO_FRAME_SIZE)) {
3392                         /**
3393                          * MSS outside the range are considered malicious
3394                          */
3395                         rte_errno = EINVAL;
3396                         return i;
3397                 }
3398
3399 #ifdef RTE_ETHDEV_DEBUG_TX
3400                 ret = rte_validate_tx_offload(m);
3401                 if (ret != 0) {
3402                         rte_errno = -ret;
3403                         return i;
3404                 }
3405 #endif
3406                 ret = rte_net_intel_cksum_prepare(m);
3407                 if (ret != 0) {
3408                         rte_errno = -ret;
3409                         return i;
3410                 }
3411         }
3412         return i;
3413 }
3414
3415 void __rte_cold
3416 ice_set_tx_function(struct rte_eth_dev *dev)
3417 {
3418         struct ice_adapter *ad =
3419                 ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
3420 #ifdef RTE_ARCH_X86
3421         struct ice_tx_queue *txq;
3422         int i;
3423         int tx_check_ret = -1;
3424
3425         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
3426                 ad->tx_use_avx2 = false;
3427                 ad->tx_use_avx512 = false;
3428                 tx_check_ret = ice_tx_vec_dev_check(dev);
3429                 if (tx_check_ret >= 0 &&
3430                     rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) {
3431                         ad->tx_vec_allowed = true;
3432
3433                         if (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_512 &&
3434                         rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1 &&
3435                         rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) == 1)
3436 #ifdef CC_AVX512_SUPPORT
3437                                 ad->tx_use_avx512 = true;
3438 #else
3439                         PMD_DRV_LOG(NOTICE,
3440                                 "AVX512 is not supported in build env");
3441 #endif
3442                         if (!ad->tx_use_avx512 &&
3443                                 (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) == 1 ||
3444                                 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1) &&
3445                                 rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_256)
3446                                 ad->tx_use_avx2 = true;
3447
3448                         if (!ad->tx_use_avx2 && !ad->tx_use_avx512 &&
3449                                 tx_check_ret == ICE_VECTOR_OFFLOAD_PATH)
3450                                 ad->tx_vec_allowed = false;
3451
3452                         if (ad->tx_vec_allowed) {
3453                                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
3454                                         txq = dev->data->tx_queues[i];
3455                                         if (txq && ice_txq_vec_setup(txq)) {
3456                                                 ad->tx_vec_allowed = false;
3457                                                 break;
3458                                         }
3459                                 }
3460                         }
3461                 } else {
3462                         ad->tx_vec_allowed = false;
3463                 }
3464         }
3465
3466         if (ad->tx_vec_allowed) {
3467                 dev->tx_pkt_prepare = NULL;
3468                 if (ad->tx_use_avx512) {
3469 #ifdef CC_AVX512_SUPPORT
3470                         if (tx_check_ret == ICE_VECTOR_OFFLOAD_PATH) {
3471                                 PMD_DRV_LOG(NOTICE,
3472                                             "Using AVX512 OFFLOAD Vector Tx (port %d).",
3473                                             dev->data->port_id);
3474                                 dev->tx_pkt_burst =
3475                                         ice_xmit_pkts_vec_avx512_offload;
3476                                 dev->tx_pkt_prepare = ice_prep_pkts;
3477                         } else {
3478                                 PMD_DRV_LOG(NOTICE,
3479                                             "Using AVX512 Vector Tx (port %d).",
3480                                             dev->data->port_id);
3481                                 dev->tx_pkt_burst = ice_xmit_pkts_vec_avx512;
3482                         }
3483 #endif
3484                 } else {
3485                         if (tx_check_ret == ICE_VECTOR_OFFLOAD_PATH) {
3486                                 PMD_DRV_LOG(NOTICE,
3487                                             "Using AVX2 OFFLOAD Vector Tx (port %d).",
3488                                             dev->data->port_id);
3489                                 dev->tx_pkt_burst =
3490                                         ice_xmit_pkts_vec_avx2_offload;
3491                                 dev->tx_pkt_prepare = ice_prep_pkts;
3492                         } else {
3493                                 PMD_DRV_LOG(DEBUG, "Using %sVector Tx (port %d).",
3494                                             ad->tx_use_avx2 ? "avx2 " : "",
3495                                             dev->data->port_id);
3496                                 dev->tx_pkt_burst = ad->tx_use_avx2 ?
3497                                                     ice_xmit_pkts_vec_avx2 :
3498                                                     ice_xmit_pkts_vec;
3499                         }
3500                 }
3501
3502                 return;
3503         }
3504 #endif
3505
3506         if (ad->tx_simple_allowed) {
3507                 PMD_INIT_LOG(DEBUG, "Simple tx finally be used.");
3508                 dev->tx_pkt_burst = ice_xmit_pkts_simple;
3509                 dev->tx_pkt_prepare = NULL;
3510         } else {
3511                 PMD_INIT_LOG(DEBUG, "Normal tx finally be used.");
3512                 dev->tx_pkt_burst = ice_xmit_pkts;
3513                 dev->tx_pkt_prepare = ice_prep_pkts;
3514         }
3515 }
3516
3517 static const struct {
3518         eth_tx_burst_t pkt_burst;
3519         const char *info;
3520 } ice_tx_burst_infos[] = {
3521         { ice_xmit_pkts_simple,   "Scalar Simple" },
3522         { ice_xmit_pkts,          "Scalar" },
3523 #ifdef RTE_ARCH_X86
3524 #ifdef CC_AVX512_SUPPORT
3525         { ice_xmit_pkts_vec_avx512, "Vector AVX512" },
3526         { ice_xmit_pkts_vec_avx512_offload, "Offload Vector AVX512" },
3527 #endif
3528         { ice_xmit_pkts_vec_avx2, "Vector AVX2" },
3529         { ice_xmit_pkts_vec,      "Vector SSE" },
3530 #endif
3531 };
3532
3533 int
3534 ice_tx_burst_mode_get(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id,
3535                       struct rte_eth_burst_mode *mode)
3536 {
3537         eth_tx_burst_t pkt_burst = dev->tx_pkt_burst;
3538         int ret = -EINVAL;
3539         unsigned int i;
3540
3541         for (i = 0; i < RTE_DIM(ice_tx_burst_infos); ++i) {
3542                 if (pkt_burst == ice_tx_burst_infos[i].pkt_burst) {
3543                         snprintf(mode->info, sizeof(mode->info), "%s",
3544                                  ice_tx_burst_infos[i].info);
3545                         ret = 0;
3546                         break;
3547                 }
3548         }
3549
3550         return ret;
3551 }
3552
3553 /* For each value it means, datasheet of hardware can tell more details
3554  *
3555  * @note: fix ice_dev_supported_ptypes_get() if any change here.
3556  */
3557 static inline uint32_t
3558 ice_get_default_pkt_type(uint16_t ptype)
3559 {
3560         static const uint32_t type_table[ICE_MAX_PKT_TYPE]
3561                 __rte_cache_aligned = {
3562                 /* L2 types */
3563                 /* [0] reserved */
3564                 [1] = RTE_PTYPE_L2_ETHER,
3565                 [2] = RTE_PTYPE_L2_ETHER_TIMESYNC,
3566                 /* [3] - [5] reserved */
3567                 [6] = RTE_PTYPE_L2_ETHER_LLDP,
3568                 /* [7] - [10] reserved */
3569                 [11] = RTE_PTYPE_L2_ETHER_ARP,
3570                 /* [12] - [21] reserved */
3571
3572                 /* Non tunneled IPv4 */
3573                 [22] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3574                        RTE_PTYPE_L4_FRAG,
3575                 [23] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3576                        RTE_PTYPE_L4_NONFRAG,
3577                 [24] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3578                        RTE_PTYPE_L4_UDP,
3579                 /* [25] reserved */
3580                 [26] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3581                        RTE_PTYPE_L4_TCP,
3582                 [27] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3583                        RTE_PTYPE_L4_SCTP,
3584                 [28] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3585                        RTE_PTYPE_L4_ICMP,
3586
3587                 /* IPv4 --> IPv4 */
3588                 [29] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3589                        RTE_PTYPE_TUNNEL_IP |
3590                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3591                        RTE_PTYPE_INNER_L4_FRAG,
3592                 [30] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3593                        RTE_PTYPE_TUNNEL_IP |
3594                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3595                        RTE_PTYPE_INNER_L4_NONFRAG,
3596                 [31] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3597                        RTE_PTYPE_TUNNEL_IP |
3598                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3599                        RTE_PTYPE_INNER_L4_UDP,
3600                 /* [32] reserved */
3601                 [33] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3602                        RTE_PTYPE_TUNNEL_IP |
3603                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3604                        RTE_PTYPE_INNER_L4_TCP,
3605                 [34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3606                        RTE_PTYPE_TUNNEL_IP |
3607                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3608                        RTE_PTYPE_INNER_L4_SCTP,
3609                 [35] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3610                        RTE_PTYPE_TUNNEL_IP |
3611                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3612                        RTE_PTYPE_INNER_L4_ICMP,
3613
3614                 /* IPv4 --> IPv6 */
3615                 [36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3616                        RTE_PTYPE_TUNNEL_IP |
3617                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3618                        RTE_PTYPE_INNER_L4_FRAG,
3619                 [37] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3620                        RTE_PTYPE_TUNNEL_IP |
3621                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3622                        RTE_PTYPE_INNER_L4_NONFRAG,
3623                 [38] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3624                        RTE_PTYPE_TUNNEL_IP |
3625                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3626                        RTE_PTYPE_INNER_L4_UDP,
3627                 /* [39] reserved */
3628                 [40] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3629                        RTE_PTYPE_TUNNEL_IP |
3630                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3631                        RTE_PTYPE_INNER_L4_TCP,
3632                 [41] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3633                        RTE_PTYPE_TUNNEL_IP |
3634                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3635                        RTE_PTYPE_INNER_L4_SCTP,
3636                 [42] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3637                        RTE_PTYPE_TUNNEL_IP |
3638                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3639                        RTE_PTYPE_INNER_L4_ICMP,
3640
3641                 /* IPv4 --> GRE/Teredo/VXLAN */
3642                 [43] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3643                        RTE_PTYPE_TUNNEL_GRENAT,
3644
3645                 /* IPv4 --> GRE/Teredo/VXLAN --> IPv4 */
3646                 [44] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3647                        RTE_PTYPE_TUNNEL_GRENAT |
3648                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3649                        RTE_PTYPE_INNER_L4_FRAG,
3650                 [45] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3651                        RTE_PTYPE_TUNNEL_GRENAT |
3652                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3653                        RTE_PTYPE_INNER_L4_NONFRAG,
3654                 [46] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3655                        RTE_PTYPE_TUNNEL_GRENAT |
3656                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3657                        RTE_PTYPE_INNER_L4_UDP,
3658                 /* [47] reserved */
3659                 [48] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3660                        RTE_PTYPE_TUNNEL_GRENAT |
3661                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3662                        RTE_PTYPE_INNER_L4_TCP,
3663                 [49] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3664                        RTE_PTYPE_TUNNEL_GRENAT |
3665                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3666                        RTE_PTYPE_INNER_L4_SCTP,
3667                 [50] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3668                        RTE_PTYPE_TUNNEL_GRENAT |
3669                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3670                        RTE_PTYPE_INNER_L4_ICMP,
3671
3672                 /* IPv4 --> GRE/Teredo/VXLAN --> IPv6 */
3673                 [51] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3674                        RTE_PTYPE_TUNNEL_GRENAT |
3675                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3676                        RTE_PTYPE_INNER_L4_FRAG,
3677                 [52] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3678                        RTE_PTYPE_TUNNEL_GRENAT |
3679                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3680                        RTE_PTYPE_INNER_L4_NONFRAG,
3681                 [53] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3682                        RTE_PTYPE_TUNNEL_GRENAT |
3683                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3684                        RTE_PTYPE_INNER_L4_UDP,
3685                 /* [54] reserved */
3686                 [55] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3687                        RTE_PTYPE_TUNNEL_GRENAT |
3688                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3689                        RTE_PTYPE_INNER_L4_TCP,
3690                 [56] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3691                        RTE_PTYPE_TUNNEL_GRENAT |
3692                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3693                        RTE_PTYPE_INNER_L4_SCTP,
3694                 [57] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3695                        RTE_PTYPE_TUNNEL_GRENAT |
3696                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3697                        RTE_PTYPE_INNER_L4_ICMP,
3698
3699                 /* IPv4 --> GRE/Teredo/VXLAN --> MAC */
3700                 [58] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3701                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER,
3702
3703                 /* IPv4 --> GRE/Teredo/VXLAN --> MAC --> IPv4 */
3704                 [59] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3705                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3706                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3707                        RTE_PTYPE_INNER_L4_FRAG,
3708                 [60] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3709                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3710                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3711                        RTE_PTYPE_INNER_L4_NONFRAG,
3712                 [61] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3713                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3714                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3715                        RTE_PTYPE_INNER_L4_UDP,
3716                 /* [62] reserved */
3717                 [63] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3718                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3719                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3720                        RTE_PTYPE_INNER_L4_TCP,
3721                 [64] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3722                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3723                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3724                        RTE_PTYPE_INNER_L4_SCTP,
3725                 [65] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3726                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3727                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3728                        RTE_PTYPE_INNER_L4_ICMP,
3729
3730                 /* IPv4 --> GRE/Teredo/VXLAN --> MAC --> IPv6 */
3731                 [66] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3732                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3733                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3734                        RTE_PTYPE_INNER_L4_FRAG,
3735                 [67] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3736                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3737                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3738                        RTE_PTYPE_INNER_L4_NONFRAG,
3739                 [68] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3740                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3741                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3742                        RTE_PTYPE_INNER_L4_UDP,
3743                 /* [69] reserved */
3744                 [70] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3745                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3746                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3747                        RTE_PTYPE_INNER_L4_TCP,
3748                 [71] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3749                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3750                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3751                        RTE_PTYPE_INNER_L4_SCTP,
3752                 [72] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3753                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3754                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3755                        RTE_PTYPE_INNER_L4_ICMP,
3756                 /* [73] - [87] reserved */
3757
3758                 /* Non tunneled IPv6 */
3759                 [88] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3760                        RTE_PTYPE_L4_FRAG,
3761                 [89] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3762                        RTE_PTYPE_L4_NONFRAG,
3763                 [90] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3764                        RTE_PTYPE_L4_UDP,
3765                 /* [91] reserved */
3766                 [92] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3767                        RTE_PTYPE_L4_TCP,
3768                 [93] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3769                        RTE_PTYPE_L4_SCTP,
3770                 [94] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3771                        RTE_PTYPE_L4_ICMP,
3772
3773                 /* IPv6 --> IPv4 */
3774                 [95] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3775                        RTE_PTYPE_TUNNEL_IP |
3776                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3777                        RTE_PTYPE_INNER_L4_FRAG,
3778                 [96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3779                        RTE_PTYPE_TUNNEL_IP |
3780                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3781                        RTE_PTYPE_INNER_L4_NONFRAG,
3782                 [97] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3783                        RTE_PTYPE_TUNNEL_IP |
3784                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3785                        RTE_PTYPE_INNER_L4_UDP,
3786                 /* [98] reserved */
3787                 [99] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3788                        RTE_PTYPE_TUNNEL_IP |
3789                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3790                        RTE_PTYPE_INNER_L4_TCP,
3791                 [100] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3792                         RTE_PTYPE_TUNNEL_IP |
3793                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3794                         RTE_PTYPE_INNER_L4_SCTP,
3795                 [101] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3796                         RTE_PTYPE_TUNNEL_IP |
3797                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3798                         RTE_PTYPE_INNER_L4_ICMP,
3799
3800                 /* IPv6 --> IPv6 */
3801                 [102] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3802                         RTE_PTYPE_TUNNEL_IP |
3803                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3804                         RTE_PTYPE_INNER_L4_FRAG,
3805                 [103] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3806                         RTE_PTYPE_TUNNEL_IP |
3807                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3808                         RTE_PTYPE_INNER_L4_NONFRAG,
3809                 [104] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3810                         RTE_PTYPE_TUNNEL_IP |
3811                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3812                         RTE_PTYPE_INNER_L4_UDP,
3813                 /* [105] reserved */
3814                 [106] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3815                         RTE_PTYPE_TUNNEL_IP |
3816                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3817                         RTE_PTYPE_INNER_L4_TCP,
3818                 [107] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3819                         RTE_PTYPE_TUNNEL_IP |
3820                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3821                         RTE_PTYPE_INNER_L4_SCTP,
3822                 [108] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3823                         RTE_PTYPE_TUNNEL_IP |
3824                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3825                         RTE_PTYPE_INNER_L4_ICMP,
3826
3827                 /* IPv6 --> GRE/Teredo/VXLAN */
3828                 [109] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3829                         RTE_PTYPE_TUNNEL_GRENAT,
3830
3831                 /* IPv6 --> GRE/Teredo/VXLAN --> IPv4 */
3832                 [110] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3833                         RTE_PTYPE_TUNNEL_GRENAT |
3834                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3835                         RTE_PTYPE_INNER_L4_FRAG,
3836                 [111] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3837                         RTE_PTYPE_TUNNEL_GRENAT |
3838                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3839                         RTE_PTYPE_INNER_L4_NONFRAG,
3840                 [112] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3841                         RTE_PTYPE_TUNNEL_GRENAT |
3842                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3843                         RTE_PTYPE_INNER_L4_UDP,
3844                 /* [113] reserved */
3845                 [114] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3846                         RTE_PTYPE_TUNNEL_GRENAT |
3847                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3848                         RTE_PTYPE_INNER_L4_TCP,
3849                 [115] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3850                         RTE_PTYPE_TUNNEL_GRENAT |
3851                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3852                         RTE_PTYPE_INNER_L4_SCTP,
3853                 [116] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3854                         RTE_PTYPE_TUNNEL_GRENAT |
3855                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3856                         RTE_PTYPE_INNER_L4_ICMP,
3857
3858                 /* IPv6 --> GRE/Teredo/VXLAN --> IPv6 */
3859                 [117] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3860                         RTE_PTYPE_TUNNEL_GRENAT |
3861                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3862                         RTE_PTYPE_INNER_L4_FRAG,
3863                 [118] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3864                         RTE_PTYPE_TUNNEL_GRENAT |
3865                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3866                         RTE_PTYPE_INNER_L4_NONFRAG,
3867                 [119] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3868                         RTE_PTYPE_TUNNEL_GRENAT |
3869                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3870                         RTE_PTYPE_INNER_L4_UDP,
3871                 /* [120] reserved */
3872                 [121] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3873                         RTE_PTYPE_TUNNEL_GRENAT |
3874                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3875                         RTE_PTYPE_INNER_L4_TCP,
3876                 [122] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3877                         RTE_PTYPE_TUNNEL_GRENAT |
3878                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3879                         RTE_PTYPE_INNER_L4_SCTP,
3880                 [123] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3881                         RTE_PTYPE_TUNNEL_GRENAT |
3882                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3883                         RTE_PTYPE_INNER_L4_ICMP,
3884
3885                 /* IPv6 --> GRE/Teredo/VXLAN --> MAC */
3886                 [124] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3887                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER,
3888
3889                 /* IPv6 --> GRE/Teredo/VXLAN --> MAC --> IPv4 */
3890                 [125] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3891                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3892                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3893                         RTE_PTYPE_INNER_L4_FRAG,
3894                 [126] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3895                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3896                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3897                         RTE_PTYPE_INNER_L4_NONFRAG,
3898                 [127] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3899                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3900                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3901                         RTE_PTYPE_INNER_L4_UDP,
3902                 /* [128] reserved */
3903                 [129] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3904                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3905                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3906                         RTE_PTYPE_INNER_L4_TCP,
3907                 [130] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3908                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3909                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3910                         RTE_PTYPE_INNER_L4_SCTP,
3911                 [131] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3912                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3913                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3914                         RTE_PTYPE_INNER_L4_ICMP,
3915
3916                 /* IPv6 --> GRE/Teredo/VXLAN --> MAC --> IPv6 */
3917                 [132] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3918                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3919                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3920                         RTE_PTYPE_INNER_L4_FRAG,
3921                 [133] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3922                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3923                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3924                         RTE_PTYPE_INNER_L4_NONFRAG,
3925                 [134] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3926                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3927                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3928                         RTE_PTYPE_INNER_L4_UDP,
3929                 /* [135] reserved */
3930                 [136] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3931                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3932                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3933                         RTE_PTYPE_INNER_L4_TCP,
3934                 [137] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3935                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3936                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3937                         RTE_PTYPE_INNER_L4_SCTP,
3938                 [138] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3939                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3940                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3941                         RTE_PTYPE_INNER_L4_ICMP,
3942                 /* [139] - [299] reserved */
3943
3944                 /* PPPoE */
3945                 [300] = RTE_PTYPE_L2_ETHER_PPPOE,
3946                 [301] = RTE_PTYPE_L2_ETHER_PPPOE,
3947
3948                 /* PPPoE --> IPv4 */
3949                 [302] = RTE_PTYPE_L2_ETHER_PPPOE |
3950                         RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3951                         RTE_PTYPE_L4_FRAG,
3952                 [303] = RTE_PTYPE_L2_ETHER_PPPOE |
3953                         RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3954                         RTE_PTYPE_L4_NONFRAG,
3955                 [304] = RTE_PTYPE_L2_ETHER_PPPOE |
3956                         RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3957                         RTE_PTYPE_L4_UDP,
3958                 [305] = RTE_PTYPE_L2_ETHER_PPPOE |
3959                         RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3960                         RTE_PTYPE_L4_TCP,
3961                 [306] = RTE_PTYPE_L2_ETHER_PPPOE |
3962                         RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3963                         RTE_PTYPE_L4_SCTP,
3964                 [307] = RTE_PTYPE_L2_ETHER_PPPOE |
3965                         RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3966                         RTE_PTYPE_L4_ICMP,
3967
3968                 /* PPPoE --> IPv6 */
3969                 [308] = RTE_PTYPE_L2_ETHER_PPPOE |
3970                         RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3971                         RTE_PTYPE_L4_FRAG,
3972                 [309] = RTE_PTYPE_L2_ETHER_PPPOE |
3973                         RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3974                         RTE_PTYPE_L4_NONFRAG,
3975                 [310] = RTE_PTYPE_L2_ETHER_PPPOE |
3976                         RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3977                         RTE_PTYPE_L4_UDP,
3978                 [311] = RTE_PTYPE_L2_ETHER_PPPOE |
3979                         RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3980                         RTE_PTYPE_L4_TCP,
3981                 [312] = RTE_PTYPE_L2_ETHER_PPPOE |
3982                         RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3983                         RTE_PTYPE_L4_SCTP,
3984                 [313] = RTE_PTYPE_L2_ETHER_PPPOE |
3985                         RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3986                         RTE_PTYPE_L4_ICMP,
3987                 /* [314] - [324] reserved */
3988
3989                 /* IPv4/IPv6 --> GTPC/GTPU */
3990                 [325] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3991                         RTE_PTYPE_TUNNEL_GTPC,
3992                 [326] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3993                         RTE_PTYPE_TUNNEL_GTPC,
3994                 [327] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3995                         RTE_PTYPE_TUNNEL_GTPC,
3996                 [328] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3997                         RTE_PTYPE_TUNNEL_GTPC,
3998                 [329] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3999                         RTE_PTYPE_TUNNEL_GTPU,
4000                 [330] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4001                         RTE_PTYPE_TUNNEL_GTPU,
4002
4003                 /* IPv4 --> GTPU --> IPv4 */
4004                 [331] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4005                         RTE_PTYPE_TUNNEL_GTPU |
4006                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
4007                         RTE_PTYPE_INNER_L4_FRAG,
4008                 [332] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4009                         RTE_PTYPE_TUNNEL_GTPU |
4010                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
4011                         RTE_PTYPE_INNER_L4_NONFRAG,
4012                 [333] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4013                         RTE_PTYPE_TUNNEL_GTPU |
4014                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
4015                         RTE_PTYPE_INNER_L4_UDP,
4016                 [334] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4017                         RTE_PTYPE_TUNNEL_GTPU |
4018                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
4019                         RTE_PTYPE_INNER_L4_TCP,
4020                 [335] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4021                         RTE_PTYPE_TUNNEL_GTPU |
4022                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
4023                         RTE_PTYPE_INNER_L4_ICMP,
4024
4025                 /* IPv6 --> GTPU --> IPv4 */
4026                 [336] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4027                         RTE_PTYPE_TUNNEL_GTPU |
4028                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
4029                         RTE_PTYPE_INNER_L4_FRAG,
4030                 [337] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4031                         RTE_PTYPE_TUNNEL_GTPU |
4032                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
4033                         RTE_PTYPE_INNER_L4_NONFRAG,
4034                 [338] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4035                         RTE_PTYPE_TUNNEL_GTPU |
4036                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
4037                         RTE_PTYPE_INNER_L4_UDP,
4038                 [339] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4039                         RTE_PTYPE_TUNNEL_GTPU |
4040                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
4041                         RTE_PTYPE_INNER_L4_TCP,
4042                 [340] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4043                         RTE_PTYPE_TUNNEL_GTPU |
4044                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
4045                         RTE_PTYPE_INNER_L4_ICMP,
4046
4047                 /* IPv4 --> GTPU --> IPv6 */
4048                 [341] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4049                         RTE_PTYPE_TUNNEL_GTPU |
4050                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
4051                         RTE_PTYPE_INNER_L4_FRAG,
4052                 [342] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4053                         RTE_PTYPE_TUNNEL_GTPU |
4054                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
4055                         RTE_PTYPE_INNER_L4_NONFRAG,
4056                 [343] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4057                         RTE_PTYPE_TUNNEL_GTPU |
4058                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
4059                         RTE_PTYPE_INNER_L4_UDP,
4060                 [344] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4061                         RTE_PTYPE_TUNNEL_GTPU |
4062                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
4063                         RTE_PTYPE_INNER_L4_TCP,
4064                 [345] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4065                         RTE_PTYPE_TUNNEL_GTPU |
4066                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
4067                         RTE_PTYPE_INNER_L4_ICMP,
4068
4069                 /* IPv6 --> GTPU --> IPv6 */
4070                 [346] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4071                         RTE_PTYPE_TUNNEL_GTPU |
4072                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
4073                         RTE_PTYPE_INNER_L4_FRAG,
4074                 [347] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4075                         RTE_PTYPE_TUNNEL_GTPU |
4076                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
4077                         RTE_PTYPE_INNER_L4_NONFRAG,
4078                 [348] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4079                         RTE_PTYPE_TUNNEL_GTPU |
4080                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
4081                         RTE_PTYPE_INNER_L4_UDP,
4082                 [349] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4083                         RTE_PTYPE_TUNNEL_GTPU |
4084                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
4085                         RTE_PTYPE_INNER_L4_TCP,
4086                 [350] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4087                         RTE_PTYPE_TUNNEL_GTPU |
4088                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
4089                         RTE_PTYPE_INNER_L4_ICMP,
4090
4091                 /* IPv4 --> UDP ECPRI */
4092                 [372] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4093                         RTE_PTYPE_L4_UDP,
4094                 [373] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4095                         RTE_PTYPE_L4_UDP,
4096                 [374] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4097                         RTE_PTYPE_L4_UDP,
4098                 [375] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4099                         RTE_PTYPE_L4_UDP,
4100                 [376] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4101                         RTE_PTYPE_L4_UDP,
4102                 [377] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4103                         RTE_PTYPE_L4_UDP,
4104                 [378] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4105                         RTE_PTYPE_L4_UDP,
4106                 [379] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4107                         RTE_PTYPE_L4_UDP,
4108                 [380] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4109                         RTE_PTYPE_L4_UDP,
4110                 [381] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4111                         RTE_PTYPE_L4_UDP,
4112
4113                 /* IPV6 --> UDP ECPRI */
4114                 [382] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4115                         RTE_PTYPE_L4_UDP,
4116                 [383] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4117                         RTE_PTYPE_L4_UDP,
4118                 [384] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4119                         RTE_PTYPE_L4_UDP,
4120                 [385] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4121                         RTE_PTYPE_L4_UDP,
4122                 [386] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4123                         RTE_PTYPE_L4_UDP,
4124                 [387] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4125                         RTE_PTYPE_L4_UDP,
4126                 [388] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4127                         RTE_PTYPE_L4_UDP,
4128                 [389] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4129                         RTE_PTYPE_L4_UDP,
4130                 [390] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4131                         RTE_PTYPE_L4_UDP,
4132                 [391] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4133                         RTE_PTYPE_L4_UDP,
4134                 /* All others reserved */
4135         };
4136
4137         return type_table[ptype];
4138 }
4139
4140 void __rte_cold
4141 ice_set_default_ptype_table(struct rte_eth_dev *dev)
4142 {
4143         struct ice_adapter *ad =
4144                 ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
4145         int i;
4146
4147         for (i = 0; i < ICE_MAX_PKT_TYPE; i++)
4148                 ad->ptype_tbl[i] = ice_get_default_pkt_type(i);
4149 }
4150
4151 #define ICE_RX_PROG_STATUS_DESC_WB_QW1_PROGID_S 1
4152 #define ICE_RX_PROG_STATUS_DESC_WB_QW1_PROGID_M \
4153                         (0x3UL << ICE_RX_PROG_STATUS_DESC_WB_QW1_PROGID_S)
4154 #define ICE_RX_PROG_STATUS_DESC_WB_QW1_PROG_ADD 0
4155 #define ICE_RX_PROG_STATUS_DESC_WB_QW1_PROG_DEL 0x1
4156
4157 #define ICE_RX_PROG_STATUS_DESC_WB_QW1_FAIL_S   4
4158 #define ICE_RX_PROG_STATUS_DESC_WB_QW1_FAIL_M   \
4159         (1 << ICE_RX_PROG_STATUS_DESC_WB_QW1_FAIL_S)
4160 #define ICE_RX_PROG_STATUS_DESC_WB_QW1_FAIL_PROF_S      5
4161 #define ICE_RX_PROG_STATUS_DESC_WB_QW1_FAIL_PROF_M      \
4162         (1 << ICE_RX_PROG_STATUS_DESC_WB_QW1_FAIL_PROF_S)
4163
4164 /*
4165  * check the programming status descriptor in rx queue.
4166  * done after Programming Flow Director is programmed on
4167  * tx queue
4168  */
4169 static inline int
4170 ice_check_fdir_programming_status(struct ice_rx_queue *rxq)
4171 {
4172         volatile union ice_32byte_rx_desc *rxdp;
4173         uint64_t qword1;
4174         uint32_t rx_status;
4175         uint32_t error;
4176         uint32_t id;
4177         int ret = -EAGAIN;
4178
4179         rxdp = (volatile union ice_32byte_rx_desc *)
4180                 (&rxq->rx_ring[rxq->rx_tail]);
4181         qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len);
4182         rx_status = (qword1 & ICE_RXD_QW1_STATUS_M)
4183                         >> ICE_RXD_QW1_STATUS_S;
4184
4185         if (rx_status & (1 << ICE_RX_DESC_STATUS_DD_S)) {
4186                 ret = 0;
4187                 error = (qword1 & ICE_RX_PROG_STATUS_DESC_WB_QW1_FAIL_M) >>
4188                         ICE_RX_PROG_STATUS_DESC_WB_QW1_FAIL_S;
4189                 id = (qword1 & ICE_RX_PROG_STATUS_DESC_WB_QW1_PROGID_M) >>
4190                         ICE_RX_PROG_STATUS_DESC_WB_QW1_PROGID_S;
4191                 if (error) {
4192                         if (id == ICE_RX_PROG_STATUS_DESC_WB_QW1_PROG_ADD)
4193                                 PMD_DRV_LOG(ERR, "Failed to add FDIR rule.");
4194                         else if (id == ICE_RX_PROG_STATUS_DESC_WB_QW1_PROG_DEL)
4195                                 PMD_DRV_LOG(ERR, "Failed to remove FDIR rule.");
4196                         ret = -EINVAL;
4197                         goto err;
4198                 }
4199                 error = (qword1 & ICE_RX_PROG_STATUS_DESC_WB_QW1_FAIL_PROF_M) >>
4200                         ICE_RX_PROG_STATUS_DESC_WB_QW1_FAIL_PROF_S;
4201                 if (error) {
4202                         PMD_DRV_LOG(ERR, "Failed to create FDIR profile.");
4203                         ret = -EINVAL;
4204                 }
4205 err:
4206                 rxdp->wb.qword1.status_error_len = 0;
4207                 rxq->rx_tail++;
4208                 if (unlikely(rxq->rx_tail == rxq->nb_rx_desc))
4209                         rxq->rx_tail = 0;
4210                 if (rxq->rx_tail == 0)
4211                         ICE_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
4212                 else
4213                         ICE_PCI_REG_WRITE(rxq->qrx_tail, rxq->rx_tail - 1);
4214         }
4215
4216         return ret;
4217 }
4218
4219 #define ICE_FDIR_MAX_WAIT_US 10000
4220
4221 int
4222 ice_fdir_programming(struct ice_pf *pf, struct ice_fltr_desc *fdir_desc)
4223 {
4224         struct ice_tx_queue *txq = pf->fdir.txq;
4225         struct ice_rx_queue *rxq = pf->fdir.rxq;
4226         volatile struct ice_fltr_desc *fdirdp;
4227         volatile struct ice_tx_desc *txdp;
4228         uint32_t td_cmd;
4229         uint16_t i;
4230
4231         fdirdp = (volatile struct ice_fltr_desc *)
4232                 (&txq->tx_ring[txq->tx_tail]);
4233         fdirdp->qidx_compq_space_stat = fdir_desc->qidx_compq_space_stat;
4234         fdirdp->dtype_cmd_vsi_fdid = fdir_desc->dtype_cmd_vsi_fdid;
4235
4236         txdp = &txq->tx_ring[txq->tx_tail + 1];
4237         txdp->buf_addr = rte_cpu_to_le_64(pf->fdir.dma_addr);
4238         td_cmd = ICE_TX_DESC_CMD_EOP |
4239                 ICE_TX_DESC_CMD_RS  |
4240                 ICE_TX_DESC_CMD_DUMMY;
4241
4242         txdp->cmd_type_offset_bsz =
4243                 ice_build_ctob(td_cmd, 0, ICE_FDIR_PKT_LEN, 0);
4244
4245         txq->tx_tail += 2;
4246         if (txq->tx_tail >= txq->nb_tx_desc)
4247                 txq->tx_tail = 0;
4248         /* Update the tx tail register */
4249         ICE_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail);
4250         for (i = 0; i < ICE_FDIR_MAX_WAIT_US; i++) {
4251                 if ((txdp->cmd_type_offset_bsz &
4252                      rte_cpu_to_le_64(ICE_TXD_QW1_DTYPE_M)) ==
4253                     rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DESC_DONE))
4254                         break;
4255                 rte_delay_us(1);
4256         }
4257         if (i >= ICE_FDIR_MAX_WAIT_US) {
4258                 PMD_DRV_LOG(ERR,
4259                             "Failed to program FDIR filter: time out to get DD on tx queue.");
4260                 return -ETIMEDOUT;
4261         }
4262
4263         for (; i < ICE_FDIR_MAX_WAIT_US; i++) {
4264                 int ret;
4265
4266                 ret = ice_check_fdir_programming_status(rxq);
4267                 if (ret == -EAGAIN)
4268                         rte_delay_us(1);
4269                 else
4270                         return ret;
4271         }
4272
4273         PMD_DRV_LOG(ERR,
4274                     "Failed to program FDIR filter: programming status reported.");
4275         return -ETIMEDOUT;
4276
4277
4278 }