drivers/net/ice/ice_rxtx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4
   5 #include <ethdev_driver.h>
   6 #include <rte_net.h>
   7 #include <rte_vect.h>
   8
   9 #include "rte_pmd_ice.h"
  10 #include "ice_rxtx.h"
  11 #include "ice_rxtx_vec_common.h"
  12
  13 #define ICE_TX_CKSUM_OFFLOAD_MASK (              \
  14                 PKT_TX_IP_CKSUM |                \
  15                 PKT_TX_L4_MASK |                 \
  16                 PKT_TX_TCP_SEG |                 \
  17                 PKT_TX_OUTER_IP_CKSUM)
  18
  19 /* Offset of mbuf dynamic field for protocol extraction data */
  20 int rte_net_ice_dynfield_proto_xtr_metadata_offs = -1;
  21
  22 /* Mask of mbuf dynamic flags for protocol extraction type */
  23 uint64_t rte_net_ice_dynflag_proto_xtr_vlan_mask;
  24 uint64_t rte_net_ice_dynflag_proto_xtr_ipv4_mask;
  25 uint64_t rte_net_ice_dynflag_proto_xtr_ipv6_mask;
  26 uint64_t rte_net_ice_dynflag_proto_xtr_ipv6_flow_mask;
  27 uint64_t rte_net_ice_dynflag_proto_xtr_tcp_mask;
  28 uint64_t rte_net_ice_dynflag_proto_xtr_ip_offset_mask;
  29
  30 static int
  31 ice_monitor_callback(const uint64_t value,
  32                 const uint64_t arg[RTE_POWER_MONITOR_OPAQUE_SZ] __rte_unused)
  33 {
  34         const uint64_t m = rte_cpu_to_le_16(1 << ICE_RX_FLEX_DESC_STATUS0_DD_S);
  35         /*
  36          * we expect the DD bit to be set to 1 if this descriptor was already
  37          * written to.
  38          */
  39         return (value & m) == m ? -1 : 0;
  40 }
  41
  42 int
  43 ice_get_monitor_addr(void *rx_queue, struct rte_power_monitor_cond *pmc)
  44 {
  45         volatile union ice_rx_flex_desc *rxdp;
  46         struct ice_rx_queue *rxq = rx_queue;
  47         uint16_t desc;
  48
  49         desc = rxq->rx_tail;
  50         rxdp = &rxq->rx_ring[desc];
  51         /* watch for changes in status bit */
  52         pmc->addr = &rxdp->wb.status_error0;
  53
  54         /* comparison callback */
  55         pmc->fn = ice_monitor_callback;
  56
  57         /* register is 16-bit */
  58         pmc->size = sizeof(uint16_t);
  59
  60         return 0;
  61 }
  62
  63
  64 static inline uint8_t
  65 ice_proto_xtr_type_to_rxdid(uint8_t xtr_type)
  66 {
  67         static uint8_t rxdid_map[] = {
  68                 [PROTO_XTR_NONE]      = ICE_RXDID_COMMS_OVS,
  69                 [PROTO_XTR_VLAN]      = ICE_RXDID_COMMS_AUX_VLAN,
  70                 [PROTO_XTR_IPV4]      = ICE_RXDID_COMMS_AUX_IPV4,
  71                 [PROTO_XTR_IPV6]      = ICE_RXDID_COMMS_AUX_IPV6,
  72                 [PROTO_XTR_IPV6_FLOW] = ICE_RXDID_COMMS_AUX_IPV6_FLOW,
  73                 [PROTO_XTR_TCP]       = ICE_RXDID_COMMS_AUX_TCP,
  74                 [PROTO_XTR_IP_OFFSET] = ICE_RXDID_COMMS_AUX_IP_OFFSET,
  75         };
  76
  77         return xtr_type < RTE_DIM(rxdid_map) ?
  78                                 rxdid_map[xtr_type] : ICE_RXDID_COMMS_OVS;
  79 }
  80
  81 static inline void
  82 ice_rxd_to_pkt_fields_by_comms_generic(__rte_unused struct ice_rx_queue *rxq,
  83                                        struct rte_mbuf *mb,
  84                                        volatile union ice_rx_flex_desc *rxdp)
  85 {
  86         volatile struct ice_32b_rx_flex_desc_comms *desc =
  87                         (volatile struct ice_32b_rx_flex_desc_comms *)rxdp;
  88         uint16_t stat_err = rte_le_to_cpu_16(desc->status_error0);
  89
  90         if (likely(stat_err & (1 << ICE_RX_FLEX_DESC_STATUS0_RSS_VALID_S))) {
  91                 mb->ol_flags |= PKT_RX_RSS_HASH;
  92                 mb->hash.rss = rte_le_to_cpu_32(desc->rss_hash);
  93         }
  94
  95 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
  96         if (desc->flow_id != 0xFFFFFFFF) {
  97                 mb->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID;
  98                 mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id);
  99         }
 100 #endif
 101 }
 102
 103 static inline void
 104 ice_rxd_to_pkt_fields_by_comms_ovs(__rte_unused struct ice_rx_queue *rxq,
 105                                    struct rte_mbuf *mb,
 106                                    volatile union ice_rx_flex_desc *rxdp)
 107 {
 108         volatile struct ice_32b_rx_flex_desc_comms_ovs *desc =
 109                         (volatile struct ice_32b_rx_flex_desc_comms_ovs *)rxdp;
 110 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
 111         uint16_t stat_err;
 112 #endif
 113
 114         if (desc->flow_id != 0xFFFFFFFF) {
 115                 mb->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID;
 116                 mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id);
 117         }
 118
 119 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
 120         stat_err = rte_le_to_cpu_16(desc->status_error0);
 121         if (likely(stat_err & (1 << ICE_RX_FLEX_DESC_STATUS0_RSS_VALID_S))) {
 122                 mb->ol_flags |= PKT_RX_RSS_HASH;
 123                 mb->hash.rss = rte_le_to_cpu_32(desc->rss_hash);
 124         }
 125 #endif
 126 }
 127
 128 static inline void
 129 ice_rxd_to_pkt_fields_by_comms_aux_v1(struct ice_rx_queue *rxq,
 130                                       struct rte_mbuf *mb,
 131                                       volatile union ice_rx_flex_desc *rxdp)
 132 {
 133         volatile struct ice_32b_rx_flex_desc_comms *desc =
 134                         (volatile struct ice_32b_rx_flex_desc_comms *)rxdp;
 135         uint16_t stat_err;
 136
 137         stat_err = rte_le_to_cpu_16(desc->status_error0);
 138         if (likely(stat_err & (1 << ICE_RX_FLEX_DESC_STATUS0_RSS_VALID_S))) {
 139                 mb->ol_flags |= PKT_RX_RSS_HASH;
 140                 mb->hash.rss = rte_le_to_cpu_32(desc->rss_hash);
 141         }
 142
 143 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
 144         if (desc->flow_id != 0xFFFFFFFF) {
 145                 mb->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID;
 146                 mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id);
 147         }
 148
 149         if (rxq->xtr_ol_flag) {
 150                 uint32_t metadata = 0;
 151
 152                 stat_err = rte_le_to_cpu_16(desc->status_error1);
 153
 154                 if (stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S))
 155                         metadata = rte_le_to_cpu_16(desc->flex_ts.flex.aux0);
 156
 157                 if (stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S))
 158                         metadata |=
 159                                 rte_le_to_cpu_16(desc->flex_ts.flex.aux1) << 16;
 160
 161                 if (metadata) {
 162                         mb->ol_flags |= rxq->xtr_ol_flag;
 163
 164                         *RTE_NET_ICE_DYNF_PROTO_XTR_METADATA(mb) = metadata;
 165                 }
 166         }
 167 #endif
 168 }
 169
 170 static inline void
 171 ice_rxd_to_pkt_fields_by_comms_aux_v2(struct ice_rx_queue *rxq,
 172                                       struct rte_mbuf *mb,
 173                                       volatile union ice_rx_flex_desc *rxdp)
 174 {
 175         volatile struct ice_32b_rx_flex_desc_comms *desc =
 176                         (volatile struct ice_32b_rx_flex_desc_comms *)rxdp;
 177         uint16_t stat_err;
 178
 179         stat_err = rte_le_to_cpu_16(desc->status_error0);
 180         if (likely(stat_err & (1 << ICE_RX_FLEX_DESC_STATUS0_RSS_VALID_S))) {
 181                 mb->ol_flags |= PKT_RX_RSS_HASH;
 182                 mb->hash.rss = rte_le_to_cpu_32(desc->rss_hash);
 183         }
 184
 185 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
 186         if (desc->flow_id != 0xFFFFFFFF) {
 187                 mb->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID;
 188                 mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id);
 189         }
 190
 191         if (rxq->xtr_ol_flag) {
 192                 uint32_t metadata = 0;
 193
 194                 if (desc->flex_ts.flex.aux0 != 0xFFFF)
 195                         metadata = rte_le_to_cpu_16(desc->flex_ts.flex.aux0);
 196                 else if (desc->flex_ts.flex.aux1 != 0xFFFF)
 197                         metadata = rte_le_to_cpu_16(desc->flex_ts.flex.aux1);
 198
 199                 if (metadata) {
 200                         mb->ol_flags |= rxq->xtr_ol_flag;
 201
 202                         *RTE_NET_ICE_DYNF_PROTO_XTR_METADATA(mb) = metadata;
 203                 }
 204         }
 205 #endif
 206 }
 207
 208 void
 209 ice_select_rxd_to_pkt_fields_handler(struct ice_rx_queue *rxq, uint32_t rxdid)
 210 {
 211         switch (rxdid) {
 212         case ICE_RXDID_COMMS_AUX_VLAN:
 213                 rxq->xtr_ol_flag = rte_net_ice_dynflag_proto_xtr_vlan_mask;
 214                 rxq->rxd_to_pkt_fields = ice_rxd_to_pkt_fields_by_comms_aux_v1;
 215                 break;
 216
 217         case ICE_RXDID_COMMS_AUX_IPV4:
 218                 rxq->xtr_ol_flag = rte_net_ice_dynflag_proto_xtr_ipv4_mask;
 219                 rxq->rxd_to_pkt_fields = ice_rxd_to_pkt_fields_by_comms_aux_v1;
 220                 break;
 221
 222         case ICE_RXDID_COMMS_AUX_IPV6:
 223                 rxq->xtr_ol_flag = rte_net_ice_dynflag_proto_xtr_ipv6_mask;
 224                 rxq->rxd_to_pkt_fields = ice_rxd_to_pkt_fields_by_comms_aux_v1;
 225                 break;
 226
 227         case ICE_RXDID_COMMS_AUX_IPV6_FLOW:
 228                 rxq->xtr_ol_flag = rte_net_ice_dynflag_proto_xtr_ipv6_flow_mask;
 229                 rxq->rxd_to_pkt_fields = ice_rxd_to_pkt_fields_by_comms_aux_v1;
 230                 break;
 231
 232         case ICE_RXDID_COMMS_AUX_TCP:
 233                 rxq->xtr_ol_flag = rte_net_ice_dynflag_proto_xtr_tcp_mask;
 234                 rxq->rxd_to_pkt_fields = ice_rxd_to_pkt_fields_by_comms_aux_v1;
 235                 break;
 236
 237         case ICE_RXDID_COMMS_AUX_IP_OFFSET:
 238                 rxq->xtr_ol_flag = rte_net_ice_dynflag_proto_xtr_ip_offset_mask;
 239                 rxq->rxd_to_pkt_fields = ice_rxd_to_pkt_fields_by_comms_aux_v2;
 240                 break;
 241
 242         case ICE_RXDID_COMMS_GENERIC:
 243                 rxq->rxd_to_pkt_fields = ice_rxd_to_pkt_fields_by_comms_generic;
 244                 break;
 245
 246         case ICE_RXDID_COMMS_OVS:
 247                 rxq->rxd_to_pkt_fields = ice_rxd_to_pkt_fields_by_comms_ovs;
 248                 break;
 249
 250         default:
 251                 /* update this according to the RXDID for PROTO_XTR_NONE */
 252                 rxq->rxd_to_pkt_fields = ice_rxd_to_pkt_fields_by_comms_ovs;
 253                 break;
 254         }
 255
 256         if (!rte_net_ice_dynf_proto_xtr_metadata_avail())
 257                 rxq->xtr_ol_flag = 0;
 258 }
 259
 260 static enum ice_status
 261 ice_program_hw_rx_queue(struct ice_rx_queue *rxq)
 262 {
 263         struct ice_vsi *vsi = rxq->vsi;
 264         struct ice_hw *hw = ICE_VSI_TO_HW(vsi);
 265         struct ice_pf *pf = ICE_VSI_TO_PF(vsi);
 266         struct rte_eth_dev_data *dev_data = rxq->vsi->adapter->pf.dev_data;
 267         struct ice_rlan_ctx rx_ctx;
 268         enum ice_status err;
 269         uint16_t buf_size;
 270         struct rte_eth_rxmode *rxmode = &dev_data->dev_conf.rxmode;
 271         uint32_t rxdid = ICE_RXDID_COMMS_OVS;
 272         uint32_t regval;
 273         struct ice_adapter *ad = rxq->vsi->adapter;
 274
 275         /* Set buffer size as the head split is disabled. */
 276         buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 277                               RTE_PKTMBUF_HEADROOM);
 278         rxq->rx_hdr_len = 0;
 279         rxq->rx_buf_len = RTE_ALIGN(buf_size, (1 << ICE_RLAN_CTX_DBUF_S));
 280         rxq->max_pkt_len = RTE_MIN((uint32_t)
 281                                    ICE_SUPPORT_CHAIN_NUM * rxq->rx_buf_len,
 282                                    dev_data->dev_conf.rxmode.max_rx_pkt_len);
 283
 284         if (rxmode->offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) {
 285                 if (rxq->max_pkt_len <= ICE_ETH_MAX_LEN ||
 286                     rxq->max_pkt_len > ICE_FRAME_SIZE_MAX) {
 287                         PMD_DRV_LOG(ERR, "maximum packet length must "
 288                                     "be larger than %u and smaller than %u,"
 289                                     "as jumbo frame is enabled",
 290                                     (uint32_t)ICE_ETH_MAX_LEN,
 291                                     (uint32_t)ICE_FRAME_SIZE_MAX);
 292                         return -EINVAL;
 293                 }
 294         } else {
 295                 if (rxq->max_pkt_len < RTE_ETHER_MIN_LEN ||
 296                     rxq->max_pkt_len > ICE_ETH_MAX_LEN) {
 297                         PMD_DRV_LOG(ERR, "maximum packet length must be "
 298                                     "larger than %u and smaller than %u, "
 299                                     "as jumbo frame is disabled",
 300                                     (uint32_t)RTE_ETHER_MIN_LEN,
 301                                     (uint32_t)ICE_ETH_MAX_LEN);
 302                         return -EINVAL;
 303                 }
 304         }
 305
 306         if (rxq->offloads & DEV_RX_OFFLOAD_TIMESTAMP) {
 307                 /* Register mbuf field and flag for Rx timestamp */
 308                 err = rte_mbuf_dyn_rx_timestamp_register(
 309                                 &ice_timestamp_dynfield_offset,
 310                                 &ice_timestamp_dynflag);
 311                 if (err) {
 312                         PMD_DRV_LOG(ERR,
 313                                 "Cannot register mbuf field/flag for timestamp");
 314                         return -EINVAL;
 315                 }
 316         }
 317
 318         memset(&rx_ctx, 0, sizeof(rx_ctx));
 319
 320         rx_ctx.base = rxq->rx_ring_dma / ICE_QUEUE_BASE_ADDR_UNIT;
 321         rx_ctx.qlen = rxq->nb_rx_desc;
 322         rx_ctx.dbuf = rxq->rx_buf_len >> ICE_RLAN_CTX_DBUF_S;
 323         rx_ctx.hbuf = rxq->rx_hdr_len >> ICE_RLAN_CTX_HBUF_S;
 324         rx_ctx.dtype = 0; /* No Header Split mode */
 325 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
 326         rx_ctx.dsize = 1; /* 32B descriptors */
 327 #endif
 328         rx_ctx.rxmax = rxq->max_pkt_len;
 329         /* TPH: Transaction Layer Packet (TLP) processing hints */
 330         rx_ctx.tphrdesc_ena = 1;
 331         rx_ctx.tphwdesc_ena = 1;
 332         rx_ctx.tphdata_ena = 1;
 333         rx_ctx.tphhead_ena = 1;
 334         /* Low Receive Queue Threshold defined in 64 descriptors units.
 335          * When the number of free descriptors goes below the lrxqthresh,
 336          * an immediate interrupt is triggered.
 337          */
 338         rx_ctx.lrxqthresh = 2;
 339         /*default use 32 byte descriptor, vlan tag extract to L2TAG2(1st)*/
 340         rx_ctx.l2tsel = 1;
 341         rx_ctx.showiv = 0;
 342         rx_ctx.crcstrip = (rxq->crc_len == 0) ? 1 : 0;
 343
 344         rxdid = ice_proto_xtr_type_to_rxdid(rxq->proto_xtr);
 345
 346         PMD_DRV_LOG(DEBUG, "Port (%u) - Rx queue (%u) is set with RXDID : %u",
 347                     rxq->port_id, rxq->queue_id, rxdid);
 348
 349         if (!(pf->supported_rxdid & BIT(rxdid))) {
 350                 PMD_DRV_LOG(ERR, "currently package doesn't support RXDID (%u)",
 351                             rxdid);
 352                 return -EINVAL;
 353         }
 354
 355         ice_select_rxd_to_pkt_fields_handler(rxq, rxdid);
 356
 357         /* Enable Flexible Descriptors in the queue context which
 358          * allows this driver to select a specific receive descriptor format
 359          */
 360         regval = (rxdid << QRXFLXP_CNTXT_RXDID_IDX_S) &
 361                 QRXFLXP_CNTXT_RXDID_IDX_M;
 362
 363         /* increasing context priority to pick up profile ID;
 364          * default is 0x01; setting to 0x03 to ensure profile
 365          * is programming if prev context is of same priority
 366          */
 367         regval |= (0x03 << QRXFLXP_CNTXT_RXDID_PRIO_S) &
 368                 QRXFLXP_CNTXT_RXDID_PRIO_M;
 369
 370         if (ad->ptp_ena || rxq->offloads & DEV_RX_OFFLOAD_TIMESTAMP)
 371                 regval |= QRXFLXP_CNTXT_TS_M;
 372
 373         ICE_WRITE_REG(hw, QRXFLXP_CNTXT(rxq->reg_idx), regval);
 374
 375         err = ice_clear_rxq_ctx(hw, rxq->reg_idx);
 376         if (err) {
 377                 PMD_DRV_LOG(ERR, "Failed to clear Lan Rx queue (%u) context",
 378                             rxq->queue_id);
 379                 return -EINVAL;
 380         }
 381         err = ice_write_rxq_ctx(hw, &rx_ctx, rxq->reg_idx);
 382         if (err) {
 383                 PMD_DRV_LOG(ERR, "Failed to write Lan Rx queue (%u) context",
 384                             rxq->queue_id);
 385                 return -EINVAL;
 386         }
 387
 388         buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 389                               RTE_PKTMBUF_HEADROOM);
 390
 391         /* Check if scattered RX needs to be used. */
 392         if (rxq->max_pkt_len > buf_size)
 393                 dev_data->scattered_rx = 1;
 394
 395         rxq->qrx_tail = hw->hw_addr + QRX_TAIL(rxq->reg_idx);
 396
 397         /* Init the Rx tail register*/
 398         ICE_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
 399
 400         return 0;
 401 }
 402
 403 /* Allocate mbufs for all descriptors in rx queue */
 404 static int
 405 ice_alloc_rx_queue_mbufs(struct ice_rx_queue *rxq)
 406 {
 407         struct ice_rx_entry *rxe = rxq->sw_ring;
 408         uint64_t dma_addr;
 409         uint16_t i;
 410
 411         for (i = 0; i < rxq->nb_rx_desc; i++) {
 412                 volatile union ice_rx_flex_desc *rxd;
 413                 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mp);
 414
 415                 if (unlikely(!mbuf)) {
 416                         PMD_DRV_LOG(ERR, "Failed to allocate mbuf for RX");
 417                         return -ENOMEM;
 418                 }
 419
 420                 rte_mbuf_refcnt_set(mbuf, 1);
 421                 mbuf->next = NULL;
 422                 mbuf->data_off = RTE_PKTMBUF_HEADROOM;
 423                 mbuf->nb_segs = 1;
 424                 mbuf->port = rxq->port_id;
 425
 426                 dma_addr =
 427                         rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf));
 428
 429                 rxd = &rxq->rx_ring[i];
 430                 rxd->read.pkt_addr = dma_addr;
 431                 rxd->read.hdr_addr = 0;
 432 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
 433                 rxd->read.rsvd1 = 0;
 434                 rxd->read.rsvd2 = 0;
 435 #endif
 436                 rxe[i].mbuf = mbuf;
 437         }
 438
 439         return 0;
 440 }
 441
 442 /* Free all mbufs for descriptors in rx queue */
 443 static void
 444 _ice_rx_queue_release_mbufs(struct ice_rx_queue *rxq)
 445 {
 446         uint16_t i;
 447
 448         if (!rxq || !rxq->sw_ring) {
 449                 PMD_DRV_LOG(DEBUG, "Pointer to sw_ring is NULL");
 450                 return;
 451         }
 452
 453         for (i = 0; i < rxq->nb_rx_desc; i++) {
 454                 if (rxq->sw_ring[i].mbuf) {
 455                         rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
 456                         rxq->sw_ring[i].mbuf = NULL;
 457                 }
 458         }
 459         if (rxq->rx_nb_avail == 0)
 460                 return;
 461         for (i = 0; i < rxq->rx_nb_avail; i++)
 462                 rte_pktmbuf_free_seg(rxq->rx_stage[rxq->rx_next_avail + i]);
 463
 464         rxq->rx_nb_avail = 0;
 465 }
 466
 467 /* turn on or off rx queue
 468  * @q_idx: queue index in pf scope
 469  * @on: turn on or off the queue
 470  */
 471 static int
 472 ice_switch_rx_queue(struct ice_hw *hw, uint16_t q_idx, bool on)
 473 {
 474         uint32_t reg;
 475         uint16_t j;
 476
 477         /* QRX_CTRL = QRX_ENA */
 478         reg = ICE_READ_REG(hw, QRX_CTRL(q_idx));
 479
 480         if (on) {
 481                 if (reg & QRX_CTRL_QENA_STAT_M)
 482                         return 0; /* Already on, skip */
 483                 reg |= QRX_CTRL_QENA_REQ_M;
 484         } else {
 485                 if (!(reg & QRX_CTRL_QENA_STAT_M))
 486                         return 0; /* Already off, skip */
 487                 reg &= ~QRX_CTRL_QENA_REQ_M;
 488         }
 489
 490         /* Write the register */
 491         ICE_WRITE_REG(hw, QRX_CTRL(q_idx), reg);
 492         /* Check the result. It is said that QENA_STAT
 493          * follows the QENA_REQ not more than 10 use.
 494          * TODO: need to change the wait counter later
 495          */
 496         for (j = 0; j < ICE_CHK_Q_ENA_COUNT; j++) {
 497                 rte_delay_us(ICE_CHK_Q_ENA_INTERVAL_US);
 498                 reg = ICE_READ_REG(hw, QRX_CTRL(q_idx));
 499                 if (on) {
 500                         if ((reg & QRX_CTRL_QENA_REQ_M) &&
 501                             (reg & QRX_CTRL_QENA_STAT_M))
 502                                 break;
 503                 } else {
 504                         if (!(reg & QRX_CTRL_QENA_REQ_M) &&
 505                             !(reg & QRX_CTRL_QENA_STAT_M))
 506                                 break;
 507                 }
 508         }
 509
 510         /* Check if it is timeout */
 511         if (j >= ICE_CHK_Q_ENA_COUNT) {
 512                 PMD_DRV_LOG(ERR, "Failed to %s rx queue[%u]",
 513                             (on ? "enable" : "disable"), q_idx);
 514                 return -ETIMEDOUT;
 515         }
 516
 517         return 0;
 518 }
 519
 520 static inline int
 521 ice_check_rx_burst_bulk_alloc_preconditions(struct ice_rx_queue *rxq)
 522 {
 523         int ret = 0;
 524
 525         if (!(rxq->rx_free_thresh >= ICE_RX_MAX_BURST)) {
 526                 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
 527                              "rxq->rx_free_thresh=%d, "
 528                              "ICE_RX_MAX_BURST=%d",
 529                              rxq->rx_free_thresh, ICE_RX_MAX_BURST);
 530                 ret = -EINVAL;
 531         } else if (!(rxq->rx_free_thresh < rxq->nb_rx_desc)) {
 532                 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
 533                              "rxq->rx_free_thresh=%d, "
 534                              "rxq->nb_rx_desc=%d",
 535                              rxq->rx_free_thresh, rxq->nb_rx_desc);
 536                 ret = -EINVAL;
 537         } else if (rxq->nb_rx_desc % rxq->rx_free_thresh != 0) {
 538                 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
 539                              "rxq->nb_rx_desc=%d, "
 540                              "rxq->rx_free_thresh=%d",
 541                              rxq->nb_rx_desc, rxq->rx_free_thresh);
 542                 ret = -EINVAL;
 543         }
 544
 545         return ret;
 546 }
 547
 548 /* reset fields in ice_rx_queue back to default */
 549 static void
 550 ice_reset_rx_queue(struct ice_rx_queue *rxq)
 551 {
 552         unsigned int i;
 553         uint16_t len;
 554
 555         if (!rxq) {
 556                 PMD_DRV_LOG(DEBUG, "Pointer to rxq is NULL");
 557                 return;
 558         }
 559
 560         len = (uint16_t)(rxq->nb_rx_desc + ICE_RX_MAX_BURST);
 561
 562         for (i = 0; i < len * sizeof(union ice_rx_flex_desc); i++)
 563                 ((volatile char *)rxq->rx_ring)[i] = 0;
 564
 565         memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf));
 566         for (i = 0; i < ICE_RX_MAX_BURST; ++i)
 567                 rxq->sw_ring[rxq->nb_rx_desc + i].mbuf = &rxq->fake_mbuf;
 568
 569         rxq->rx_nb_avail = 0;
 570         rxq->rx_next_avail = 0;
 571         rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1);
 572
 573         rxq->rx_tail = 0;
 574         rxq->nb_rx_hold = 0;
 575         rxq->pkt_first_seg = NULL;
 576         rxq->pkt_last_seg = NULL;
 577
 578         rxq->rxrearm_start = 0;
 579         rxq->rxrearm_nb = 0;
 580 }
 581
 582 int
 583 ice_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 584 {
 585         struct ice_rx_queue *rxq;
 586         int err;
 587         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 588
 589         PMD_INIT_FUNC_TRACE();
 590
 591         if (rx_queue_id >= dev->data->nb_rx_queues) {
 592                 PMD_DRV_LOG(ERR, "RX queue %u is out of range %u",
 593                             rx_queue_id, dev->data->nb_rx_queues);
 594                 return -EINVAL;
 595         }
 596
 597         rxq = dev->data->rx_queues[rx_queue_id];
 598         if (!rxq || !rxq->q_set) {
 599                 PMD_DRV_LOG(ERR, "RX queue %u not available or setup",
 600                             rx_queue_id);
 601                 return -EINVAL;
 602         }
 603
 604         err = ice_program_hw_rx_queue(rxq);
 605         if (err) {
 606                 PMD_DRV_LOG(ERR, "fail to program RX queue %u",
 607                             rx_queue_id);
 608                 return -EIO;
 609         }
 610
 611         err = ice_alloc_rx_queue_mbufs(rxq);
 612         if (err) {
 613                 PMD_DRV_LOG(ERR, "Failed to allocate RX queue mbuf");
 614                 return -ENOMEM;
 615         }
 616
 617         /* Init the RX tail register. */
 618         ICE_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
 619
 620         err = ice_switch_rx_queue(hw, rxq->reg_idx, true);
 621         if (err) {
 622                 PMD_DRV_LOG(ERR, "Failed to switch RX queue %u on",
 623                             rx_queue_id);
 624
 625                 rxq->rx_rel_mbufs(rxq);
 626                 ice_reset_rx_queue(rxq);
 627                 return -EINVAL;
 628         }
 629
 630         dev->data->rx_queue_state[rx_queue_id] =
 631                 RTE_ETH_QUEUE_STATE_STARTED;
 632
 633         return 0;
 634 }
 635
 636 int
 637 ice_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 638 {
 639         struct ice_rx_queue *rxq;
 640         int err;
 641         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 642
 643         if (rx_queue_id < dev->data->nb_rx_queues) {
 644                 rxq = dev->data->rx_queues[rx_queue_id];
 645
 646                 err = ice_switch_rx_queue(hw, rxq->reg_idx, false);
 647                 if (err) {
 648                         PMD_DRV_LOG(ERR, "Failed to switch RX queue %u off",
 649                                     rx_queue_id);
 650                         return -EINVAL;
 651                 }
 652                 rxq->rx_rel_mbufs(rxq);
 653                 ice_reset_rx_queue(rxq);
 654                 dev->data->rx_queue_state[rx_queue_id] =
 655                         RTE_ETH_QUEUE_STATE_STOPPED;
 656         }
 657
 658         return 0;
 659 }
 660
 661 int
 662 ice_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 663 {
 664         struct ice_tx_queue *txq;
 665         int err;
 666         struct ice_vsi *vsi;
 667         struct ice_hw *hw;
 668         struct ice_aqc_add_tx_qgrp *txq_elem;
 669         struct ice_tlan_ctx tx_ctx;
 670         int buf_len;
 671
 672         PMD_INIT_FUNC_TRACE();
 673
 674         if (tx_queue_id >= dev->data->nb_tx_queues) {
 675                 PMD_DRV_LOG(ERR, "TX queue %u is out of range %u",
 676                             tx_queue_id, dev->data->nb_tx_queues);
 677                 return -EINVAL;
 678         }
 679
 680         txq = dev->data->tx_queues[tx_queue_id];
 681         if (!txq || !txq->q_set) {
 682                 PMD_DRV_LOG(ERR, "TX queue %u is not available or setup",
 683                             tx_queue_id);
 684                 return -EINVAL;
 685         }
 686
 687         buf_len = ice_struct_size(txq_elem, txqs, 1);
 688         txq_elem = ice_malloc(hw, buf_len);
 689         if (!txq_elem)
 690                 return -ENOMEM;
 691
 692         vsi = txq->vsi;
 693         hw = ICE_VSI_TO_HW(vsi);
 694
 695         memset(&tx_ctx, 0, sizeof(tx_ctx));
 696         txq_elem->num_txqs = 1;
 697         txq_elem->txqs[0].txq_id = rte_cpu_to_le_16(txq->reg_idx);
 698
 699         tx_ctx.base = txq->tx_ring_dma / ICE_QUEUE_BASE_ADDR_UNIT;
 700         tx_ctx.qlen = txq->nb_tx_desc;
 701         tx_ctx.pf_num = hw->pf_id;
 702         tx_ctx.vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_PF;
 703         tx_ctx.src_vsi = vsi->vsi_id;
 704         tx_ctx.port_num = hw->port_info->lport;
 705         tx_ctx.tso_ena = 1; /* tso enable */
 706         tx_ctx.tso_qnum = txq->reg_idx; /* index for tso state structure */
 707         tx_ctx.legacy_int = 1; /* Legacy or Advanced Host Interface */
 708         tx_ctx.tsyn_ena = 1;
 709
 710         ice_set_ctx(hw, (uint8_t *)&tx_ctx, txq_elem->txqs[0].txq_ctx,
 711                     ice_tlan_ctx_info);
 712
 713         txq->qtx_tail = hw->hw_addr + QTX_COMM_DBELL(txq->reg_idx);
 714
 715         /* Init the Tx tail register*/
 716         ICE_PCI_REG_WRITE(txq->qtx_tail, 0);
 717
 718         /* Fix me, we assume TC always 0 here */
 719         err = ice_ena_vsi_txq(hw->port_info, vsi->idx, 0, tx_queue_id, 1,
 720                         txq_elem, buf_len, NULL);
 721         if (err) {
 722                 PMD_DRV_LOG(ERR, "Failed to add lan txq");
 723                 rte_free(txq_elem);
 724                 return -EIO;
 725         }
 726         /* store the schedule node id */
 727         txq->q_teid = txq_elem->txqs[0].q_teid;
 728
 729         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 730
 731         rte_free(txq_elem);
 732         return 0;
 733 }
 734
 735 static enum ice_status
 736 ice_fdir_program_hw_rx_queue(struct ice_rx_queue *rxq)
 737 {
 738         struct ice_vsi *vsi = rxq->vsi;
 739         struct ice_hw *hw = ICE_VSI_TO_HW(vsi);
 740         uint32_t rxdid = ICE_RXDID_LEGACY_1;
 741         struct ice_rlan_ctx rx_ctx;
 742         enum ice_status err;
 743         uint32_t regval;
 744
 745         rxq->rx_hdr_len = 0;
 746         rxq->rx_buf_len = 1024;
 747
 748         memset(&rx_ctx, 0, sizeof(rx_ctx));
 749
 750         rx_ctx.base = rxq->rx_ring_dma / ICE_QUEUE_BASE_ADDR_UNIT;
 751         rx_ctx.qlen = rxq->nb_rx_desc;
 752         rx_ctx.dbuf = rxq->rx_buf_len >> ICE_RLAN_CTX_DBUF_S;
 753         rx_ctx.hbuf = rxq->rx_hdr_len >> ICE_RLAN_CTX_HBUF_S;
 754         rx_ctx.dtype = 0; /* No Header Split mode */
 755         rx_ctx.dsize = 1; /* 32B descriptors */
 756         rx_ctx.rxmax = ICE_ETH_MAX_LEN;
 757         /* TPH: Transaction Layer Packet (TLP) processing hints */
 758         rx_ctx.tphrdesc_ena = 1;
 759         rx_ctx.tphwdesc_ena = 1;
 760         rx_ctx.tphdata_ena = 1;
 761         rx_ctx.tphhead_ena = 1;
 762         /* Low Receive Queue Threshold defined in 64 descriptors units.
 763          * When the number of free descriptors goes below the lrxqthresh,
 764          * an immediate interrupt is triggered.
 765          */
 766         rx_ctx.lrxqthresh = 2;
 767         /*default use 32 byte descriptor, vlan tag extract to L2TAG2(1st)*/
 768         rx_ctx.l2tsel = 1;
 769         rx_ctx.showiv = 0;
 770         rx_ctx.crcstrip = (rxq->crc_len == 0) ? 1 : 0;
 771
 772         /* Enable Flexible Descriptors in the queue context which
 773          * allows this driver to select a specific receive descriptor format
 774          */
 775         regval = (rxdid << QRXFLXP_CNTXT_RXDID_IDX_S) &
 776                 QRXFLXP_CNTXT_RXDID_IDX_M;
 777
 778         /* increasing context priority to pick up profile ID;
 779          * default is 0x01; setting to 0x03 to ensure profile
 780          * is programming if prev context is of same priority
 781          */
 782         regval |= (0x03 << QRXFLXP_CNTXT_RXDID_PRIO_S) &
 783                 QRXFLXP_CNTXT_RXDID_PRIO_M;
 784
 785         ICE_WRITE_REG(hw, QRXFLXP_CNTXT(rxq->reg_idx), regval);
 786
 787         err = ice_clear_rxq_ctx(hw, rxq->reg_idx);
 788         if (err) {
 789                 PMD_DRV_LOG(ERR, "Failed to clear Lan Rx queue (%u) context",
 790                             rxq->queue_id);
 791                 return -EINVAL;
 792         }
 793         err = ice_write_rxq_ctx(hw, &rx_ctx, rxq->reg_idx);
 794         if (err) {
 795                 PMD_DRV_LOG(ERR, "Failed to write Lan Rx queue (%u) context",
 796                             rxq->queue_id);
 797                 return -EINVAL;
 798         }
 799
 800         rxq->qrx_tail = hw->hw_addr + QRX_TAIL(rxq->reg_idx);
 801
 802         /* Init the Rx tail register*/
 803         ICE_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
 804
 805         return 0;
 806 }
 807
 808 int
 809 ice_fdir_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 810 {
 811         struct ice_rx_queue *rxq;
 812         int err;
 813         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 814         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 815
 816         PMD_INIT_FUNC_TRACE();
 817
 818         rxq = pf->fdir.rxq;
 819         if (!rxq || !rxq->q_set) {
 820                 PMD_DRV_LOG(ERR, "FDIR RX queue %u not available or setup",
 821                             rx_queue_id);
 822                 return -EINVAL;
 823         }
 824
 825         err = ice_fdir_program_hw_rx_queue(rxq);
 826         if (err) {
 827                 PMD_DRV_LOG(ERR, "fail to program FDIR RX queue %u",
 828                             rx_queue_id);
 829                 return -EIO;
 830         }
 831
 832         /* Init the RX tail register. */
 833         ICE_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
 834
 835         err = ice_switch_rx_queue(hw, rxq->reg_idx, true);
 836         if (err) {
 837                 PMD_DRV_LOG(ERR, "Failed to switch FDIR RX queue %u on",
 838                             rx_queue_id);
 839
 840                 ice_reset_rx_queue(rxq);
 841                 return -EINVAL;
 842         }
 843
 844         return 0;
 845 }
 846
 847 int
 848 ice_fdir_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 849 {
 850         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 851         struct ice_tx_queue *txq;
 852         int err;
 853         struct ice_vsi *vsi;
 854         struct ice_hw *hw;
 855         struct ice_aqc_add_tx_qgrp *txq_elem;
 856         struct ice_tlan_ctx tx_ctx;
 857         int buf_len;
 858
 859         PMD_INIT_FUNC_TRACE();
 860
 861         txq = pf->fdir.txq;
 862         if (!txq || !txq->q_set) {
 863                 PMD_DRV_LOG(ERR, "FDIR TX queue %u is not available or setup",
 864                             tx_queue_id);
 865                 return -EINVAL;
 866         }
 867
 868         buf_len = ice_struct_size(txq_elem, txqs, 1);
 869         txq_elem = ice_malloc(hw, buf_len);
 870         if (!txq_elem)
 871                 return -ENOMEM;
 872
 873         vsi = txq->vsi;
 874         hw = ICE_VSI_TO_HW(vsi);
 875
 876         memset(&tx_ctx, 0, sizeof(tx_ctx));
 877         txq_elem->num_txqs = 1;
 878         txq_elem->txqs[0].txq_id = rte_cpu_to_le_16(txq->reg_idx);
 879
 880         tx_ctx.base = txq->tx_ring_dma / ICE_QUEUE_BASE_ADDR_UNIT;
 881         tx_ctx.qlen = txq->nb_tx_desc;
 882         tx_ctx.pf_num = hw->pf_id;
 883         tx_ctx.vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_PF;
 884         tx_ctx.src_vsi = vsi->vsi_id;
 885         tx_ctx.port_num = hw->port_info->lport;
 886         tx_ctx.tso_ena = 1; /* tso enable */
 887         tx_ctx.tso_qnum = txq->reg_idx; /* index for tso state structure */
 888         tx_ctx.legacy_int = 1; /* Legacy or Advanced Host Interface */
 889
 890         ice_set_ctx(hw, (uint8_t *)&tx_ctx, txq_elem->txqs[0].txq_ctx,
 891                     ice_tlan_ctx_info);
 892
 893         txq->qtx_tail = hw->hw_addr + QTX_COMM_DBELL(txq->reg_idx);
 894
 895         /* Init the Tx tail register*/
 896         ICE_PCI_REG_WRITE(txq->qtx_tail, 0);
 897
 898         /* Fix me, we assume TC always 0 here */
 899         err = ice_ena_vsi_txq(hw->port_info, vsi->idx, 0, tx_queue_id, 1,
 900                               txq_elem, buf_len, NULL);
 901         if (err) {
 902                 PMD_DRV_LOG(ERR, "Failed to add FDIR txq");
 903                 rte_free(txq_elem);
 904                 return -EIO;
 905         }
 906         /* store the schedule node id */
 907         txq->q_teid = txq_elem->txqs[0].q_teid;
 908
 909         rte_free(txq_elem);
 910         return 0;
 911 }
 912
 913 /* Free all mbufs for descriptors in tx queue */
 914 static void
 915 _ice_tx_queue_release_mbufs(struct ice_tx_queue *txq)
 916 {
 917         uint16_t i;
 918
 919         if (!txq || !txq->sw_ring) {
 920                 PMD_DRV_LOG(DEBUG, "Pointer to txq or sw_ring is NULL");
 921                 return;
 922         }
 923
 924         for (i = 0; i < txq->nb_tx_desc; i++) {
 925                 if (txq->sw_ring[i].mbuf) {
 926                         rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
 927                         txq->sw_ring[i].mbuf = NULL;
 928                 }
 929         }
 930 }
 931
 932 static void
 933 ice_reset_tx_queue(struct ice_tx_queue *txq)
 934 {
 935         struct ice_tx_entry *txe;
 936         uint16_t i, prev, size;
 937
 938         if (!txq) {
 939                 PMD_DRV_LOG(DEBUG, "Pointer to txq is NULL");
 940                 return;
 941         }
 942
 943         txe = txq->sw_ring;
 944         size = sizeof(struct ice_tx_desc) * txq->nb_tx_desc;
 945         for (i = 0; i < size; i++)
 946                 ((volatile char *)txq->tx_ring)[i] = 0;
 947
 948         prev = (uint16_t)(txq->nb_tx_desc - 1);
 949         for (i = 0; i < txq->nb_tx_desc; i++) {
 950                 volatile struct ice_tx_desc *txd = &txq->tx_ring[i];
 951
 952                 txd->cmd_type_offset_bsz =
 953                         rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DESC_DONE);
 954                 txe[i].mbuf =  NULL;
 955                 txe[i].last_id = i;
 956                 txe[prev].next_id = i;
 957                 prev = i;
 958         }
 959
 960         txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
 961         txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
 962
 963         txq->tx_tail = 0;
 964         txq->nb_tx_used = 0;
 965
 966         txq->last_desc_cleaned = (uint16_t)(txq->nb_tx_desc - 1);
 967         txq->nb_tx_free = (uint16_t)(txq->nb_tx_desc - 1);
 968 }
 969
 970 int
 971 ice_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 972 {
 973         struct ice_tx_queue *txq;
 974         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 975         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 976         struct ice_vsi *vsi = pf->main_vsi;
 977         enum ice_status status;
 978         uint16_t q_ids[1];
 979         uint32_t q_teids[1];
 980         uint16_t q_handle = tx_queue_id;
 981
 982         if (tx_queue_id >= dev->data->nb_tx_queues) {
 983                 PMD_DRV_LOG(ERR, "TX queue %u is out of range %u",
 984                             tx_queue_id, dev->data->nb_tx_queues);
 985                 return -EINVAL;
 986         }
 987
 988         txq = dev->data->tx_queues[tx_queue_id];
 989         if (!txq) {
 990                 PMD_DRV_LOG(ERR, "TX queue %u is not available",
 991                             tx_queue_id);
 992                 return -EINVAL;
 993         }
 994
 995         q_ids[0] = txq->reg_idx;
 996         q_teids[0] = txq->q_teid;
 997
 998         /* Fix me, we assume TC always 0 here */
 999         status = ice_dis_vsi_txq(hw->port_info, vsi->idx, 0, 1, &q_handle,
1000                                 q_ids, q_teids, ICE_NO_RESET, 0, NULL);
1001         if (status != ICE_SUCCESS) {
1002                 PMD_DRV_LOG(DEBUG, "Failed to disable Lan Tx queue");
1003                 return -EINVAL;
1004         }
1005
1006         txq->tx_rel_mbufs(txq);
1007         ice_reset_tx_queue(txq);
1008         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
1009
1010         return 0;
1011 }
1012
1013 int
1014 ice_fdir_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1015 {
1016         struct ice_rx_queue *rxq;
1017         int err;
1018         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1019         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1020
1021         rxq = pf->fdir.rxq;
1022
1023         err = ice_switch_rx_queue(hw, rxq->reg_idx, false);
1024         if (err) {
1025                 PMD_DRV_LOG(ERR, "Failed to switch FDIR RX queue %u off",
1026                             rx_queue_id);
1027                 return -EINVAL;
1028         }
1029         rxq->rx_rel_mbufs(rxq);
1030
1031         return 0;
1032 }
1033
1034 int
1035 ice_fdir_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
1036 {
1037         struct ice_tx_queue *txq;
1038         struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1039         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1040         struct ice_vsi *vsi = pf->main_vsi;
1041         enum ice_status status;
1042         uint16_t q_ids[1];
1043         uint32_t q_teids[1];
1044         uint16_t q_handle = tx_queue_id;
1045
1046         txq = pf->fdir.txq;
1047         if (!txq) {
1048                 PMD_DRV_LOG(ERR, "TX queue %u is not available",
1049                             tx_queue_id);
1050                 return -EINVAL;
1051         }
1052         vsi = txq->vsi;
1053
1054         q_ids[0] = txq->reg_idx;
1055         q_teids[0] = txq->q_teid;
1056
1057         /* Fix me, we assume TC always 0 here */
1058         status = ice_dis_vsi_txq(hw->port_info, vsi->idx, 0, 1, &q_handle,
1059                                  q_ids, q_teids, ICE_NO_RESET, 0, NULL);
1060         if (status != ICE_SUCCESS) {
1061                 PMD_DRV_LOG(DEBUG, "Failed to disable Lan Tx queue");
1062                 return -EINVAL;
1063         }
1064
1065         txq->tx_rel_mbufs(txq);
1066
1067         return 0;
1068 }
1069
1070 int
1071 ice_rx_queue_setup(struct rte_eth_dev *dev,
1072                    uint16_t queue_idx,
1073                    uint16_t nb_desc,
1074                    unsigned int socket_id,
1075                    const struct rte_eth_rxconf *rx_conf,
1076                    struct rte_mempool *mp)
1077 {
1078         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1079         struct ice_adapter *ad =
1080                 ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
1081         struct ice_vsi *vsi = pf->main_vsi;
1082         struct ice_rx_queue *rxq;
1083         const struct rte_memzone *rz;
1084         uint32_t ring_size;
1085         uint16_t len;
1086         int use_def_burst_func = 1;
1087         uint64_t offloads;
1088
1089         if (nb_desc % ICE_ALIGN_RING_DESC != 0 ||
1090             nb_desc > ICE_MAX_RING_DESC ||
1091             nb_desc < ICE_MIN_RING_DESC) {
1092                 PMD_INIT_LOG(ERR, "Number (%u) of receive descriptors is "
1093                              "invalid", nb_desc);
1094                 return -EINVAL;
1095         }
1096
1097         offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads;
1098
1099         /* Free memory if needed */
1100         if (dev->data->rx_queues[queue_idx]) {
1101                 ice_rx_queue_release(dev->data->rx_queues[queue_idx]);
1102                 dev->data->rx_queues[queue_idx] = NULL;
1103         }
1104
1105         /* Allocate the rx queue data structure */
1106         rxq = rte_zmalloc_socket(NULL,
1107                                  sizeof(struct ice_rx_queue),
1108                                  RTE_CACHE_LINE_SIZE,
1109                                  socket_id);
1110         if (!rxq) {
1111                 PMD_INIT_LOG(ERR, "Failed to allocate memory for "
1112                              "rx queue data structure");
1113                 return -ENOMEM;
1114         }
1115         rxq->mp = mp;
1116         rxq->nb_rx_desc = nb_desc;
1117         rxq->rx_free_thresh = rx_conf->rx_free_thresh;
1118         rxq->queue_id = queue_idx;
1119         rxq->offloads = offloads;
1120
1121         rxq->reg_idx = vsi->base_queue + queue_idx;
1122         rxq->port_id = dev->data->port_id;
1123         if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC)
1124                 rxq->crc_len = RTE_ETHER_CRC_LEN;
1125         else
1126                 rxq->crc_len = 0;
1127
1128         rxq->drop_en = rx_conf->rx_drop_en;
1129         rxq->vsi = vsi;
1130         rxq->rx_deferred_start = rx_conf->rx_deferred_start;
1131         rxq->proto_xtr = pf->proto_xtr != NULL ?
1132                          pf->proto_xtr[queue_idx] : PROTO_XTR_NONE;
1133
1134         /* Allocate the maximun number of RX ring hardware descriptor. */
1135         len = ICE_MAX_RING_DESC;
1136
1137         /**
1138          * Allocating a little more memory because vectorized/bulk_alloc Rx
1139          * functions doesn't check boundaries each time.
1140          */
1141         len += ICE_RX_MAX_BURST;
1142
1143         /* Allocate the maximum number of RX ring hardware descriptor. */
1144         ring_size = sizeof(union ice_rx_flex_desc) * len;
1145         ring_size = RTE_ALIGN(ring_size, ICE_DMA_MEM_ALIGN);
1146         rz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx,
1147                                       ring_size, ICE_RING_BASE_ALIGN,
1148                                       socket_id);
1149         if (!rz) {
1150                 ice_rx_queue_release(rxq);
1151                 PMD_INIT_LOG(ERR, "Failed to reserve DMA memory for RX");
1152                 return -ENOMEM;
1153         }
1154
1155         /* Zero all the descriptors in the ring. */
1156         memset(rz->addr, 0, ring_size);
1157
1158         rxq->rx_ring_dma = rz->iova;
1159         rxq->rx_ring = rz->addr;
1160
1161         /* always reserve more for bulk alloc */
1162         len = (uint16_t)(nb_desc + ICE_RX_MAX_BURST);
1163
1164         /* Allocate the software ring. */
1165         rxq->sw_ring = rte_zmalloc_socket(NULL,
1166                                           sizeof(struct ice_rx_entry) * len,
1167                                           RTE_CACHE_LINE_SIZE,
1168                                           socket_id);
1169         if (!rxq->sw_ring) {
1170                 ice_rx_queue_release(rxq);
1171                 PMD_INIT_LOG(ERR, "Failed to allocate memory for SW ring");
1172                 return -ENOMEM;
1173         }
1174
1175         ice_reset_rx_queue(rxq);
1176         rxq->q_set = true;
1177         dev->data->rx_queues[queue_idx] = rxq;
1178         rxq->rx_rel_mbufs = _ice_rx_queue_release_mbufs;
1179
1180         use_def_burst_func = ice_check_rx_burst_bulk_alloc_preconditions(rxq);
1181
1182         if (!use_def_burst_func) {
1183                 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are "
1184                              "satisfied. Rx Burst Bulk Alloc function will be "
1185                              "used on port=%d, queue=%d.",
1186                              rxq->port_id, rxq->queue_id);
1187         } else {
1188                 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are "
1189                              "not satisfied, Scattered Rx is requested. "
1190                              "on port=%d, queue=%d.",
1191                              rxq->port_id, rxq->queue_id);
1192                 ad->rx_bulk_alloc_allowed = false;
1193         }
1194
1195         return 0;
1196 }
1197
1198 void
1199 ice_rx_queue_release(void *rxq)
1200 {
1201         struct ice_rx_queue *q = (struct ice_rx_queue *)rxq;
1202
1203         if (!q) {
1204                 PMD_DRV_LOG(DEBUG, "Pointer to rxq is NULL");
1205                 return;
1206         }
1207
1208         q->rx_rel_mbufs(q);
1209         rte_free(q->sw_ring);
1210         rte_free(q);
1211 }
1212
1213 int
1214 ice_tx_queue_setup(struct rte_eth_dev *dev,
1215                    uint16_t queue_idx,
1216                    uint16_t nb_desc,
1217                    unsigned int socket_id,
1218                    const struct rte_eth_txconf *tx_conf)
1219 {
1220         struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1221         struct ice_vsi *vsi = pf->main_vsi;
1222         struct ice_tx_queue *txq;
1223         const struct rte_memzone *tz;
1224         uint32_t ring_size;
1225         uint16_t tx_rs_thresh, tx_free_thresh;
1226         uint64_t offloads;
1227
1228         offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads;
1229
1230         if (nb_desc % ICE_ALIGN_RING_DESC != 0 ||
1231             nb_desc > ICE_MAX_RING_DESC ||
1232             nb_desc < ICE_MIN_RING_DESC) {
1233                 PMD_INIT_LOG(ERR, "Number (%u) of transmit descriptors is "
1234                              "invalid", nb_desc);
1235                 return -EINVAL;
1236         }
1237
1238         /**
1239          * The following two parameters control the setting of the RS bit on
1240          * transmit descriptors. TX descriptors will have their RS bit set
1241          * after txq->tx_rs_thresh descriptors have been used. The TX
1242          * descriptor ring will be cleaned after txq->tx_free_thresh
1243          * descriptors are used or if the number of descriptors required to
1244          * transmit a packet is greater than the number of free TX descriptors.
1245          *
1246          * The following constraints must be satisfied:
1247          *  - tx_rs_thresh must be greater than 0.
1248          *  - tx_rs_thresh must be less than the size of the ring minus 2.
1249          *  - tx_rs_thresh must be less than or equal to tx_free_thresh.
1250          *  - tx_rs_thresh must be a divisor of the ring size.
1251          *  - tx_free_thresh must be greater than 0.
1252          *  - tx_free_thresh must be less than the size of the ring minus 3.
1253          *  - tx_free_thresh + tx_rs_thresh must not exceed nb_desc.
1254          *
1255          * One descriptor in the TX ring is used as a sentinel to avoid a H/W
1256          * race condition, hence the maximum threshold constraints. When set
1257          * to zero use default values.
1258          */
1259         tx_free_thresh = (uint16_t)(tx_conf->tx_free_thresh ?
1260                                     tx_conf->tx_free_thresh :
1261                                     ICE_DEFAULT_TX_FREE_THRESH);
1262         /* force tx_rs_thresh to adapt an aggresive tx_free_thresh */
1263         tx_rs_thresh =
1264                 (ICE_DEFAULT_TX_RSBIT_THRESH + tx_free_thresh > nb_desc) ?
1265                         nb_desc - tx_free_thresh : ICE_DEFAULT_TX_RSBIT_THRESH;
1266         if (tx_conf->tx_rs_thresh)
1267                 tx_rs_thresh = tx_conf->tx_rs_thresh;
1268         if (tx_rs_thresh + tx_free_thresh > nb_desc) {
1269                 PMD_INIT_LOG(ERR, "tx_rs_thresh + tx_free_thresh must not "
1270                                 "exceed nb_desc. (tx_rs_thresh=%u "
1271                                 "tx_free_thresh=%u nb_desc=%u port = %d queue=%d)",
1272                                 (unsigned int)tx_rs_thresh,
1273                                 (unsigned int)tx_free_thresh,
1274                                 (unsigned int)nb_desc,
1275                                 (int)dev->data->port_id,
1276                                 (int)queue_idx);
1277                 return -EINVAL;
1278         }
1279         if (tx_rs_thresh >= (nb_desc - 2)) {
1280                 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than the "
1281                              "number of TX descriptors minus 2. "
1282                              "(tx_rs_thresh=%u port=%d queue=%d)",
1283                              (unsigned int)tx_rs_thresh,
1284                              (int)dev->data->port_id,
1285                              (int)queue_idx);
1286                 return -EINVAL;
1287         }
1288         if (tx_free_thresh >= (nb_desc - 3)) {
1289                 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than the "
1290                              "tx_free_thresh must be less than the "
1291                              "number of TX descriptors minus 3. "
1292                              "(tx_free_thresh=%u port=%d queue=%d)",
1293                              (unsigned int)tx_free_thresh,
1294                              (int)dev->data->port_id,
1295                              (int)queue_idx);
1296                 return -EINVAL;
1297         }
1298         if (tx_rs_thresh > tx_free_thresh) {
1299                 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than or "
1300                              "equal to tx_free_thresh. (tx_free_thresh=%u"
1301                              " tx_rs_thresh=%u port=%d queue=%d)",
1302                              (unsigned int)tx_free_thresh,
1303                              (unsigned int)tx_rs_thresh,
1304                              (int)dev->data->port_id,
1305                              (int)queue_idx);
1306                 return -EINVAL;
1307         }
1308         if ((nb_desc % tx_rs_thresh) != 0) {
1309                 PMD_INIT_LOG(ERR, "tx_rs_thresh must be a divisor of the "
1310                              "number of TX descriptors. (tx_rs_thresh=%u"
1311                              " port=%d queue=%d)",
1312                              (unsigned int)tx_rs_thresh,
1313                              (int)dev->data->port_id,
1314                              (int)queue_idx);
1315                 return -EINVAL;
1316         }
1317         if (tx_rs_thresh > 1 && tx_conf->tx_thresh.wthresh != 0) {
1318                 PMD_INIT_LOG(ERR, "TX WTHRESH must be set to 0 if "
1319                              "tx_rs_thresh is greater than 1. "
1320                              "(tx_rs_thresh=%u port=%d queue=%d)",
1321                              (unsigned int)tx_rs_thresh,
1322                              (int)dev->data->port_id,
1323                              (int)queue_idx);
1324                 return -EINVAL;
1325         }
1326
1327         /* Free memory if needed. */
1328         if (dev->data->tx_queues[queue_idx]) {
1329                 ice_tx_queue_release(dev->data->tx_queues[queue_idx]);
1330                 dev->data->tx_queues[queue_idx] = NULL;
1331         }
1332
1333         /* Allocate the TX queue data structure. */
1334         txq = rte_zmalloc_socket(NULL,
1335                                  sizeof(struct ice_tx_queue),
1336                                  RTE_CACHE_LINE_SIZE,
1337                                  socket_id);
1338         if (!txq) {
1339                 PMD_INIT_LOG(ERR, "Failed to allocate memory for "
1340                              "tx queue structure");
1341                 return -ENOMEM;
1342         }
1343
1344         /* Allocate TX hardware ring descriptors. */
1345         ring_size = sizeof(struct ice_tx_desc) * ICE_MAX_RING_DESC;
1346         ring_size = RTE_ALIGN(ring_size, ICE_DMA_MEM_ALIGN);
1347         tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx,
1348                                       ring_size, ICE_RING_BASE_ALIGN,
1349                                       socket_id);
1350         if (!tz) {
1351                 ice_tx_queue_release(txq);
1352                 PMD_INIT_LOG(ERR, "Failed to reserve DMA memory for TX");
1353                 return -ENOMEM;
1354         }
1355
1356         txq->nb_tx_desc = nb_desc;
1357         txq->tx_rs_thresh = tx_rs_thresh;
1358         txq->tx_free_thresh = tx_free_thresh;
1359         txq->pthresh = tx_conf->tx_thresh.pthresh;
1360         txq->hthresh = tx_conf->tx_thresh.hthresh;
1361         txq->wthresh = tx_conf->tx_thresh.wthresh;
1362         txq->queue_id = queue_idx;
1363
1364         txq->reg_idx = vsi->base_queue + queue_idx;
1365         txq->port_id = dev->data->port_id;
1366         txq->offloads = offloads;
1367         txq->vsi = vsi;
1368         txq->tx_deferred_start = tx_conf->tx_deferred_start;
1369
1370         txq->tx_ring_dma = tz->iova;
1371         txq->tx_ring = tz->addr;
1372
1373         /* Allocate software ring */
1374         txq->sw_ring =
1375                 rte_zmalloc_socket(NULL,
1376                                    sizeof(struct ice_tx_entry) * nb_desc,
1377                                    RTE_CACHE_LINE_SIZE,
1378                                    socket_id);
1379         if (!txq->sw_ring) {
1380                 ice_tx_queue_release(txq);
1381                 PMD_INIT_LOG(ERR, "Failed to allocate memory for SW TX ring");
1382                 return -ENOMEM;
1383         }
1384
1385         ice_reset_tx_queue(txq);
1386         txq->q_set = true;
1387         dev->data->tx_queues[queue_idx] = txq;
1388         txq->tx_rel_mbufs = _ice_tx_queue_release_mbufs;
1389         ice_set_tx_function_flag(dev, txq);
1390
1391         return 0;
1392 }
1393
1394 void
1395 ice_tx_queue_release(void *txq)
1396 {
1397         struct ice_tx_queue *q = (struct ice_tx_queue *)txq;
1398
1399         if (!q) {
1400                 PMD_DRV_LOG(DEBUG, "Pointer to TX queue is NULL");
1401                 return;
1402         }
1403
1404         q->tx_rel_mbufs(q);
1405         rte_free(q->sw_ring);
1406         rte_free(q);
1407 }
1408
1409 void
1410 ice_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
1411                  struct rte_eth_rxq_info *qinfo)
1412 {
1413         struct ice_rx_queue *rxq;
1414
1415         rxq = dev->data->rx_queues[queue_id];
1416
1417         qinfo->mp = rxq->mp;
1418         qinfo->scattered_rx = dev->data->scattered_rx;
1419         qinfo->nb_desc = rxq->nb_rx_desc;
1420
1421         qinfo->conf.rx_free_thresh = rxq->rx_free_thresh;
1422         qinfo->conf.rx_drop_en = rxq->drop_en;
1423         qinfo->conf.rx_deferred_start = rxq->rx_deferred_start;
1424 }
1425
1426 void
1427 ice_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
1428                  struct rte_eth_txq_info *qinfo)
1429 {
1430         struct ice_tx_queue *txq;
1431
1432         txq = dev->data->tx_queues[queue_id];
1433
1434         qinfo->nb_desc = txq->nb_tx_desc;
1435
1436         qinfo->conf.tx_thresh.pthresh = txq->pthresh;
1437         qinfo->conf.tx_thresh.hthresh = txq->hthresh;
1438         qinfo->conf.tx_thresh.wthresh = txq->wthresh;
1439
1440         qinfo->conf.tx_free_thresh = txq->tx_free_thresh;
1441         qinfo->conf.tx_rs_thresh = txq->tx_rs_thresh;
1442         qinfo->conf.offloads = txq->offloads;
1443         qinfo->conf.tx_deferred_start = txq->tx_deferred_start;
1444 }
1445
1446 uint32_t
1447 ice_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1448 {
1449 #define ICE_RXQ_SCAN_INTERVAL 4
1450         volatile union ice_rx_flex_desc *rxdp;
1451         struct ice_rx_queue *rxq;
1452         uint16_t desc = 0;
1453
1454         rxq = dev->data->rx_queues[rx_queue_id];
1455         rxdp = &rxq->rx_ring[rxq->rx_tail];
1456         while ((desc < rxq->nb_rx_desc) &&
1457                rte_le_to_cpu_16(rxdp->wb.status_error0) &
1458                (1 << ICE_RX_FLEX_DESC_STATUS0_DD_S)) {
1459                 /**
1460                  * Check the DD bit of a rx descriptor of each 4 in a group,
1461                  * to avoid checking too frequently and downgrading performance
1462                  * too much.
1463                  */
1464                 desc += ICE_RXQ_SCAN_INTERVAL;
1465                 rxdp += ICE_RXQ_SCAN_INTERVAL;
1466                 if (rxq->rx_tail + desc >= rxq->nb_rx_desc)
1467                         rxdp = &(rxq->rx_ring[rxq->rx_tail +
1468                                  desc - rxq->nb_rx_desc]);
1469         }
1470
1471         return desc;
1472 }
1473
1474 #define ICE_RX_FLEX_ERR0_BITS   \
1475         ((1 << ICE_RX_FLEX_DESC_STATUS0_HBO_S) |        \
1476          (1 << ICE_RX_FLEX_DESC_STATUS0_XSUM_IPE_S) |   \
1477          (1 << ICE_RX_FLEX_DESC_STATUS0_XSUM_L4E_S) |   \
1478          (1 << ICE_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S) |  \
1479          (1 << ICE_RX_FLEX_DESC_STATUS0_XSUM_EUDPE_S) | \
1480          (1 << ICE_RX_FLEX_DESC_STATUS0_RXE_S))
1481
1482 /* Rx L3/L4 checksum */
1483 static inline uint64_t
1484 ice_rxd_error_to_pkt_flags(uint16_t stat_err0)
1485 {
1486         uint64_t flags = 0;
1487
1488         /* check if HW has decoded the packet and checksum */
1489         if (unlikely(!(stat_err0 & (1 << ICE_RX_FLEX_DESC_STATUS0_L3L4P_S))))
1490                 return 0;
1491
1492         if (likely(!(stat_err0 & ICE_RX_FLEX_ERR0_BITS))) {
1493                 flags |= (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD);
1494                 return flags;
1495         }
1496
1497         if (unlikely(stat_err0 & (1 << ICE_RX_FLEX_DESC_STATUS0_XSUM_IPE_S)))
1498                 flags |= PKT_RX_IP_CKSUM_BAD;
1499         else
1500                 flags |= PKT_RX_IP_CKSUM_GOOD;
1501
1502         if (unlikely(stat_err0 & (1 << ICE_RX_FLEX_DESC_STATUS0_XSUM_L4E_S)))
1503                 flags |= PKT_RX_L4_CKSUM_BAD;
1504         else
1505                 flags |= PKT_RX_L4_CKSUM_GOOD;
1506
1507         if (unlikely(stat_err0 & (1 << ICE_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S)))
1508                 flags |= PKT_RX_OUTER_IP_CKSUM_BAD;
1509
1510         if (unlikely(stat_err0 & (1 << ICE_RX_FLEX_DESC_STATUS0_XSUM_EUDPE_S)))
1511                 flags |= PKT_RX_OUTER_L4_CKSUM_BAD;
1512         else
1513                 flags |= PKT_RX_OUTER_L4_CKSUM_GOOD;
1514
1515         return flags;
1516 }
1517
1518 static inline void
1519 ice_rxd_to_vlan_tci(struct rte_mbuf *mb, volatile union ice_rx_flex_desc *rxdp)
1520 {
1521         if (rte_le_to_cpu_16(rxdp->wb.status_error0) &
1522             (1 << ICE_RX_FLEX_DESC_STATUS0_L2TAG1P_S)) {
1523                 mb->ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED;
1524                 mb->vlan_tci =
1525                         rte_le_to_cpu_16(rxdp->wb.l2tag1);
1526                 PMD_RX_LOG(DEBUG, "Descriptor l2tag1: %u",
1527                            rte_le_to_cpu_16(rxdp->wb.l2tag1));
1528         } else {
1529                 mb->vlan_tci = 0;
1530         }
1531
1532 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
1533         if (rte_le_to_cpu_16(rxdp->wb.status_error1) &
1534             (1 << ICE_RX_FLEX_DESC_STATUS1_L2TAG2P_S)) {
1535                 mb->ol_flags |= PKT_RX_QINQ_STRIPPED | PKT_RX_QINQ |
1536                                 PKT_RX_VLAN_STRIPPED | PKT_RX_VLAN;
1537                 mb->vlan_tci_outer = mb->vlan_tci;
1538                 mb->vlan_tci = rte_le_to_cpu_16(rxdp->wb.l2tag2_2nd);
1539                 PMD_RX_LOG(DEBUG, "Descriptor l2tag2_1: %u, l2tag2_2: %u",
1540                            rte_le_to_cpu_16(rxdp->wb.l2tag2_1st),
1541                            rte_le_to_cpu_16(rxdp->wb.l2tag2_2nd));
1542         } else {
1543                 mb->vlan_tci_outer = 0;
1544         }
1545 #endif
1546         PMD_RX_LOG(DEBUG, "Mbuf vlan_tci: %u, vlan_tci_outer: %u",
1547                    mb->vlan_tci, mb->vlan_tci_outer);
1548 }
1549
1550 #define ICE_LOOK_AHEAD 8
1551 #if (ICE_LOOK_AHEAD != 8)
1552 #error "PMD ICE: ICE_LOOK_AHEAD must be 8\n"
1553 #endif
1554 static inline int
1555 ice_rx_scan_hw_ring(struct ice_rx_queue *rxq)
1556 {
1557         volatile union ice_rx_flex_desc *rxdp;
1558         struct ice_rx_entry *rxep;
1559         struct rte_mbuf *mb;
1560         uint16_t stat_err0;
1561         uint16_t pkt_len;
1562         int32_t s[ICE_LOOK_AHEAD], nb_dd;
1563         int32_t i, j, nb_rx = 0;
1564         uint64_t pkt_flags = 0;
1565         uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl;
1566         struct ice_vsi *vsi = rxq->vsi;
1567         struct ice_hw *hw = ICE_VSI_TO_HW(vsi);
1568         uint64_t ts_ns;
1569         struct ice_adapter *ad = rxq->vsi->adapter;
1570
1571         rxdp = &rxq->rx_ring[rxq->rx_tail];
1572         rxep = &rxq->sw_ring[rxq->rx_tail];
1573
1574         stat_err0 = rte_le_to_cpu_16(rxdp->wb.status_error0);
1575
1576         /* Make sure there is at least 1 packet to receive */
1577         if (!(stat_err0 & (1 << ICE_RX_FLEX_DESC_STATUS0_DD_S)))
1578                 return 0;
1579
1580         /**
1581          * Scan LOOK_AHEAD descriptors at a time to determine which
1582          * descriptors reference packets that are ready to be received.
1583          */
1584         for (i = 0; i < ICE_RX_MAX_BURST; i += ICE_LOOK_AHEAD,
1585              rxdp += ICE_LOOK_AHEAD, rxep += ICE_LOOK_AHEAD) {
1586                 /* Read desc statuses backwards to avoid race condition */
1587                 for (j = ICE_LOOK_AHEAD - 1; j >= 0; j--)
1588                         s[j] = rte_le_to_cpu_16(rxdp[j].wb.status_error0);
1589
1590                 rte_smp_rmb();
1591
1592                 /* Compute how many status bits were set */
1593                 for (j = 0, nb_dd = 0; j < ICE_LOOK_AHEAD; j++)
1594                         nb_dd += s[j] & (1 << ICE_RX_FLEX_DESC_STATUS0_DD_S);
1595
1596                 nb_rx += nb_dd;
1597
1598                 /* Translate descriptor info to mbuf parameters */
1599                 for (j = 0; j < nb_dd; j++) {
1600                         mb = rxep[j].mbuf;
1601                         pkt_len = (rte_le_to_cpu_16(rxdp[j].wb.pkt_len) &
1602                                    ICE_RX_FLX_DESC_PKT_LEN_M) - rxq->crc_len;
1603                         mb->data_len = pkt_len;
1604                         mb->pkt_len = pkt_len;
1605                         mb->ol_flags = 0;
1606                         stat_err0 = rte_le_to_cpu_16(rxdp[j].wb.status_error0);
1607                         pkt_flags = ice_rxd_error_to_pkt_flags(stat_err0);
1608                         mb->packet_type = ptype_tbl[ICE_RX_FLEX_DESC_PTYPE_M &
1609                                 rte_le_to_cpu_16(rxdp[j].wb.ptype_flex_flags0)];
1610                         ice_rxd_to_vlan_tci(mb, &rxdp[j]);
1611                         rxq->rxd_to_pkt_fields(rxq, mb, &rxdp[j]);
1612
1613                         if (rxq->offloads & DEV_RX_OFFLOAD_TIMESTAMP) {
1614                                 ts_ns = ice_tstamp_convert_32b_64b(hw,
1615                                         rte_le_to_cpu_32(rxdp[j].wb.flex_ts.ts_high));
1616                                 if (ice_timestamp_dynflag > 0) {
1617                                         *RTE_MBUF_DYNFIELD(mb,
1618                                                 ice_timestamp_dynfield_offset,
1619                                                 rte_mbuf_timestamp_t *) = ts_ns;
1620                                         mb->ol_flags |= ice_timestamp_dynflag;
1621                                 }
1622                         }
1623
1624                         if (ad->ptp_ena && ((mb->packet_type &
1625                             RTE_PTYPE_L2_MASK) == RTE_PTYPE_L2_ETHER_TIMESYNC)) {
1626                                 rxq->time_high =
1627                                    rte_le_to_cpu_32(rxdp[j].wb.flex_ts.ts_high);
1628                                 mb->timesync = rxq->queue_id;
1629                                 pkt_flags |= PKT_RX_IEEE1588_PTP;
1630                         }
1631
1632                         mb->ol_flags |= pkt_flags;
1633                 }
1634
1635                 for (j = 0; j < ICE_LOOK_AHEAD; j++)
1636                         rxq->rx_stage[i + j] = rxep[j].mbuf;
1637
1638                 if (nb_dd != ICE_LOOK_AHEAD)
1639                         break;
1640         }
1641
1642         /* Clear software ring entries */
1643         for (i = 0; i < nb_rx; i++)
1644                 rxq->sw_ring[rxq->rx_tail + i].mbuf = NULL;
1645
1646         PMD_RX_LOG(DEBUG, "ice_rx_scan_hw_ring: "
1647                    "port_id=%u, queue_id=%u, nb_rx=%d",
1648                    rxq->port_id, rxq->queue_id, nb_rx);
1649
1650         return nb_rx;
1651 }
1652
1653 static inline uint16_t
1654 ice_rx_fill_from_stage(struct ice_rx_queue *rxq,
1655                        struct rte_mbuf **rx_pkts,
1656                        uint16_t nb_pkts)
1657 {
1658         uint16_t i;
1659         struct rte_mbuf **stage = &rxq->rx_stage[rxq->rx_next_avail];
1660
1661         nb_pkts = (uint16_t)RTE_MIN(nb_pkts, rxq->rx_nb_avail);
1662
1663         for (i = 0; i < nb_pkts; i++)
1664                 rx_pkts[i] = stage[i];
1665
1666         rxq->rx_nb_avail = (uint16_t)(rxq->rx_nb_avail - nb_pkts);
1667         rxq->rx_next_avail = (uint16_t)(rxq->rx_next_avail + nb_pkts);
1668
1669         return nb_pkts;
1670 }
1671
1672 static inline int
1673 ice_rx_alloc_bufs(struct ice_rx_queue *rxq)
1674 {
1675         volatile union ice_rx_flex_desc *rxdp;
1676         struct ice_rx_entry *rxep;
1677         struct rte_mbuf *mb;
1678         uint16_t alloc_idx, i;
1679         uint64_t dma_addr;
1680         int diag;
1681
1682         /* Allocate buffers in bulk */
1683         alloc_idx = (uint16_t)(rxq->rx_free_trigger -
1684                                (rxq->rx_free_thresh - 1));
1685         rxep = &rxq->sw_ring[alloc_idx];
1686         diag = rte_mempool_get_bulk(rxq->mp, (void *)rxep,
1687                                     rxq->rx_free_thresh);
1688         if (unlikely(diag != 0)) {
1689                 PMD_RX_LOG(ERR, "Failed to get mbufs in bulk");
1690                 return -ENOMEM;
1691         }
1692
1693         rxdp = &rxq->rx_ring[alloc_idx];
1694         for (i = 0; i < rxq->rx_free_thresh; i++) {
1695                 if (likely(i < (rxq->rx_free_thresh - 1)))
1696                         /* Prefetch next mbuf */
1697                         rte_prefetch0(rxep[i + 1].mbuf);
1698
1699                 mb = rxep[i].mbuf;
1700                 rte_mbuf_refcnt_set(mb, 1);
1701                 mb->next = NULL;
1702                 mb->data_off = RTE_PKTMBUF_HEADROOM;
1703                 mb->nb_segs = 1;
1704                 mb->port = rxq->port_id;
1705                 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mb));
1706                 rxdp[i].read.hdr_addr = 0;
1707                 rxdp[i].read.pkt_addr = dma_addr;
1708         }
1709
1710         /* Update rx tail regsiter */
1711         ICE_PCI_REG_WRITE(rxq->qrx_tail, rxq->rx_free_trigger);
1712
1713         rxq->rx_free_trigger =
1714                 (uint16_t)(rxq->rx_free_trigger + rxq->rx_free_thresh);
1715         if (rxq->rx_free_trigger >= rxq->nb_rx_desc)
1716                 rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1);
1717
1718         return 0;
1719 }
1720
1721 static inline uint16_t
1722 rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
1723 {
1724         struct ice_rx_queue *rxq = (struct ice_rx_queue *)rx_queue;
1725         uint16_t nb_rx = 0;
1726
1727         if (!nb_pkts)
1728                 return 0;
1729
1730         if (rxq->rx_nb_avail)
1731                 return ice_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
1732
1733         nb_rx = (uint16_t)ice_rx_scan_hw_ring(rxq);
1734         rxq->rx_next_avail = 0;
1735         rxq->rx_nb_avail = nb_rx;
1736         rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_rx);
1737
1738         if (rxq->rx_tail > rxq->rx_free_trigger) {
1739                 if (ice_rx_alloc_bufs(rxq) != 0) {
1740                         uint16_t i, j;
1741
1742                         rxq->vsi->adapter->pf.dev_data->rx_mbuf_alloc_failed +=
1743                                 rxq->rx_free_thresh;
1744                         PMD_RX_LOG(DEBUG, "Rx mbuf alloc failed for "
1745                                    "port_id=%u, queue_id=%u",
1746                                    rxq->port_id, rxq->queue_id);
1747                         rxq->rx_nb_avail = 0;
1748                         rxq->rx_tail = (uint16_t)(rxq->rx_tail - nb_rx);
1749                         for (i = 0, j = rxq->rx_tail; i < nb_rx; i++, j++)
1750                                 rxq->sw_ring[j].mbuf = rxq->rx_stage[i];
1751
1752                         return 0;
1753                 }
1754         }
1755
1756         if (rxq->rx_tail >= rxq->nb_rx_desc)
1757                 rxq->rx_tail = 0;
1758
1759         if (rxq->rx_nb_avail)
1760                 return ice_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
1761
1762         return 0;
1763 }
1764
1765 static uint16_t
1766 ice_recv_pkts_bulk_alloc(void *rx_queue,
1767                          struct rte_mbuf **rx_pkts,
1768                          uint16_t nb_pkts)
1769 {
1770         uint16_t nb_rx = 0;
1771         uint16_t n;
1772         uint16_t count;
1773
1774         if (unlikely(nb_pkts == 0))
1775                 return nb_rx;
1776
1777         if (likely(nb_pkts <= ICE_RX_MAX_BURST))
1778                 return rx_recv_pkts(rx_queue, rx_pkts, nb_pkts);
1779
1780         while (nb_pkts) {
1781                 n = RTE_MIN(nb_pkts, ICE_RX_MAX_BURST);
1782                 count = rx_recv_pkts(rx_queue, &rx_pkts[nb_rx], n);
1783                 nb_rx = (uint16_t)(nb_rx + count);
1784                 nb_pkts = (uint16_t)(nb_pkts - count);
1785                 if (count < n)
1786                         break;
1787         }
1788
1789         return nb_rx;
1790 }
1791
1792 static uint16_t
1793 ice_recv_scattered_pkts(void *rx_queue,
1794                         struct rte_mbuf **rx_pkts,
1795                         uint16_t nb_pkts)
1796 {
1797         struct ice_rx_queue *rxq = rx_queue;
1798         volatile union ice_rx_flex_desc *rx_ring = rxq->rx_ring;
1799         volatile union ice_rx_flex_desc *rxdp;
1800         union ice_rx_flex_desc rxd;
1801         struct ice_rx_entry *sw_ring = rxq->sw_ring;
1802         struct ice_rx_entry *rxe;
1803         struct rte_mbuf *first_seg = rxq->pkt_first_seg;
1804         struct rte_mbuf *last_seg = rxq->pkt_last_seg;
1805         struct rte_mbuf *nmb; /* new allocated mbuf */
1806         struct rte_mbuf *rxm; /* pointer to store old mbuf in SW ring */
1807         uint16_t rx_id = rxq->rx_tail;
1808         uint16_t nb_rx = 0;
1809         uint16_t nb_hold = 0;
1810         uint16_t rx_packet_len;
1811         uint16_t rx_stat_err0;
1812         uint64_t dma_addr;
1813         uint64_t pkt_flags;
1814         uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl;
1815         struct ice_vsi *vsi = rxq->vsi;
1816         struct ice_hw *hw = ICE_VSI_TO_HW(vsi);
1817         uint64_t ts_ns;
1818         struct ice_adapter *ad = rxq->vsi->adapter;
1819
1820         while (nb_rx < nb_pkts) {
1821                 rxdp = &rx_ring[rx_id];
1822                 rx_stat_err0 = rte_le_to_cpu_16(rxdp->wb.status_error0);
1823
1824                 /* Check the DD bit first */
1825                 if (!(rx_stat_err0 & (1 << ICE_RX_FLEX_DESC_STATUS0_DD_S)))
1826                         break;
1827
1828                 /* allocate mbuf */
1829                 nmb = rte_mbuf_raw_alloc(rxq->mp);
1830                 if (unlikely(!nmb)) {
1831                         rxq->vsi->adapter->pf.dev_data->rx_mbuf_alloc_failed++;
1832                         break;
1833                 }
1834                 rxd = *rxdp; /* copy descriptor in ring to temp variable*/
1835
1836                 nb_hold++;
1837                 rxe = &sw_ring[rx_id]; /* get corresponding mbuf in SW ring */
1838                 rx_id++;
1839                 if (unlikely(rx_id == rxq->nb_rx_desc))
1840                         rx_id = 0;
1841
1842                 /* Prefetch next mbuf */
1843                 rte_prefetch0(sw_ring[rx_id].mbuf);
1844
1845                 /**
1846                  * When next RX descriptor is on a cache line boundary,
1847                  * prefetch the next 4 RX descriptors and next 8 pointers
1848                  * to mbufs.
1849                  */
1850                 if ((rx_id & 0x3) == 0) {
1851                         rte_prefetch0(&rx_ring[rx_id]);
1852                         rte_prefetch0(&sw_ring[rx_id]);
1853                 }
1854
1855                 rxm = rxe->mbuf;
1856                 rxe->mbuf = nmb;
1857                 dma_addr =
1858                         rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
1859
1860                 /* Set data buffer address and data length of the mbuf */
1861                 rxdp->read.hdr_addr = 0;
1862                 rxdp->read.pkt_addr = dma_addr;
1863                 rx_packet_len = rte_le_to_cpu_16(rxd.wb.pkt_len) &
1864                                 ICE_RX_FLX_DESC_PKT_LEN_M;
1865                 rxm->data_len = rx_packet_len;
1866                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1867
1868                 /**
1869                  * If this is the first buffer of the received packet, set the
1870                  * pointer to the first mbuf of the packet and initialize its
1871                  * context. Otherwise, update the total length and the number
1872                  * of segments of the current scattered packet, and update the
1873                  * pointer to the last mbuf of the current packet.
1874                  */
1875                 if (!first_seg) {
1876                         first_seg = rxm;
1877                         first_seg->nb_segs = 1;
1878                         first_seg->pkt_len = rx_packet_len;
1879                 } else {
1880                         first_seg->pkt_len =
1881                                 (uint16_t)(first_seg->pkt_len +
1882                                            rx_packet_len);
1883                         first_seg->nb_segs++;
1884                         last_seg->next = rxm;
1885                 }
1886
1887                 /**
1888                  * If this is not the last buffer of the received packet,
1889                  * update the pointer to the last mbuf of the current scattered
1890                  * packet and continue to parse the RX ring.
1891                  */
1892                 if (!(rx_stat_err0 & (1 << ICE_RX_FLEX_DESC_STATUS0_EOF_S))) {
1893                         last_seg = rxm;
1894                         continue;
1895                 }
1896
1897                 /**
1898                  * This is the last buffer of the received packet. If the CRC
1899                  * is not stripped by the hardware:
1900                  *  - Subtract the CRC length from the total packet length.
1901                  *  - If the last buffer only contains the whole CRC or a part
1902                  *  of it, free the mbuf associated to the last buffer. If part
1903                  *  of the CRC is also contained in the previous mbuf, subtract
1904                  *  the length of that CRC part from the data length of the
1905                  *  previous mbuf.
1906                  */
1907                 rxm->next = NULL;
1908                 if (unlikely(rxq->crc_len > 0)) {
1909                         first_seg->pkt_len -= RTE_ETHER_CRC_LEN;
1910                         if (rx_packet_len <= RTE_ETHER_CRC_LEN) {
1911                                 rte_pktmbuf_free_seg(rxm);
1912                                 first_seg->nb_segs--;
1913                                 last_seg->data_len =
1914                                         (uint16_t)(last_seg->data_len -
1915                                         (RTE_ETHER_CRC_LEN - rx_packet_len));
1916                                 last_seg->next = NULL;
1917                         } else
1918                                 rxm->data_len = (uint16_t)(rx_packet_len -
1919                                                            RTE_ETHER_CRC_LEN);
1920                 }
1921
1922                 first_seg->port = rxq->port_id;
1923                 first_seg->ol_flags = 0;
1924                 first_seg->packet_type = ptype_tbl[ICE_RX_FLEX_DESC_PTYPE_M &
1925                         rte_le_to_cpu_16(rxd.wb.ptype_flex_flags0)];
1926                 ice_rxd_to_vlan_tci(first_seg, &rxd);
1927                 rxq->rxd_to_pkt_fields(rxq, first_seg, &rxd);
1928                 pkt_flags = ice_rxd_error_to_pkt_flags(rx_stat_err0);
1929
1930                 if (rxq->offloads & DEV_RX_OFFLOAD_TIMESTAMP) {
1931                         ts_ns = ice_tstamp_convert_32b_64b(hw,
1932                                 rte_le_to_cpu_32(rxd.wb.flex_ts.ts_high));
1933                         if (ice_timestamp_dynflag > 0) {
1934                                 *RTE_MBUF_DYNFIELD(first_seg,
1935                                         ice_timestamp_dynfield_offset,
1936                                         rte_mbuf_timestamp_t *) = ts_ns;
1937                                 first_seg->ol_flags |= ice_timestamp_dynflag;
1938                         }
1939                 }
1940
1941                 if (ad->ptp_ena && ((first_seg->packet_type & RTE_PTYPE_L2_MASK)
1942                     == RTE_PTYPE_L2_ETHER_TIMESYNC)) {
1943                         rxq->time_high =
1944                            rte_le_to_cpu_32(rxd.wb.flex_ts.ts_high);
1945                         first_seg->timesync = rxq->queue_id;
1946                         pkt_flags |= PKT_RX_IEEE1588_PTP;
1947                 }
1948
1949                 first_seg->ol_flags |= pkt_flags;
1950                 /* Prefetch data of first segment, if configured to do so. */
1951                 rte_prefetch0(RTE_PTR_ADD(first_seg->buf_addr,
1952                                           first_seg->data_off));
1953                 rx_pkts[nb_rx++] = first_seg;
1954                 first_seg = NULL;
1955         }
1956
1957         /* Record index of the next RX descriptor to probe. */
1958         rxq->rx_tail = rx_id;
1959         rxq->pkt_first_seg = first_seg;
1960         rxq->pkt_last_seg = last_seg;
1961
1962         /**
1963          * If the number of free RX descriptors is greater than the RX free
1964          * threshold of the queue, advance the Receive Descriptor Tail (RDT)
1965          * register. Update the RDT with the value of the last processed RX
1966          * descriptor minus 1, to guarantee that the RDT register is never
1967          * equal to the RDH register, which creates a "full" ring situtation
1968          * from the hardware point of view.
1969          */
1970         nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold);
1971         if (nb_hold > rxq->rx_free_thresh) {
1972                 rx_id = (uint16_t)(rx_id == 0 ?
1973                                    (rxq->nb_rx_desc - 1) : (rx_id - 1));
1974                 /* write TAIL register */
1975                 ICE_PCI_REG_WC_WRITE(rxq->qrx_tail, rx_id);
1976                 nb_hold = 0;
1977         }
1978         rxq->nb_rx_hold = nb_hold;
1979
1980         /* return received packet in the burst */
1981         return nb_rx;
1982 }
1983
1984 const uint32_t *
1985 ice_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1986 {
1987         struct ice_adapter *ad =
1988                 ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
1989         const uint32_t *ptypes;
1990
1991         static const uint32_t ptypes_os[] = {
1992                 /* refers to ice_get_default_pkt_type() */
1993                 RTE_PTYPE_L2_ETHER,
1994                 RTE_PTYPE_L2_ETHER_TIMESYNC,
1995                 RTE_PTYPE_L2_ETHER_LLDP,
1996                 RTE_PTYPE_L2_ETHER_ARP,
1997                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
1998                 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
1999                 RTE_PTYPE_L4_FRAG,
2000                 RTE_PTYPE_L4_ICMP,
2001                 RTE_PTYPE_L4_NONFRAG,
2002                 RTE_PTYPE_L4_SCTP,
2003                 RTE_PTYPE_L4_TCP,
2004                 RTE_PTYPE_L4_UDP,
2005                 RTE_PTYPE_TUNNEL_GRENAT,
2006                 RTE_PTYPE_TUNNEL_IP,
2007                 RTE_PTYPE_INNER_L2_ETHER,
2008                 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN,
2009                 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN,
2010                 RTE_PTYPE_INNER_L4_FRAG,
2011                 RTE_PTYPE_INNER_L4_ICMP,
2012                 RTE_PTYPE_INNER_L4_NONFRAG,
2013                 RTE_PTYPE_INNER_L4_SCTP,
2014                 RTE_PTYPE_INNER_L4_TCP,
2015                 RTE_PTYPE_INNER_L4_UDP,
2016                 RTE_PTYPE_UNKNOWN
2017         };
2018
2019         static const uint32_t ptypes_comms[] = {
2020                 /* refers to ice_get_default_pkt_type() */
2021                 RTE_PTYPE_L2_ETHER,
2022                 RTE_PTYPE_L2_ETHER_TIMESYNC,
2023                 RTE_PTYPE_L2_ETHER_LLDP,
2024                 RTE_PTYPE_L2_ETHER_ARP,
2025                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
2026                 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
2027                 RTE_PTYPE_L4_FRAG,
2028                 RTE_PTYPE_L4_ICMP,
2029                 RTE_PTYPE_L4_NONFRAG,
2030                 RTE_PTYPE_L4_SCTP,
2031                 RTE_PTYPE_L4_TCP,
2032                 RTE_PTYPE_L4_UDP,
2033                 RTE_PTYPE_TUNNEL_GRENAT,
2034                 RTE_PTYPE_TUNNEL_IP,
2035                 RTE_PTYPE_INNER_L2_ETHER,
2036                 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN,
2037                 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN,
2038                 RTE_PTYPE_INNER_L4_FRAG,
2039                 RTE_PTYPE_INNER_L4_ICMP,
2040                 RTE_PTYPE_INNER_L4_NONFRAG,
2041                 RTE_PTYPE_INNER_L4_SCTP,
2042                 RTE_PTYPE_INNER_L4_TCP,
2043                 RTE_PTYPE_INNER_L4_UDP,
2044                 RTE_PTYPE_TUNNEL_GTPC,
2045                 RTE_PTYPE_TUNNEL_GTPU,
2046                 RTE_PTYPE_L2_ETHER_PPPOE,
2047                 RTE_PTYPE_UNKNOWN
2048         };
2049
2050         if (ad->active_pkg_type == ICE_PKG_TYPE_COMMS)
2051                 ptypes = ptypes_comms;
2052         else
2053                 ptypes = ptypes_os;
2054
2055         if (dev->rx_pkt_burst == ice_recv_pkts ||
2056             dev->rx_pkt_burst == ice_recv_pkts_bulk_alloc ||
2057             dev->rx_pkt_burst == ice_recv_scattered_pkts)
2058                 return ptypes;
2059
2060 #ifdef RTE_ARCH_X86
2061         if (dev->rx_pkt_burst == ice_recv_pkts_vec ||
2062             dev->rx_pkt_burst == ice_recv_scattered_pkts_vec ||
2063 #ifdef CC_AVX512_SUPPORT
2064             dev->rx_pkt_burst == ice_recv_pkts_vec_avx512 ||
2065             dev->rx_pkt_burst == ice_recv_pkts_vec_avx512_offload ||
2066             dev->rx_pkt_burst == ice_recv_scattered_pkts_vec_avx512 ||
2067             dev->rx_pkt_burst == ice_recv_scattered_pkts_vec_avx512_offload ||
2068 #endif
2069             dev->rx_pkt_burst == ice_recv_pkts_vec_avx2 ||
2070             dev->rx_pkt_burst == ice_recv_pkts_vec_avx2_offload ||
2071             dev->rx_pkt_burst == ice_recv_scattered_pkts_vec_avx2 ||
2072             dev->rx_pkt_burst == ice_recv_scattered_pkts_vec_avx2_offload)
2073                 return ptypes;
2074 #endif
2075
2076         return NULL;
2077 }
2078
2079 int
2080 ice_rx_descriptor_status(void *rx_queue, uint16_t offset)
2081 {
2082         volatile union ice_rx_flex_desc *rxdp;
2083         struct ice_rx_queue *rxq = rx_queue;
2084         uint32_t desc;
2085
2086         if (unlikely(offset >= rxq->nb_rx_desc))
2087                 return -EINVAL;
2088
2089         if (offset >= rxq->nb_rx_desc - rxq->nb_rx_hold)
2090                 return RTE_ETH_RX_DESC_UNAVAIL;
2091
2092         desc = rxq->rx_tail + offset;
2093         if (desc >= rxq->nb_rx_desc)
2094                 desc -= rxq->nb_rx_desc;
2095
2096         rxdp = &rxq->rx_ring[desc];
2097         if (rte_le_to_cpu_16(rxdp->wb.status_error0) &
2098             (1 << ICE_RX_FLEX_DESC_STATUS0_DD_S))
2099                 return RTE_ETH_RX_DESC_DONE;
2100
2101         return RTE_ETH_RX_DESC_AVAIL;
2102 }
2103
2104 int
2105 ice_tx_descriptor_status(void *tx_queue, uint16_t offset)
2106 {
2107         struct ice_tx_queue *txq = tx_queue;
2108         volatile uint64_t *status;
2109         uint64_t mask, expect;
2110         uint32_t desc;
2111
2112         if (unlikely(offset >= txq->nb_tx_desc))
2113                 return -EINVAL;
2114
2115         desc = txq->tx_tail + offset;
2116         /* go to next desc that has the RS bit */
2117         desc = ((desc + txq->tx_rs_thresh - 1) / txq->tx_rs_thresh) *
2118                 txq->tx_rs_thresh;
2119         if (desc >= txq->nb_tx_desc) {
2120                 desc -= txq->nb_tx_desc;
2121                 if (desc >= txq->nb_tx_desc)
2122                         desc -= txq->nb_tx_desc;
2123         }
2124
2125         status = &txq->tx_ring[desc].cmd_type_offset_bsz;
2126         mask = rte_cpu_to_le_64(ICE_TXD_QW1_DTYPE_M);
2127         expect = rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DESC_DONE <<
2128                                   ICE_TXD_QW1_DTYPE_S);
2129         if ((*status & mask) == expect)
2130                 return RTE_ETH_TX_DESC_DONE;
2131
2132         return RTE_ETH_TX_DESC_FULL;
2133 }
2134
2135 void
2136 ice_free_queues(struct rte_eth_dev *dev)
2137 {
2138         uint16_t i;
2139
2140         PMD_INIT_FUNC_TRACE();
2141
2142         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2143                 if (!dev->data->rx_queues[i])
2144                         continue;
2145                 ice_rx_queue_release(dev->data->rx_queues[i]);
2146                 dev->data->rx_queues[i] = NULL;
2147                 rte_eth_dma_zone_free(dev, "rx_ring", i);
2148         }
2149         dev->data->nb_rx_queues = 0;
2150
2151         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2152                 if (!dev->data->tx_queues[i])
2153                         continue;
2154                 ice_tx_queue_release(dev->data->tx_queues[i]);
2155                 dev->data->tx_queues[i] = NULL;
2156                 rte_eth_dma_zone_free(dev, "tx_ring", i);
2157         }
2158         dev->data->nb_tx_queues = 0;
2159 }
2160
2161 #define ICE_FDIR_NUM_TX_DESC  ICE_MIN_RING_DESC
2162 #define ICE_FDIR_NUM_RX_DESC  ICE_MIN_RING_DESC
2163
2164 int
2165 ice_fdir_setup_tx_resources(struct ice_pf *pf)
2166 {
2167         struct ice_tx_queue *txq;
2168         const struct rte_memzone *tz = NULL;
2169         uint32_t ring_size;
2170         struct rte_eth_dev *dev;
2171
2172         if (!pf) {
2173                 PMD_DRV_LOG(ERR, "PF is not available");
2174                 return -EINVAL;
2175         }
2176
2177         dev = &rte_eth_devices[pf->adapter->pf.dev_data->port_id];
2178
2179         /* Allocate the TX queue data structure. */
2180         txq = rte_zmalloc_socket("ice fdir tx queue",
2181                                  sizeof(struct ice_tx_queue),
2182                                  RTE_CACHE_LINE_SIZE,
2183                                  SOCKET_ID_ANY);
2184         if (!txq) {
2185                 PMD_DRV_LOG(ERR, "Failed to allocate memory for "
2186                             "tx queue structure.");
2187                 return -ENOMEM;
2188         }
2189
2190         /* Allocate TX hardware ring descriptors. */
2191         ring_size = sizeof(struct ice_tx_desc) * ICE_FDIR_NUM_TX_DESC;
2192         ring_size = RTE_ALIGN(ring_size, ICE_DMA_MEM_ALIGN);
2193
2194         tz = rte_eth_dma_zone_reserve(dev, "fdir_tx_ring",
2195                                       ICE_FDIR_QUEUE_ID, ring_size,
2196                                       ICE_RING_BASE_ALIGN, SOCKET_ID_ANY);
2197         if (!tz) {
2198                 ice_tx_queue_release(txq);
2199                 PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for TX.");
2200                 return -ENOMEM;
2201         }
2202
2203         txq->nb_tx_desc = ICE_FDIR_NUM_TX_DESC;
2204         txq->queue_id = ICE_FDIR_QUEUE_ID;
2205         txq->reg_idx = pf->fdir.fdir_vsi->base_queue;
2206         txq->vsi = pf->fdir.fdir_vsi;
2207
2208         txq->tx_ring_dma = tz->iova;
2209         txq->tx_ring = (struct ice_tx_desc *)tz->addr;
2210         /*
2211          * don't need to allocate software ring and reset for the fdir
2212          * program queue just set the queue has been configured.
2213          */
2214         txq->q_set = true;
2215         pf->fdir.txq = txq;
2216
2217         txq->tx_rel_mbufs = _ice_tx_queue_release_mbufs;
2218
2219         return ICE_SUCCESS;
2220 }
2221
2222 int
2223 ice_fdir_setup_rx_resources(struct ice_pf *pf)
2224 {
2225         struct ice_rx_queue *rxq;
2226         const struct rte_memzone *rz = NULL;
2227         uint32_t ring_size;
2228         struct rte_eth_dev *dev;
2229
2230         if (!pf) {
2231                 PMD_DRV_LOG(ERR, "PF is not available");
2232                 return -EINVAL;
2233         }
2234
2235         dev = &rte_eth_devices[pf->adapter->pf.dev_data->port_id];
2236
2237         /* Allocate the RX queue data structure. */
2238         rxq = rte_zmalloc_socket("ice fdir rx queue",
2239                                  sizeof(struct ice_rx_queue),
2240                                  RTE_CACHE_LINE_SIZE,
2241                                  SOCKET_ID_ANY);
2242         if (!rxq) {
2243                 PMD_DRV_LOG(ERR, "Failed to allocate memory for "
2244                             "rx queue structure.");
2245                 return -ENOMEM;
2246         }
2247
2248         /* Allocate RX hardware ring descriptors. */
2249         ring_size = sizeof(union ice_32byte_rx_desc) * ICE_FDIR_NUM_RX_DESC;
2250         ring_size = RTE_ALIGN(ring_size, ICE_DMA_MEM_ALIGN);
2251
2252         rz = rte_eth_dma_zone_reserve(dev, "fdir_rx_ring",
2253                                       ICE_FDIR_QUEUE_ID, ring_size,
2254                                       ICE_RING_BASE_ALIGN, SOCKET_ID_ANY);
2255         if (!rz) {
2256                 ice_rx_queue_release(rxq);
2257                 PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for RX.");
2258                 return -ENOMEM;
2259         }
2260
2261         rxq->nb_rx_desc = ICE_FDIR_NUM_RX_DESC;
2262         rxq->queue_id = ICE_FDIR_QUEUE_ID;
2263         rxq->reg_idx = pf->fdir.fdir_vsi->base_queue;
2264         rxq->vsi = pf->fdir.fdir_vsi;
2265
2266         rxq->rx_ring_dma = rz->iova;
2267         memset(rz->addr, 0, ICE_FDIR_NUM_RX_DESC *
2268                sizeof(union ice_32byte_rx_desc));
2269         rxq->rx_ring = (union ice_rx_flex_desc *)rz->addr;
2270
2271         /*
2272          * Don't need to allocate software ring and reset for the fdir
2273          * rx queue, just set the queue has been configured.
2274          */
2275         rxq->q_set = true;
2276         pf->fdir.rxq = rxq;
2277
2278         rxq->rx_rel_mbufs = _ice_rx_queue_release_mbufs;
2279
2280         return ICE_SUCCESS;
2281 }
2282
2283 uint16_t
2284 ice_recv_pkts(void *rx_queue,
2285               struct rte_mbuf **rx_pkts,
2286               uint16_t nb_pkts)
2287 {
2288         struct ice_rx_queue *rxq = rx_queue;
2289         volatile union ice_rx_flex_desc *rx_ring = rxq->rx_ring;
2290         volatile union ice_rx_flex_desc *rxdp;
2291         union ice_rx_flex_desc rxd;
2292         struct ice_rx_entry *sw_ring = rxq->sw_ring;
2293         struct ice_rx_entry *rxe;
2294         struct rte_mbuf *nmb; /* new allocated mbuf */
2295         struct rte_mbuf *rxm; /* pointer to store old mbuf in SW ring */
2296         uint16_t rx_id = rxq->rx_tail;
2297         uint16_t nb_rx = 0;
2298         uint16_t nb_hold = 0;
2299         uint16_t rx_packet_len;
2300         uint16_t rx_stat_err0;
2301         uint64_t dma_addr;
2302         uint64_t pkt_flags;
2303         uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl;
2304         struct ice_vsi *vsi = rxq->vsi;
2305         struct ice_hw *hw = ICE_VSI_TO_HW(vsi);
2306         uint64_t ts_ns;
2307         struct ice_adapter *ad = rxq->vsi->adapter;
2308
2309         while (nb_rx < nb_pkts) {
2310                 rxdp = &rx_ring[rx_id];
2311                 rx_stat_err0 = rte_le_to_cpu_16(rxdp->wb.status_error0);
2312
2313                 /* Check the DD bit first */
2314                 if (!(rx_stat_err0 & (1 << ICE_RX_FLEX_DESC_STATUS0_DD_S)))
2315                         break;
2316
2317                 /* allocate mbuf */
2318                 nmb = rte_mbuf_raw_alloc(rxq->mp);
2319                 if (unlikely(!nmb)) {
2320                         rxq->vsi->adapter->pf.dev_data->rx_mbuf_alloc_failed++;
2321                         break;
2322                 }
2323                 rxd = *rxdp; /* copy descriptor in ring to temp variable*/
2324
2325                 nb_hold++;
2326                 rxe = &sw_ring[rx_id]; /* get corresponding mbuf in SW ring */
2327                 rx_id++;
2328                 if (unlikely(rx_id == rxq->nb_rx_desc))
2329                         rx_id = 0;
2330                 rxm = rxe->mbuf;
2331                 rxe->mbuf = nmb;
2332                 dma_addr =
2333                         rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
2334
2335                 /**
2336                  * fill the read format of descriptor with physic address in
2337                  * new allocated mbuf: nmb
2338                  */
2339                 rxdp->read.hdr_addr = 0;
2340                 rxdp->read.pkt_addr = dma_addr;
2341
2342                 /* calculate rx_packet_len of the received pkt */
2343                 rx_packet_len = (rte_le_to_cpu_16(rxd.wb.pkt_len) &
2344                                  ICE_RX_FLX_DESC_PKT_LEN_M) - rxq->crc_len;
2345
2346                 /* fill old mbuf with received descriptor: rxd */
2347                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
2348                 rte_prefetch0(RTE_PTR_ADD(rxm->buf_addr, RTE_PKTMBUF_HEADROOM));
2349                 rxm->nb_segs = 1;
2350                 rxm->next = NULL;
2351                 rxm->pkt_len = rx_packet_len;
2352                 rxm->data_len = rx_packet_len;
2353                 rxm->port = rxq->port_id;
2354                 rxm->packet_type = ptype_tbl[ICE_RX_FLEX_DESC_PTYPE_M &
2355                         rte_le_to_cpu_16(rxd.wb.ptype_flex_flags0)];
2356                 ice_rxd_to_vlan_tci(rxm, &rxd);
2357                 rxq->rxd_to_pkt_fields(rxq, rxm, &rxd);
2358                 pkt_flags = ice_rxd_error_to_pkt_flags(rx_stat_err0);
2359
2360                 if (rxq->offloads & DEV_RX_OFFLOAD_TIMESTAMP) {
2361                         ts_ns = ice_tstamp_convert_32b_64b(hw,
2362                                 rte_le_to_cpu_32(rxd.wb.flex_ts.ts_high));
2363                         if (ice_timestamp_dynflag > 0) {
2364                                 *RTE_MBUF_DYNFIELD(rxm,
2365                                         ice_timestamp_dynfield_offset,
2366                                         rte_mbuf_timestamp_t *) = ts_ns;
2367                                 rxm->ol_flags |= ice_timestamp_dynflag;
2368                         }
2369                 }
2370
2371                 if (ad->ptp_ena && ((rxm->packet_type & RTE_PTYPE_L2_MASK) ==
2372                     RTE_PTYPE_L2_ETHER_TIMESYNC)) {
2373                         rxq->time_high =
2374                            rte_le_to_cpu_32(rxd.wb.flex_ts.ts_high);
2375                         rxm->timesync = rxq->queue_id;
2376                         pkt_flags |= PKT_RX_IEEE1588_PTP;
2377                 }
2378
2379                 rxm->ol_flags |= pkt_flags;
2380                 /* copy old mbuf to rx_pkts */
2381                 rx_pkts[nb_rx++] = rxm;
2382         }
2383         rxq->rx_tail = rx_id;
2384         /**
2385          * If the number of free RX descriptors is greater than the RX free
2386          * threshold of the queue, advance the receive tail register of queue.
2387          * Update that register with the value of the last processed RX
2388          * descriptor minus 1.
2389          */
2390         nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold);
2391         if (nb_hold > rxq->rx_free_thresh) {
2392                 rx_id = (uint16_t)(rx_id == 0 ?
2393                                    (rxq->nb_rx_desc - 1) : (rx_id - 1));
2394                 /* write TAIL register */
2395                 ICE_PCI_REG_WC_WRITE(rxq->qrx_tail, rx_id);
2396                 nb_hold = 0;
2397         }
2398         rxq->nb_rx_hold = nb_hold;
2399
2400         /* return received packet in the burst */
2401         return nb_rx;
2402 }
2403
2404 static inline void
2405 ice_parse_tunneling_params(uint64_t ol_flags,
2406                             union ice_tx_offload tx_offload,
2407                             uint32_t *cd_tunneling)
2408 {
2409         /* EIPT: External (outer) IP header type */
2410         if (ol_flags & PKT_TX_OUTER_IP_CKSUM)
2411                 *cd_tunneling |= ICE_TX_CTX_EIPT_IPV4;
2412         else if (ol_flags & PKT_TX_OUTER_IPV4)
2413                 *cd_tunneling |= ICE_TX_CTX_EIPT_IPV4_NO_CSUM;
2414         else if (ol_flags & PKT_TX_OUTER_IPV6)
2415                 *cd_tunneling |= ICE_TX_CTX_EIPT_IPV6;
2416
2417         /* EIPLEN: External (outer) IP header length, in DWords */
2418         *cd_tunneling |= (tx_offload.outer_l3_len >> 2) <<
2419                 ICE_TXD_CTX_QW0_EIPLEN_S;
2420
2421         /* L4TUNT: L4 Tunneling Type */
2422         switch (ol_flags & PKT_TX_TUNNEL_MASK) {
2423         case PKT_TX_TUNNEL_IPIP:
2424                 /* for non UDP / GRE tunneling, set to 00b */
2425                 break;
2426         case PKT_TX_TUNNEL_VXLAN:
2427         case PKT_TX_TUNNEL_GTP:
2428         case PKT_TX_TUNNEL_GENEVE:
2429                 *cd_tunneling |= ICE_TXD_CTX_UDP_TUNNELING;
2430                 break;
2431         case PKT_TX_TUNNEL_GRE:
2432                 *cd_tunneling |= ICE_TXD_CTX_GRE_TUNNELING;
2433                 break;
2434         default:
2435                 PMD_TX_LOG(ERR, "Tunnel type not supported");
2436                 return;
2437         }
2438
2439         /* L4TUNLEN: L4 Tunneling Length, in Words
2440          *
2441          * We depend on app to set rte_mbuf.l2_len correctly.
2442          * For IP in GRE it should be set to the length of the GRE
2443          * header;
2444          * For MAC in GRE or MAC in UDP it should be set to the length
2445          * of the GRE or UDP headers plus the inner MAC up to including
2446          * its last Ethertype.
2447          * If MPLS labels exists, it should include them as well.
2448          */
2449         *cd_tunneling |= (tx_offload.l2_len >> 1) <<
2450                 ICE_TXD_CTX_QW0_NATLEN_S;
2451
2452         /**
2453          * Calculate the tunneling UDP checksum.
2454          * Shall be set only if L4TUNT = 01b and EIPT is not zero
2455          */
2456         if (!(*cd_tunneling & ICE_TX_CTX_EIPT_NONE) &&
2457             (*cd_tunneling & ICE_TXD_CTX_UDP_TUNNELING))
2458                 *cd_tunneling |= ICE_TXD_CTX_QW0_L4T_CS_M;
2459 }
2460
2461 static inline void
2462 ice_txd_enable_checksum(uint64_t ol_flags,
2463                         uint32_t *td_cmd,
2464                         uint32_t *td_offset,
2465                         union ice_tx_offload tx_offload)
2466 {
2467         /* Set MACLEN */
2468         if (ol_flags & PKT_TX_TUNNEL_MASK)
2469                 *td_offset |= (tx_offload.outer_l2_len >> 1)
2470                         << ICE_TX_DESC_LEN_MACLEN_S;
2471         else
2472                 *td_offset |= (tx_offload.l2_len >> 1)
2473                         << ICE_TX_DESC_LEN_MACLEN_S;
2474
2475         /* Enable L3 checksum offloads */
2476         if (ol_flags & PKT_TX_IP_CKSUM) {
2477                 *td_cmd |= ICE_TX_DESC_CMD_IIPT_IPV4_CSUM;
2478                 *td_offset |= (tx_offload.l3_len >> 2) <<
2479                               ICE_TX_DESC_LEN_IPLEN_S;
2480         } else if (ol_flags & PKT_TX_IPV4) {
2481                 *td_cmd |= ICE_TX_DESC_CMD_IIPT_IPV4;
2482                 *td_offset |= (tx_offload.l3_len >> 2) <<
2483                               ICE_TX_DESC_LEN_IPLEN_S;
2484         } else if (ol_flags & PKT_TX_IPV6) {
2485                 *td_cmd |= ICE_TX_DESC_CMD_IIPT_IPV6;
2486                 *td_offset |= (tx_offload.l3_len >> 2) <<
2487                               ICE_TX_DESC_LEN_IPLEN_S;
2488         }
2489
2490         if (ol_flags & PKT_TX_TCP_SEG) {
2491                 *td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_TCP;
2492                 *td_offset |= (tx_offload.l4_len >> 2) <<
2493                               ICE_TX_DESC_LEN_L4_LEN_S;
2494                 return;
2495         }
2496
2497         /* Enable L4 checksum offloads */
2498         switch (ol_flags & PKT_TX_L4_MASK) {
2499         case PKT_TX_TCP_CKSUM:
2500                 *td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_TCP;
2501                 *td_offset |= (sizeof(struct rte_tcp_hdr) >> 2) <<
2502                               ICE_TX_DESC_LEN_L4_LEN_S;
2503                 break;
2504         case PKT_TX_SCTP_CKSUM:
2505                 *td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_SCTP;
2506                 *td_offset |= (sizeof(struct rte_sctp_hdr) >> 2) <<
2507                               ICE_TX_DESC_LEN_L4_LEN_S;
2508                 break;
2509         case PKT_TX_UDP_CKSUM:
2510                 *td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_UDP;
2511                 *td_offset |= (sizeof(struct rte_udp_hdr) >> 2) <<
2512                               ICE_TX_DESC_LEN_L4_LEN_S;
2513                 break;
2514         default:
2515                 break;
2516         }
2517 }
2518
2519 static inline int
2520 ice_xmit_cleanup(struct ice_tx_queue *txq)
2521 {
2522         struct ice_tx_entry *sw_ring = txq->sw_ring;
2523         volatile struct ice_tx_desc *txd = txq->tx_ring;
2524         uint16_t last_desc_cleaned = txq->last_desc_cleaned;
2525         uint16_t nb_tx_desc = txq->nb_tx_desc;
2526         uint16_t desc_to_clean_to;
2527         uint16_t nb_tx_to_clean;
2528
2529         /* Determine the last descriptor needing to be cleaned */
2530         desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->tx_rs_thresh);
2531         if (desc_to_clean_to >= nb_tx_desc)
2532                 desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc);
2533
2534         /* Check to make sure the last descriptor to clean is done */
2535         desc_to_clean_to = sw_ring[desc_to_clean_to].last_id;
2536         if (!(txd[desc_to_clean_to].cmd_type_offset_bsz &
2537             rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DESC_DONE))) {
2538                 PMD_TX_LOG(DEBUG, "TX descriptor %4u is not done "
2539                            "(port=%d queue=%d) value=0x%"PRIx64"\n",
2540                            desc_to_clean_to,
2541                            txq->port_id, txq->queue_id,
2542                            txd[desc_to_clean_to].cmd_type_offset_bsz);
2543                 /* Failed to clean any descriptors */
2544                 return -1;
2545         }
2546
2547         /* Figure out how many descriptors will be cleaned */
2548         if (last_desc_cleaned > desc_to_clean_to)
2549                 nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) +
2550                                             desc_to_clean_to);
2551         else
2552                 nb_tx_to_clean = (uint16_t)(desc_to_clean_to -
2553                                             last_desc_cleaned);
2554
2555         /* The last descriptor to clean is done, so that means all the
2556          * descriptors from the last descriptor that was cleaned
2557          * up to the last descriptor with the RS bit set
2558          * are done. Only reset the threshold descriptor.
2559          */
2560         txd[desc_to_clean_to].cmd_type_offset_bsz = 0;
2561
2562         /* Update the txq to reflect the last descriptor that was cleaned */
2563         txq->last_desc_cleaned = desc_to_clean_to;
2564         txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + nb_tx_to_clean);
2565
2566         return 0;
2567 }
2568
2569 /* Construct the tx flags */
2570 static inline uint64_t
2571 ice_build_ctob(uint32_t td_cmd,
2572                uint32_t td_offset,
2573                uint16_t size,
2574                uint32_t td_tag)
2575 {
2576         return rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DATA |
2577                                 ((uint64_t)td_cmd << ICE_TXD_QW1_CMD_S) |
2578                                 ((uint64_t)td_offset << ICE_TXD_QW1_OFFSET_S) |
2579                                 ((uint64_t)size << ICE_TXD_QW1_TX_BUF_SZ_S) |
2580                                 ((uint64_t)td_tag << ICE_TXD_QW1_L2TAG1_S));
2581 }
2582
2583 /* Check if the context descriptor is needed for TX offloading */
2584 static inline uint16_t
2585 ice_calc_context_desc(uint64_t flags)
2586 {
2587         static uint64_t mask = PKT_TX_TCP_SEG |
2588                 PKT_TX_QINQ |
2589                 PKT_TX_OUTER_IP_CKSUM |
2590                 PKT_TX_TUNNEL_MASK |
2591                 PKT_TX_IEEE1588_TMST;
2592
2593         return (flags & mask) ? 1 : 0;
2594 }
2595
2596 /* set ice TSO context descriptor */
2597 static inline uint64_t
2598 ice_set_tso_ctx(struct rte_mbuf *mbuf, union ice_tx_offload tx_offload)
2599 {
2600         uint64_t ctx_desc = 0;
2601         uint32_t cd_cmd, hdr_len, cd_tso_len;
2602
2603         if (!tx_offload.l4_len) {
2604                 PMD_TX_LOG(DEBUG, "L4 length set to 0");
2605                 return ctx_desc;
2606         }
2607
2608         hdr_len = tx_offload.l2_len + tx_offload.l3_len + tx_offload.l4_len;
2609         hdr_len += (mbuf->ol_flags & PKT_TX_TUNNEL_MASK) ?
2610                    tx_offload.outer_l2_len + tx_offload.outer_l3_len : 0;
2611
2612         cd_cmd = ICE_TX_CTX_DESC_TSO;
2613         cd_tso_len = mbuf->pkt_len - hdr_len;
2614         ctx_desc |= ((uint64_t)cd_cmd << ICE_TXD_CTX_QW1_CMD_S) |
2615                     ((uint64_t)cd_tso_len << ICE_TXD_CTX_QW1_TSO_LEN_S) |
2616                     ((uint64_t)mbuf->tso_segsz << ICE_TXD_CTX_QW1_MSS_S);
2617
2618         return ctx_desc;
2619 }
2620
2621 /* HW requires that TX buffer size ranges from 1B up to (16K-1)B. */
2622 #define ICE_MAX_DATA_PER_TXD \
2623         (ICE_TXD_QW1_TX_BUF_SZ_M >> ICE_TXD_QW1_TX_BUF_SZ_S)
2624 /* Calculate the number of TX descriptors needed for each pkt */
2625 static inline uint16_t
2626 ice_calc_pkt_desc(struct rte_mbuf *tx_pkt)
2627 {
2628         struct rte_mbuf *txd = tx_pkt;
2629         uint16_t count = 0;
2630
2631         while (txd != NULL) {
2632                 count += DIV_ROUND_UP(txd->data_len, ICE_MAX_DATA_PER_TXD);
2633                 txd = txd->next;
2634         }
2635
2636         return count;
2637 }
2638
2639 uint16_t
2640 ice_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
2641 {
2642         struct ice_tx_queue *txq;
2643         volatile struct ice_tx_desc *tx_ring;
2644         volatile struct ice_tx_desc *txd;
2645         struct ice_tx_entry *sw_ring;
2646         struct ice_tx_entry *txe, *txn;
2647         struct rte_mbuf *tx_pkt;
2648         struct rte_mbuf *m_seg;
2649         uint32_t cd_tunneling_params;
2650         uint16_t tx_id;
2651         uint16_t nb_tx;
2652         uint16_t nb_used;
2653         uint16_t nb_ctx;
2654         uint32_t td_cmd = 0;
2655         uint32_t td_offset = 0;
2656         uint32_t td_tag = 0;
2657         uint16_t tx_last;
2658         uint16_t slen;
2659         uint64_t buf_dma_addr;
2660         uint64_t ol_flags;
2661         union ice_tx_offload tx_offload = {0};
2662
2663         txq = tx_queue;
2664         sw_ring = txq->sw_ring;
2665         tx_ring = txq->tx_ring;
2666         tx_id = txq->tx_tail;
2667         txe = &sw_ring[tx_id];
2668
2669         /* Check if the descriptor ring needs to be cleaned. */
2670         if (txq->nb_tx_free < txq->tx_free_thresh)
2671                 (void)ice_xmit_cleanup(txq);
2672
2673         for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
2674                 tx_pkt = *tx_pkts++;
2675
2676                 td_cmd = 0;
2677                 td_tag = 0;
2678                 td_offset = 0;
2679                 ol_flags = tx_pkt->ol_flags;
2680                 tx_offload.l2_len = tx_pkt->l2_len;
2681                 tx_offload.l3_len = tx_pkt->l3_len;
2682                 tx_offload.outer_l2_len = tx_pkt->outer_l2_len;
2683                 tx_offload.outer_l3_len = tx_pkt->outer_l3_len;
2684                 tx_offload.l4_len = tx_pkt->l4_len;
2685                 tx_offload.tso_segsz = tx_pkt->tso_segsz;
2686                 /* Calculate the number of context descriptors needed. */
2687                 nb_ctx = ice_calc_context_desc(ol_flags);
2688
2689                 /* The number of descriptors that must be allocated for
2690                  * a packet equals to the number of the segments of that
2691                  * packet plus the number of context descriptor if needed.
2692                  * Recalculate the needed tx descs when TSO enabled in case
2693                  * the mbuf data size exceeds max data size that hw allows
2694                  * per tx desc.
2695                  */
2696                 if (ol_flags & PKT_TX_TCP_SEG)
2697                         nb_used = (uint16_t)(ice_calc_pkt_desc(tx_pkt) +
2698                                              nb_ctx);
2699                 else
2700                         nb_used = (uint16_t)(tx_pkt->nb_segs + nb_ctx);
2701                 tx_last = (uint16_t)(tx_id + nb_used - 1);
2702
2703                 /* Circular ring */
2704                 if (tx_last >= txq->nb_tx_desc)
2705                         tx_last = (uint16_t)(tx_last - txq->nb_tx_desc);
2706
2707                 if (nb_used > txq->nb_tx_free) {
2708                         if (ice_xmit_cleanup(txq) != 0) {
2709                                 if (nb_tx == 0)
2710                                         return 0;
2711                                 goto end_of_tx;
2712                         }
2713                         if (unlikely(nb_used > txq->tx_rs_thresh)) {
2714                                 while (nb_used > txq->nb_tx_free) {
2715                                         if (ice_xmit_cleanup(txq) != 0) {
2716                                                 if (nb_tx == 0)
2717                                                         return 0;
2718                                                 goto end_of_tx;
2719                                         }
2720                                 }
2721                         }
2722                 }
2723
2724                 /* Descriptor based VLAN insertion */
2725                 if (ol_flags & (PKT_TX_VLAN | PKT_TX_QINQ)) {
2726                         td_cmd |= ICE_TX_DESC_CMD_IL2TAG1;
2727                         td_tag = tx_pkt->vlan_tci;
2728                 }
2729
2730                 /* Fill in tunneling parameters if necessary */
2731                 cd_tunneling_params = 0;
2732                 if (ol_flags & PKT_TX_TUNNEL_MASK)
2733                         ice_parse_tunneling_params(ol_flags, tx_offload,
2734                                                    &cd_tunneling_params);
2735
2736                 /* Enable checksum offloading */
2737                 if (ol_flags & ICE_TX_CKSUM_OFFLOAD_MASK)
2738                         ice_txd_enable_checksum(ol_flags, &td_cmd,
2739                                                 &td_offset, tx_offload);
2740
2741                 if (nb_ctx) {
2742                         /* Setup TX context descriptor if required */
2743                         volatile struct ice_tx_ctx_desc *ctx_txd =
2744                                 (volatile struct ice_tx_ctx_desc *)
2745                                         &tx_ring[tx_id];
2746                         uint16_t cd_l2tag2 = 0;
2747                         uint64_t cd_type_cmd_tso_mss = ICE_TX_DESC_DTYPE_CTX;
2748
2749                         txn = &sw_ring[txe->next_id];
2750                         RTE_MBUF_PREFETCH_TO_FREE(txn->mbuf);
2751                         if (txe->mbuf) {
2752                                 rte_pktmbuf_free_seg(txe->mbuf);
2753                                 txe->mbuf = NULL;
2754                         }
2755
2756                         if (ol_flags & PKT_TX_TCP_SEG)
2757                                 cd_type_cmd_tso_mss |=
2758                                         ice_set_tso_ctx(tx_pkt, tx_offload);
2759                         else if (ol_flags & PKT_TX_IEEE1588_TMST)
2760                                 cd_type_cmd_tso_mss |=
2761                                         ((uint64_t)ICE_TX_CTX_DESC_TSYN <<
2762                                         ICE_TXD_CTX_QW1_CMD_S);
2763
2764                         ctx_txd->tunneling_params =
2765                                 rte_cpu_to_le_32(cd_tunneling_params);
2766
2767                         /* TX context descriptor based double VLAN insert */
2768                         if (ol_flags & PKT_TX_QINQ) {
2769                                 cd_l2tag2 = tx_pkt->vlan_tci_outer;
2770                                 cd_type_cmd_tso_mss |=
2771                                         ((uint64_t)ICE_TX_CTX_DESC_IL2TAG2 <<
2772                                          ICE_TXD_CTX_QW1_CMD_S);
2773                         }
2774                         ctx_txd->l2tag2 = rte_cpu_to_le_16(cd_l2tag2);
2775                         ctx_txd->qw1 =
2776                                 rte_cpu_to_le_64(cd_type_cmd_tso_mss);
2777
2778                         txe->last_id = tx_last;
2779                         tx_id = txe->next_id;
2780                         txe = txn;
2781                 }
2782                 m_seg = tx_pkt;
2783
2784                 do {
2785                         txd = &tx_ring[tx_id];
2786                         txn = &sw_ring[txe->next_id];
2787
2788                         if (txe->mbuf)
2789                                 rte_pktmbuf_free_seg(txe->mbuf);
2790                         txe->mbuf = m_seg;
2791
2792                         /* Setup TX Descriptor */
2793                         slen = m_seg->data_len;
2794                         buf_dma_addr = rte_mbuf_data_iova(m_seg);
2795
2796                         while ((ol_flags & PKT_TX_TCP_SEG) &&
2797                                 unlikely(slen > ICE_MAX_DATA_PER_TXD)) {
2798                                 txd->buf_addr = rte_cpu_to_le_64(buf_dma_addr);
2799                                 txd->cmd_type_offset_bsz =
2800                                 rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DATA |
2801                                 ((uint64_t)td_cmd << ICE_TXD_QW1_CMD_S) |
2802                                 ((uint64_t)td_offset << ICE_TXD_QW1_OFFSET_S) |
2803                                 ((uint64_t)ICE_MAX_DATA_PER_TXD <<
2804                                  ICE_TXD_QW1_TX_BUF_SZ_S) |
2805                                 ((uint64_t)td_tag << ICE_TXD_QW1_L2TAG1_S));
2806
2807                                 buf_dma_addr += ICE_MAX_DATA_PER_TXD;
2808                                 slen -= ICE_MAX_DATA_PER_TXD;
2809
2810                                 txe->last_id = tx_last;
2811                                 tx_id = txe->next_id;
2812                                 txe = txn;
2813                                 txd = &tx_ring[tx_id];
2814                                 txn = &sw_ring[txe->next_id];
2815                         }
2816
2817                         txd->buf_addr = rte_cpu_to_le_64(buf_dma_addr);
2818                         txd->cmd_type_offset_bsz =
2819                                 rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DATA |
2820                                 ((uint64_t)td_cmd << ICE_TXD_QW1_CMD_S) |
2821                                 ((uint64_t)td_offset << ICE_TXD_QW1_OFFSET_S) |
2822                                 ((uint64_t)slen << ICE_TXD_QW1_TX_BUF_SZ_S) |
2823                                 ((uint64_t)td_tag << ICE_TXD_QW1_L2TAG1_S));
2824
2825                         txe->last_id = tx_last;
2826                         tx_id = txe->next_id;
2827                         txe = txn;
2828                         m_seg = m_seg->next;
2829                 } while (m_seg);
2830
2831                 /* fill the last descriptor with End of Packet (EOP) bit */
2832                 td_cmd |= ICE_TX_DESC_CMD_EOP;
2833                 txq->nb_tx_used = (uint16_t)(txq->nb_tx_used + nb_used);
2834                 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_used);
2835
2836                 /* set RS bit on the last descriptor of one packet */
2837                 if (txq->nb_tx_used >= txq->tx_rs_thresh) {
2838                         PMD_TX_LOG(DEBUG,
2839                                    "Setting RS bit on TXD id="
2840                                    "%4u (port=%d queue=%d)",
2841                                    tx_last, txq->port_id, txq->queue_id);
2842
2843                         td_cmd |= ICE_TX_DESC_CMD_RS;
2844
2845                         /* Update txq RS bit counters */
2846                         txq->nb_tx_used = 0;
2847                 }
2848                 txd->cmd_type_offset_bsz |=
2849                         rte_cpu_to_le_64(((uint64_t)td_cmd) <<
2850                                          ICE_TXD_QW1_CMD_S);
2851         }
2852 end_of_tx:
2853         /* update Tail register */
2854         ICE_PCI_REG_WRITE(txq->qtx_tail, tx_id);
2855         txq->tx_tail = tx_id;
2856
2857         return nb_tx;
2858 }
2859
2860 static __rte_always_inline int
2861 ice_tx_free_bufs(struct ice_tx_queue *txq)
2862 {
2863         struct ice_tx_entry *txep;
2864         uint16_t i;
2865
2866         if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &
2867              rte_cpu_to_le_64(ICE_TXD_QW1_DTYPE_M)) !=
2868             rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DESC_DONE))
2869                 return 0;
2870
2871         txep = &txq->sw_ring[txq->tx_next_dd - (txq->tx_rs_thresh - 1)];
2872
2873         for (i = 0; i < txq->tx_rs_thresh; i++)
2874                 rte_prefetch0((txep + i)->mbuf);
2875
2876         if (txq->offloads & DEV_TX_OFFLOAD_MBUF_FAST_FREE) {
2877                 for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) {
2878                         rte_mempool_put(txep->mbuf->pool, txep->mbuf);
2879                         txep->mbuf = NULL;
2880                 }
2881         } else {
2882                 for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) {
2883                         rte_pktmbuf_free_seg(txep->mbuf);
2884                         txep->mbuf = NULL;
2885                 }
2886         }
2887
2888         txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh);
2889         txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh);
2890         if (txq->tx_next_dd >= txq->nb_tx_desc)
2891                 txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
2892
2893         return txq->tx_rs_thresh;
2894 }
2895
2896 static int
2897 ice_tx_done_cleanup_full(struct ice_tx_queue *txq,
2898                         uint32_t free_cnt)
2899 {
2900         struct ice_tx_entry *swr_ring = txq->sw_ring;
2901         uint16_t i, tx_last, tx_id;
2902         uint16_t nb_tx_free_last;
2903         uint16_t nb_tx_to_clean;
2904         uint32_t pkt_cnt;
2905
2906         /* Start free mbuf from the next of tx_tail */
2907         tx_last = txq->tx_tail;
2908         tx_id  = swr_ring[tx_last].next_id;
2909
2910         if (txq->nb_tx_free == 0 && ice_xmit_cleanup(txq))
2911                 return 0;
2912
2913         nb_tx_to_clean = txq->nb_tx_free;
2914         nb_tx_free_last = txq->nb_tx_free;
2915         if (!free_cnt)
2916                 free_cnt = txq->nb_tx_desc;
2917
2918         /* Loop through swr_ring to count the amount of
2919          * freeable mubfs and packets.
2920          */
2921         for (pkt_cnt = 0; pkt_cnt < free_cnt; ) {
2922                 for (i = 0; i < nb_tx_to_clean &&
2923                         pkt_cnt < free_cnt &&
2924                         tx_id != tx_last; i++) {
2925                         if (swr_ring[tx_id].mbuf != NULL) {
2926                                 rte_pktmbuf_free_seg(swr_ring[tx_id].mbuf);
2927                                 swr_ring[tx_id].mbuf = NULL;
2928
2929                                 /*
2930                                  * last segment in the packet,
2931                                  * increment packet count
2932                                  */
2933                                 pkt_cnt += (swr_ring[tx_id].last_id == tx_id);
2934                         }
2935
2936                         tx_id = swr_ring[tx_id].next_id;
2937                 }
2938
2939                 if (txq->tx_rs_thresh > txq->nb_tx_desc -
2940                         txq->nb_tx_free || tx_id == tx_last)
2941                         break;
2942
2943                 if (pkt_cnt < free_cnt) {
2944                         if (ice_xmit_cleanup(txq))
2945                                 break;
2946
2947                         nb_tx_to_clean = txq->nb_tx_free - nb_tx_free_last;
2948                         nb_tx_free_last = txq->nb_tx_free;
2949                 }
2950         }
2951
2952         return (int)pkt_cnt;
2953 }
2954
2955 #ifdef RTE_ARCH_X86
2956 static int
2957 ice_tx_done_cleanup_vec(struct ice_tx_queue *txq __rte_unused,
2958                         uint32_t free_cnt __rte_unused)
2959 {
2960         return -ENOTSUP;
2961 }
2962 #endif
2963
2964 static int
2965 ice_tx_done_cleanup_simple(struct ice_tx_queue *txq,
2966                         uint32_t free_cnt)
2967 {
2968         int i, n, cnt;
2969
2970         if (free_cnt == 0 || free_cnt > txq->nb_tx_desc)
2971                 free_cnt = txq->nb_tx_desc;
2972
2973         cnt = free_cnt - free_cnt % txq->tx_rs_thresh;
2974
2975         for (i = 0; i < cnt; i += n) {
2976                 if (txq->nb_tx_desc - txq->nb_tx_free < txq->tx_rs_thresh)
2977                         break;
2978
2979                 n = ice_tx_free_bufs(txq);
2980
2981                 if (n == 0)
2982                         break;
2983         }
2984
2985         return i;
2986 }
2987
2988 int
2989 ice_tx_done_cleanup(void *txq, uint32_t free_cnt)
2990 {
2991         struct ice_tx_queue *q = (struct ice_tx_queue *)txq;
2992         struct rte_eth_dev *dev = &rte_eth_devices[q->port_id];
2993         struct ice_adapter *ad =
2994                 ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
2995
2996 #ifdef RTE_ARCH_X86
2997         if (ad->tx_vec_allowed)
2998                 return ice_tx_done_cleanup_vec(q, free_cnt);
2999 #endif
3000         if (ad->tx_simple_allowed)
3001                 return ice_tx_done_cleanup_simple(q, free_cnt);
3002         else
3003                 return ice_tx_done_cleanup_full(q, free_cnt);
3004 }
3005
3006 /* Populate 4 descriptors with data from 4 mbufs */
3007 static inline void
3008 tx4(volatile struct ice_tx_desc *txdp, struct rte_mbuf **pkts)
3009 {
3010         uint64_t dma_addr;
3011         uint32_t i;
3012
3013         for (i = 0; i < 4; i++, txdp++, pkts++) {
3014                 dma_addr = rte_mbuf_data_iova(*pkts);
3015                 txdp->buf_addr = rte_cpu_to_le_64(dma_addr);
3016                 txdp->cmd_type_offset_bsz =
3017                         ice_build_ctob((uint32_t)ICE_TD_CMD, 0,
3018                                        (*pkts)->data_len, 0);
3019         }
3020 }
3021
3022 /* Populate 1 descriptor with data from 1 mbuf */
3023 static inline void
3024 tx1(volatile struct ice_tx_desc *txdp, struct rte_mbuf **pkts)
3025 {
3026         uint64_t dma_addr;
3027
3028         dma_addr = rte_mbuf_data_iova(*pkts);
3029         txdp->buf_addr = rte_cpu_to_le_64(dma_addr);
3030         txdp->cmd_type_offset_bsz =
3031                 ice_build_ctob((uint32_t)ICE_TD_CMD, 0,
3032                                (*pkts)->data_len, 0);
3033 }
3034
3035 static inline void
3036 ice_tx_fill_hw_ring(struct ice_tx_queue *txq, struct rte_mbuf **pkts,
3037                     uint16_t nb_pkts)
3038 {
3039         volatile struct ice_tx_desc *txdp = &txq->tx_ring[txq->tx_tail];
3040         struct ice_tx_entry *txep = &txq->sw_ring[txq->tx_tail];
3041         const int N_PER_LOOP = 4;
3042         const int N_PER_LOOP_MASK = N_PER_LOOP - 1;
3043         int mainpart, leftover;
3044         int i, j;
3045
3046         /**
3047          * Process most of the packets in chunks of N pkts.  Any
3048          * leftover packets will get processed one at a time.
3049          */
3050         mainpart = nb_pkts & ((uint32_t)~N_PER_LOOP_MASK);
3051         leftover = nb_pkts & ((uint32_t)N_PER_LOOP_MASK);
3052         for (i = 0; i < mainpart; i += N_PER_LOOP) {
3053                 /* Copy N mbuf pointers to the S/W ring */
3054                 for (j = 0; j < N_PER_LOOP; ++j)
3055                         (txep + i + j)->mbuf = *(pkts + i + j);
3056                 tx4(txdp + i, pkts + i);
3057         }
3058
3059         if (unlikely(leftover > 0)) {
3060                 for (i = 0; i < leftover; ++i) {
3061                         (txep + mainpart + i)->mbuf = *(pkts + mainpart + i);
3062                         tx1(txdp + mainpart + i, pkts + mainpart + i);
3063                 }
3064         }
3065 }
3066
3067 static inline uint16_t
3068 tx_xmit_pkts(struct ice_tx_queue *txq,
3069              struct rte_mbuf **tx_pkts,
3070              uint16_t nb_pkts)
3071 {
3072         volatile struct ice_tx_desc *txr = txq->tx_ring;
3073         uint16_t n = 0;
3074
3075         /**
3076          * Begin scanning the H/W ring for done descriptors when the number
3077          * of available descriptors drops below tx_free_thresh. For each done
3078          * descriptor, free the associated buffer.
3079          */
3080         if (txq->nb_tx_free < txq->tx_free_thresh)
3081                 ice_tx_free_bufs(txq);
3082
3083         /* Use available descriptor only */
3084         nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts);
3085         if (unlikely(!nb_pkts))
3086                 return 0;
3087
3088         txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts);
3089         if ((txq->tx_tail + nb_pkts) > txq->nb_tx_desc) {
3090                 n = (uint16_t)(txq->nb_tx_desc - txq->tx_tail);
3091                 ice_tx_fill_hw_ring(txq, tx_pkts, n);
3092                 txr[txq->tx_next_rs].cmd_type_offset_bsz |=
3093                         rte_cpu_to_le_64(((uint64_t)ICE_TX_DESC_CMD_RS) <<
3094                                          ICE_TXD_QW1_CMD_S);
3095                 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
3096                 txq->tx_tail = 0;
3097         }
3098
3099         /* Fill hardware descriptor ring with mbuf data */
3100         ice_tx_fill_hw_ring(txq, tx_pkts + n, (uint16_t)(nb_pkts - n));
3101         txq->tx_tail = (uint16_t)(txq->tx_tail + (nb_pkts - n));
3102
3103         /* Determin if RS bit needs to be set */
3104         if (txq->tx_tail > txq->tx_next_rs) {
3105                 txr[txq->tx_next_rs].cmd_type_offset_bsz |=
3106                         rte_cpu_to_le_64(((uint64_t)ICE_TX_DESC_CMD_RS) <<
3107                                          ICE_TXD_QW1_CMD_S);
3108                 txq->tx_next_rs =
3109                         (uint16_t)(txq->tx_next_rs + txq->tx_rs_thresh);
3110                 if (txq->tx_next_rs >= txq->nb_tx_desc)
3111                         txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
3112         }
3113
3114         if (txq->tx_tail >= txq->nb_tx_desc)
3115                 txq->tx_tail = 0;
3116
3117         /* Update the tx tail register */
3118         ICE_PCI_REG_WC_WRITE(txq->qtx_tail, txq->tx_tail);
3119
3120         return nb_pkts;
3121 }
3122
3123 static uint16_t
3124 ice_xmit_pkts_simple(void *tx_queue,
3125                      struct rte_mbuf **tx_pkts,
3126                      uint16_t nb_pkts)
3127 {
3128         uint16_t nb_tx = 0;
3129
3130         if (likely(nb_pkts <= ICE_TX_MAX_BURST))
3131                 return tx_xmit_pkts((struct ice_tx_queue *)tx_queue,
3132                                     tx_pkts, nb_pkts);
3133
3134         while (nb_pkts) {
3135                 uint16_t ret, num = (uint16_t)RTE_MIN(nb_pkts,
3136                                                       ICE_TX_MAX_BURST);
3137
3138                 ret = tx_xmit_pkts((struct ice_tx_queue *)tx_queue,
3139                                    &tx_pkts[nb_tx], num);
3140                 nb_tx = (uint16_t)(nb_tx + ret);
3141                 nb_pkts = (uint16_t)(nb_pkts - ret);
3142                 if (ret < num)
3143                         break;
3144         }
3145
3146         return nb_tx;
3147 }
3148
3149 void __rte_cold
3150 ice_set_rx_function(struct rte_eth_dev *dev)
3151 {
3152         PMD_INIT_FUNC_TRACE();
3153         struct ice_adapter *ad =
3154                 ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
3155 #ifdef RTE_ARCH_X86
3156         struct ice_rx_queue *rxq;
3157         int i;
3158         int rx_check_ret = -1;
3159
3160         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
3161                 ad->rx_use_avx512 = false;
3162                 ad->rx_use_avx2 = false;
3163                 rx_check_ret = ice_rx_vec_dev_check(dev);
3164                 if (ad->ptp_ena)
3165                         rx_check_ret = -1;
3166                 if (rx_check_ret >= 0 && ad->rx_bulk_alloc_allowed &&
3167                     rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) {
3168                         ad->rx_vec_allowed = true;
3169                         for (i = 0; i < dev->data->nb_rx_queues; i++) {
3170                                 rxq = dev->data->rx_queues[i];
3171                                 if (rxq && ice_rxq_vec_setup(rxq)) {
3172                                         ad->rx_vec_allowed = false;
3173                                         break;
3174                                 }
3175                         }
3176
3177                         if (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_512 &&
3178                         rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1 &&
3179                         rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) == 1)
3180 #ifdef CC_AVX512_SUPPORT
3181                                 ad->rx_use_avx512 = true;
3182 #else
3183                         PMD_DRV_LOG(NOTICE,
3184                                 "AVX512 is not supported in build env");
3185 #endif
3186                         if (!ad->rx_use_avx512 &&
3187                         (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) == 1 ||
3188                         rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1) &&
3189                         rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_256)
3190                                 ad->rx_use_avx2 = true;
3191
3192                 } else {
3193                         ad->rx_vec_allowed = false;
3194                 }
3195         }
3196
3197         if (ad->rx_vec_allowed) {
3198                 if (dev->data->scattered_rx) {
3199                         if (ad->rx_use_avx512) {
3200 #ifdef CC_AVX512_SUPPORT
3201                                 if (rx_check_ret == ICE_VECTOR_OFFLOAD_PATH) {
3202                                         PMD_DRV_LOG(NOTICE,
3203                                                 "Using AVX512 OFFLOAD Vector Scattered Rx (port %d).",
3204                                                 dev->data->port_id);
3205                                         dev->rx_pkt_burst =
3206                                                 ice_recv_scattered_pkts_vec_avx512_offload;
3207                                 } else {
3208                                         PMD_DRV_LOG(NOTICE,
3209                                                 "Using AVX512 Vector Scattered Rx (port %d).",
3210                                                 dev->data->port_id);
3211                                         dev->rx_pkt_burst =
3212                                                 ice_recv_scattered_pkts_vec_avx512;
3213                                 }
3214 #endif
3215                         } else if (ad->rx_use_avx2) {
3216                                 if (rx_check_ret == ICE_VECTOR_OFFLOAD_PATH) {
3217                                         PMD_DRV_LOG(NOTICE,
3218                                                     "Using AVX2 OFFLOAD Vector Scattered Rx (port %d).",
3219                                                     dev->data->port_id);
3220                                         dev->rx_pkt_burst =
3221                                                 ice_recv_scattered_pkts_vec_avx2_offload;
3222                                 } else {
3223                                         PMD_DRV_LOG(NOTICE,
3224                                                     "Using AVX2 Vector Scattered Rx (port %d).",
3225                                                     dev->data->port_id);
3226                                         dev->rx_pkt_burst =
3227                                                 ice_recv_scattered_pkts_vec_avx2;
3228                                 }
3229                         } else {
3230                                 PMD_DRV_LOG(DEBUG,
3231                                         "Using Vector Scattered Rx (port %d).",
3232                                         dev->data->port_id);
3233                                 dev->rx_pkt_burst = ice_recv_scattered_pkts_vec;
3234                         }
3235                 } else {
3236                         if (ad->rx_use_avx512) {
3237 #ifdef CC_AVX512_SUPPORT
3238                                 if (rx_check_ret == ICE_VECTOR_OFFLOAD_PATH) {
3239                                         PMD_DRV_LOG(NOTICE,
3240                                                 "Using AVX512 OFFLOAD Vector Rx (port %d).",
3241                                                 dev->data->port_id);
3242                                         dev->rx_pkt_burst =
3243                                                 ice_recv_pkts_vec_avx512_offload;
3244                                 } else {
3245                                         PMD_DRV_LOG(NOTICE,
3246                                                 "Using AVX512 Vector Rx (port %d).",
3247                                                 dev->data->port_id);
3248                                         dev->rx_pkt_burst =
3249                                                 ice_recv_pkts_vec_avx512;
3250                                 }
3251 #endif
3252                         } else if (ad->rx_use_avx2) {
3253                                 if (rx_check_ret == ICE_VECTOR_OFFLOAD_PATH) {
3254                                         PMD_DRV_LOG(NOTICE,
3255                                                     "Using AVX2 OFFLOAD Vector Rx (port %d).",
3256                                                     dev->data->port_id);
3257                                         dev->rx_pkt_burst =
3258                                                 ice_recv_pkts_vec_avx2_offload;
3259                                 } else {
3260                                         PMD_DRV_LOG(NOTICE,
3261                                                     "Using AVX2 Vector Rx (port %d).",
3262                                                     dev->data->port_id);
3263                                         dev->rx_pkt_burst =
3264                                                 ice_recv_pkts_vec_avx2;
3265                                 }
3266                         } else {
3267                                 PMD_DRV_LOG(DEBUG,
3268                                         "Using Vector Rx (port %d).",
3269                                         dev->data->port_id);
3270                                 dev->rx_pkt_burst = ice_recv_pkts_vec;
3271                         }
3272                 }
3273                 return;
3274         }
3275
3276 #endif
3277
3278         if (dev->data->scattered_rx) {
3279                 /* Set the non-LRO scattered function */
3280                 PMD_INIT_LOG(DEBUG,
3281                              "Using a Scattered function on port %d.",
3282                              dev->data->port_id);
3283                 dev->rx_pkt_burst = ice_recv_scattered_pkts;
3284         } else if (ad->rx_bulk_alloc_allowed) {
3285                 PMD_INIT_LOG(DEBUG,
3286                              "Rx Burst Bulk Alloc Preconditions are "
3287                              "satisfied. Rx Burst Bulk Alloc function "
3288                              "will be used on port %d.",
3289                              dev->data->port_id);
3290                 dev->rx_pkt_burst = ice_recv_pkts_bulk_alloc;
3291         } else {
3292                 PMD_INIT_LOG(DEBUG,
3293                              "Rx Burst Bulk Alloc Preconditions are not "
3294                              "satisfied, Normal Rx will be used on port %d.",
3295                              dev->data->port_id);
3296                 dev->rx_pkt_burst = ice_recv_pkts;
3297         }
3298 }
3299
3300 static const struct {
3301         eth_rx_burst_t pkt_burst;
3302         const char *info;
3303 } ice_rx_burst_infos[] = {
3304         { ice_recv_scattered_pkts,          "Scalar Scattered" },
3305         { ice_recv_pkts_bulk_alloc,         "Scalar Bulk Alloc" },
3306         { ice_recv_pkts,                    "Scalar" },
3307 #ifdef RTE_ARCH_X86
3308 #ifdef CC_AVX512_SUPPORT
3309         { ice_recv_scattered_pkts_vec_avx512, "Vector AVX512 Scattered" },
3310         { ice_recv_scattered_pkts_vec_avx512_offload, "Offload Vector AVX512 Scattered" },
3311         { ice_recv_pkts_vec_avx512,           "Vector AVX512" },
3312         { ice_recv_pkts_vec_avx512_offload,   "Offload Vector AVX512" },
3313 #endif
3314         { ice_recv_scattered_pkts_vec_avx2, "Vector AVX2 Scattered" },
3315         { ice_recv_scattered_pkts_vec_avx2_offload, "Offload Vector AVX2 Scattered" },
3316         { ice_recv_pkts_vec_avx2,           "Vector AVX2" },
3317         { ice_recv_pkts_vec_avx2_offload,   "Offload Vector AVX2" },
3318         { ice_recv_scattered_pkts_vec,      "Vector SSE Scattered" },
3319         { ice_recv_pkts_vec,                "Vector SSE" },
3320 #endif
3321 };
3322
3323 int
3324 ice_rx_burst_mode_get(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id,
3325                       struct rte_eth_burst_mode *mode)
3326 {
3327         eth_rx_burst_t pkt_burst = dev->rx_pkt_burst;
3328         int ret = -EINVAL;
3329         unsigned int i;
3330
3331         for (i = 0; i < RTE_DIM(ice_rx_burst_infos); ++i) {
3332                 if (pkt_burst == ice_rx_burst_infos[i].pkt_burst) {
3333                         snprintf(mode->info, sizeof(mode->info), "%s",
3334                                  ice_rx_burst_infos[i].info);
3335                         ret = 0;
3336                         break;
3337                 }
3338         }
3339
3340         return ret;
3341 }
3342
3343 void __rte_cold
3344 ice_set_tx_function_flag(struct rte_eth_dev *dev, struct ice_tx_queue *txq)
3345 {
3346         struct ice_adapter *ad =
3347                 ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
3348
3349         /* Use a simple Tx queue if possible (only fast free is allowed) */
3350         ad->tx_simple_allowed =
3351                 (txq->offloads ==
3352                 (txq->offloads & DEV_TX_OFFLOAD_MBUF_FAST_FREE) &&
3353                 txq->tx_rs_thresh >= ICE_TX_MAX_BURST);
3354
3355         if (ad->tx_simple_allowed)
3356                 PMD_INIT_LOG(DEBUG, "Simple Tx can be enabled on Tx queue %u.",
3357                              txq->queue_id);
3358         else
3359                 PMD_INIT_LOG(DEBUG,
3360                              "Simple Tx can NOT be enabled on Tx queue %u.",
3361                              txq->queue_id);
3362 }
3363
3364 /*********************************************************************
3365  *
3366  *  TX prep functions
3367  *
3368  **********************************************************************/
3369 /* The default values of TSO MSS */
3370 #define ICE_MIN_TSO_MSS            64
3371 #define ICE_MAX_TSO_MSS            9728
3372 #define ICE_MAX_TSO_FRAME_SIZE     262144
3373 uint16_t
3374 ice_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts,
3375               uint16_t nb_pkts)
3376 {
3377         int i, ret;
3378         uint64_t ol_flags;
3379         struct rte_mbuf *m;
3380
3381         for (i = 0; i < nb_pkts; i++) {
3382                 m = tx_pkts[i];
3383                 ol_flags = m->ol_flags;
3384
3385                 if (ol_flags & PKT_TX_TCP_SEG &&
3386                     (m->tso_segsz < ICE_MIN_TSO_MSS ||
3387                      m->tso_segsz > ICE_MAX_TSO_MSS ||
3388                      m->pkt_len > ICE_MAX_TSO_FRAME_SIZE)) {
3389                         /**
3390                          * MSS outside the range are considered malicious
3391                          */
3392                         rte_errno = EINVAL;
3393                         return i;
3394                 }
3395
3396 #ifdef RTE_ETHDEV_DEBUG_TX
3397                 ret = rte_validate_tx_offload(m);
3398                 if (ret != 0) {
3399                         rte_errno = -ret;
3400                         return i;
3401                 }
3402 #endif
3403                 ret = rte_net_intel_cksum_prepare(m);
3404                 if (ret != 0) {
3405                         rte_errno = -ret;
3406                         return i;
3407                 }
3408         }
3409         return i;
3410 }
3411
3412 void __rte_cold
3413 ice_set_tx_function(struct rte_eth_dev *dev)
3414 {
3415         struct ice_adapter *ad =
3416                 ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
3417 #ifdef RTE_ARCH_X86
3418         struct ice_tx_queue *txq;
3419         int i;
3420         int tx_check_ret = -1;
3421
3422         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
3423                 ad->tx_use_avx2 = false;
3424                 ad->tx_use_avx512 = false;
3425                 tx_check_ret = ice_tx_vec_dev_check(dev);
3426                 if (tx_check_ret >= 0 &&
3427                     rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) {
3428                         ad->tx_vec_allowed = true;
3429
3430                         if (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_512 &&
3431                         rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1 &&
3432                         rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) == 1)
3433 #ifdef CC_AVX512_SUPPORT
3434                                 ad->tx_use_avx512 = true;
3435 #else
3436                         PMD_DRV_LOG(NOTICE,
3437                                 "AVX512 is not supported in build env");
3438 #endif
3439                         if (!ad->tx_use_avx512 &&
3440                                 (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) == 1 ||
3441                                 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1) &&
3442                                 rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_256)
3443                                 ad->tx_use_avx2 = true;
3444
3445                         if (!ad->tx_use_avx2 && !ad->tx_use_avx512 &&
3446                                 tx_check_ret == ICE_VECTOR_OFFLOAD_PATH)
3447                                 ad->tx_vec_allowed = false;
3448
3449                         if (ad->tx_vec_allowed) {
3450                                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
3451                                         txq = dev->data->tx_queues[i];
3452                                         if (txq && ice_txq_vec_setup(txq)) {
3453                                                 ad->tx_vec_allowed = false;
3454                                                 break;
3455                                         }
3456                                 }
3457                         }
3458                 } else {
3459                         ad->tx_vec_allowed = false;
3460                 }
3461         }
3462
3463         if (ad->tx_vec_allowed) {
3464                 dev->tx_pkt_prepare = NULL;
3465                 if (ad->tx_use_avx512) {
3466 #ifdef CC_AVX512_SUPPORT
3467                         if (tx_check_ret == ICE_VECTOR_OFFLOAD_PATH) {
3468                                 PMD_DRV_LOG(NOTICE,
3469                                             "Using AVX512 OFFLOAD Vector Tx (port %d).",
3470                                             dev->data->port_id);
3471                                 dev->tx_pkt_burst =
3472                                         ice_xmit_pkts_vec_avx512_offload;
3473                                 dev->tx_pkt_prepare = ice_prep_pkts;
3474                         } else {
3475                                 PMD_DRV_LOG(NOTICE,
3476                                             "Using AVX512 Vector Tx (port %d).",
3477                                             dev->data->port_id);
3478                                 dev->tx_pkt_burst = ice_xmit_pkts_vec_avx512;
3479                         }
3480 #endif
3481                 } else {
3482                         if (tx_check_ret == ICE_VECTOR_OFFLOAD_PATH) {
3483                                 PMD_DRV_LOG(NOTICE,
3484                                             "Using AVX2 OFFLOAD Vector Tx (port %d).",
3485                                             dev->data->port_id);
3486                                 dev->tx_pkt_burst =
3487                                         ice_xmit_pkts_vec_avx2_offload;
3488                                 dev->tx_pkt_prepare = ice_prep_pkts;
3489                         } else {
3490                                 PMD_DRV_LOG(DEBUG, "Using %sVector Tx (port %d).",
3491                                             ad->tx_use_avx2 ? "avx2 " : "",
3492                                             dev->data->port_id);
3493                                 dev->tx_pkt_burst = ad->tx_use_avx2 ?
3494                                                     ice_xmit_pkts_vec_avx2 :
3495                                                     ice_xmit_pkts_vec;
3496                         }
3497                 }
3498
3499                 return;
3500         }
3501 #endif
3502
3503         if (ad->tx_simple_allowed) {
3504                 PMD_INIT_LOG(DEBUG, "Simple tx finally be used.");
3505                 dev->tx_pkt_burst = ice_xmit_pkts_simple;
3506                 dev->tx_pkt_prepare = NULL;
3507         } else {
3508                 PMD_INIT_LOG(DEBUG, "Normal tx finally be used.");
3509                 dev->tx_pkt_burst = ice_xmit_pkts;
3510                 dev->tx_pkt_prepare = ice_prep_pkts;
3511         }
3512 }
3513
3514 static const struct {
3515         eth_tx_burst_t pkt_burst;
3516         const char *info;
3517 } ice_tx_burst_infos[] = {
3518         { ice_xmit_pkts_simple,   "Scalar Simple" },
3519         { ice_xmit_pkts,          "Scalar" },
3520 #ifdef RTE_ARCH_X86
3521 #ifdef CC_AVX512_SUPPORT
3522         { ice_xmit_pkts_vec_avx512, "Vector AVX512" },
3523         { ice_xmit_pkts_vec_avx512_offload, "Offload Vector AVX512" },
3524 #endif
3525         { ice_xmit_pkts_vec_avx2, "Vector AVX2" },
3526         { ice_xmit_pkts_vec,      "Vector SSE" },
3527 #endif
3528 };
3529
3530 int
3531 ice_tx_burst_mode_get(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id,
3532                       struct rte_eth_burst_mode *mode)
3533 {
3534         eth_tx_burst_t pkt_burst = dev->tx_pkt_burst;
3535         int ret = -EINVAL;
3536         unsigned int i;
3537
3538         for (i = 0; i < RTE_DIM(ice_tx_burst_infos); ++i) {
3539                 if (pkt_burst == ice_tx_burst_infos[i].pkt_burst) {
3540                         snprintf(mode->info, sizeof(mode->info), "%s",
3541                                  ice_tx_burst_infos[i].info);
3542                         ret = 0;
3543                         break;
3544                 }
3545         }
3546
3547         return ret;
3548 }
3549
3550 /* For each value it means, datasheet of hardware can tell more details
3551  *
3552  * @note: fix ice_dev_supported_ptypes_get() if any change here.
3553  */
3554 static inline uint32_t
3555 ice_get_default_pkt_type(uint16_t ptype)
3556 {
3557         static const uint32_t type_table[ICE_MAX_PKT_TYPE]
3558                 __rte_cache_aligned = {
3559                 /* L2 types */
3560                 /* [0] reserved */
3561                 [1] = RTE_PTYPE_L2_ETHER,
3562                 [2] = RTE_PTYPE_L2_ETHER_TIMESYNC,
3563                 /* [3] - [5] reserved */
3564                 [6] = RTE_PTYPE_L2_ETHER_LLDP,
3565                 /* [7] - [10] reserved */
3566                 [11] = RTE_PTYPE_L2_ETHER_ARP,
3567                 /* [12] - [21] reserved */
3568
3569                 /* Non tunneled IPv4 */
3570                 [22] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3571                        RTE_PTYPE_L4_FRAG,
3572                 [23] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3573                        RTE_PTYPE_L4_NONFRAG,
3574                 [24] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3575                        RTE_PTYPE_L4_UDP,
3576                 /* [25] reserved */
3577                 [26] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3578                        RTE_PTYPE_L4_TCP,
3579                 [27] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3580                        RTE_PTYPE_L4_SCTP,
3581                 [28] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3582                        RTE_PTYPE_L4_ICMP,
3583
3584                 /* IPv4 --> IPv4 */
3585                 [29] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3586                        RTE_PTYPE_TUNNEL_IP |
3587                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3588                        RTE_PTYPE_INNER_L4_FRAG,
3589                 [30] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3590                        RTE_PTYPE_TUNNEL_IP |
3591                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3592                        RTE_PTYPE_INNER_L4_NONFRAG,
3593                 [31] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3594                        RTE_PTYPE_TUNNEL_IP |
3595                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3596                        RTE_PTYPE_INNER_L4_UDP,
3597                 /* [32] reserved */
3598                 [33] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3599                        RTE_PTYPE_TUNNEL_IP |
3600                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3601                        RTE_PTYPE_INNER_L4_TCP,
3602                 [34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3603                        RTE_PTYPE_TUNNEL_IP |
3604                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3605                        RTE_PTYPE_INNER_L4_SCTP,
3606                 [35] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3607                        RTE_PTYPE_TUNNEL_IP |
3608                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3609                        RTE_PTYPE_INNER_L4_ICMP,
3610
3611                 /* IPv4 --> IPv6 */
3612                 [36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3613                        RTE_PTYPE_TUNNEL_IP |
3614                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3615                        RTE_PTYPE_INNER_L4_FRAG,
3616                 [37] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3617                        RTE_PTYPE_TUNNEL_IP |
3618                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3619                        RTE_PTYPE_INNER_L4_NONFRAG,
3620                 [38] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3621                        RTE_PTYPE_TUNNEL_IP |
3622                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3623                        RTE_PTYPE_INNER_L4_UDP,
3624                 /* [39] reserved */
3625                 [40] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3626                        RTE_PTYPE_TUNNEL_IP |
3627                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3628                        RTE_PTYPE_INNER_L4_TCP,
3629                 [41] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3630                        RTE_PTYPE_TUNNEL_IP |
3631                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3632                        RTE_PTYPE_INNER_L4_SCTP,
3633                 [42] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3634                        RTE_PTYPE_TUNNEL_IP |
3635                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3636                        RTE_PTYPE_INNER_L4_ICMP,
3637
3638                 /* IPv4 --> GRE/Teredo/VXLAN */
3639                 [43] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3640                        RTE_PTYPE_TUNNEL_GRENAT,
3641
3642                 /* IPv4 --> GRE/Teredo/VXLAN --> IPv4 */
3643                 [44] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3644                        RTE_PTYPE_TUNNEL_GRENAT |
3645                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3646                        RTE_PTYPE_INNER_L4_FRAG,
3647                 [45] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3648                        RTE_PTYPE_TUNNEL_GRENAT |
3649                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3650                        RTE_PTYPE_INNER_L4_NONFRAG,
3651                 [46] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3652                        RTE_PTYPE_TUNNEL_GRENAT |
3653                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3654                        RTE_PTYPE_INNER_L4_UDP,
3655                 /* [47] reserved */
3656                 [48] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3657                        RTE_PTYPE_TUNNEL_GRENAT |
3658                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3659                        RTE_PTYPE_INNER_L4_TCP,
3660                 [49] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3661                        RTE_PTYPE_TUNNEL_GRENAT |
3662                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3663                        RTE_PTYPE_INNER_L4_SCTP,
3664                 [50] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3665                        RTE_PTYPE_TUNNEL_GRENAT |
3666                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3667                        RTE_PTYPE_INNER_L4_ICMP,
3668
3669                 /* IPv4 --> GRE/Teredo/VXLAN --> IPv6 */
3670                 [51] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3671                        RTE_PTYPE_TUNNEL_GRENAT |
3672                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3673                        RTE_PTYPE_INNER_L4_FRAG,
3674                 [52] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3675                        RTE_PTYPE_TUNNEL_GRENAT |
3676                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3677                        RTE_PTYPE_INNER_L4_NONFRAG,
3678                 [53] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3679                        RTE_PTYPE_TUNNEL_GRENAT |
3680                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3681                        RTE_PTYPE_INNER_L4_UDP,
3682                 /* [54] reserved */
3683                 [55] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3684                        RTE_PTYPE_TUNNEL_GRENAT |
3685                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3686                        RTE_PTYPE_INNER_L4_TCP,
3687                 [56] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3688                        RTE_PTYPE_TUNNEL_GRENAT |
3689                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3690                        RTE_PTYPE_INNER_L4_SCTP,
3691                 [57] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3692                        RTE_PTYPE_TUNNEL_GRENAT |
3693                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3694                        RTE_PTYPE_INNER_L4_ICMP,
3695
3696                 /* IPv4 --> GRE/Teredo/VXLAN --> MAC */
3697                 [58] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3698                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER,
3699
3700                 /* IPv4 --> GRE/Teredo/VXLAN --> MAC --> IPv4 */
3701                 [59] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3702                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3703                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3704                        RTE_PTYPE_INNER_L4_FRAG,
3705                 [60] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3706                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3707                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3708                        RTE_PTYPE_INNER_L4_NONFRAG,
3709                 [61] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3710                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3711                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3712                        RTE_PTYPE_INNER_L4_UDP,
3713                 /* [62] reserved */
3714                 [63] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3715                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3716                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3717                        RTE_PTYPE_INNER_L4_TCP,
3718                 [64] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3719                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3720                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3721                        RTE_PTYPE_INNER_L4_SCTP,
3722                 [65] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3723                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3724                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3725                        RTE_PTYPE_INNER_L4_ICMP,
3726
3727                 /* IPv4 --> GRE/Teredo/VXLAN --> MAC --> IPv6 */
3728                 [66] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3729                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3730                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3731                        RTE_PTYPE_INNER_L4_FRAG,
3732                 [67] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3733                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3734                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3735                        RTE_PTYPE_INNER_L4_NONFRAG,
3736                 [68] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3737                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3738                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3739                        RTE_PTYPE_INNER_L4_UDP,
3740                 /* [69] reserved */
3741                 [70] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3742                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3743                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3744                        RTE_PTYPE_INNER_L4_TCP,
3745                 [71] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3746                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3747                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3748                        RTE_PTYPE_INNER_L4_SCTP,
3749                 [72] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3750                        RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3751                        RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3752                        RTE_PTYPE_INNER_L4_ICMP,
3753                 /* [73] - [87] reserved */
3754
3755                 /* Non tunneled IPv6 */
3756                 [88] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3757                        RTE_PTYPE_L4_FRAG,
3758                 [89] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3759                        RTE_PTYPE_L4_NONFRAG,
3760                 [90] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3761                        RTE_PTYPE_L4_UDP,
3762                 /* [91] reserved */
3763                 [92] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3764                        RTE_PTYPE_L4_TCP,
3765                 [93] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3766                        RTE_PTYPE_L4_SCTP,
3767                 [94] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3768                        RTE_PTYPE_L4_ICMP,
3769
3770                 /* IPv6 --> IPv4 */
3771                 [95] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3772                        RTE_PTYPE_TUNNEL_IP |
3773                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3774                        RTE_PTYPE_INNER_L4_FRAG,
3775                 [96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3776                        RTE_PTYPE_TUNNEL_IP |
3777                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3778                        RTE_PTYPE_INNER_L4_NONFRAG,
3779                 [97] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3780                        RTE_PTYPE_TUNNEL_IP |
3781                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3782                        RTE_PTYPE_INNER_L4_UDP,
3783                 /* [98] reserved */
3784                 [99] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3785                        RTE_PTYPE_TUNNEL_IP |
3786                        RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3787                        RTE_PTYPE_INNER_L4_TCP,
3788                 [100] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3789                         RTE_PTYPE_TUNNEL_IP |
3790                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3791                         RTE_PTYPE_INNER_L4_SCTP,
3792                 [101] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3793                         RTE_PTYPE_TUNNEL_IP |
3794                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3795                         RTE_PTYPE_INNER_L4_ICMP,
3796
3797                 /* IPv6 --> IPv6 */
3798                 [102] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3799                         RTE_PTYPE_TUNNEL_IP |
3800                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3801                         RTE_PTYPE_INNER_L4_FRAG,
3802                 [103] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3803                         RTE_PTYPE_TUNNEL_IP |
3804                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3805                         RTE_PTYPE_INNER_L4_NONFRAG,
3806                 [104] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3807                         RTE_PTYPE_TUNNEL_IP |
3808                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3809                         RTE_PTYPE_INNER_L4_UDP,
3810                 /* [105] reserved */
3811                 [106] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3812                         RTE_PTYPE_TUNNEL_IP |
3813                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3814                         RTE_PTYPE_INNER_L4_TCP,
3815                 [107] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3816                         RTE_PTYPE_TUNNEL_IP |
3817                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3818                         RTE_PTYPE_INNER_L4_SCTP,
3819                 [108] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3820                         RTE_PTYPE_TUNNEL_IP |
3821                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3822                         RTE_PTYPE_INNER_L4_ICMP,
3823
3824                 /* IPv6 --> GRE/Teredo/VXLAN */
3825                 [109] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3826                         RTE_PTYPE_TUNNEL_GRENAT,
3827
3828                 /* IPv6 --> GRE/Teredo/VXLAN --> IPv4 */
3829                 [110] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3830                         RTE_PTYPE_TUNNEL_GRENAT |
3831                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3832                         RTE_PTYPE_INNER_L4_FRAG,
3833                 [111] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3834                         RTE_PTYPE_TUNNEL_GRENAT |
3835                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3836                         RTE_PTYPE_INNER_L4_NONFRAG,
3837                 [112] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3838                         RTE_PTYPE_TUNNEL_GRENAT |
3839                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3840                         RTE_PTYPE_INNER_L4_UDP,
3841                 /* [113] reserved */
3842                 [114] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3843                         RTE_PTYPE_TUNNEL_GRENAT |
3844                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3845                         RTE_PTYPE_INNER_L4_TCP,
3846                 [115] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3847                         RTE_PTYPE_TUNNEL_GRENAT |
3848                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3849                         RTE_PTYPE_INNER_L4_SCTP,
3850                 [116] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3851                         RTE_PTYPE_TUNNEL_GRENAT |
3852                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3853                         RTE_PTYPE_INNER_L4_ICMP,
3854
3855                 /* IPv6 --> GRE/Teredo/VXLAN --> IPv6 */
3856                 [117] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3857                         RTE_PTYPE_TUNNEL_GRENAT |
3858                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3859                         RTE_PTYPE_INNER_L4_FRAG,
3860                 [118] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3861                         RTE_PTYPE_TUNNEL_GRENAT |
3862                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3863                         RTE_PTYPE_INNER_L4_NONFRAG,
3864                 [119] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3865                         RTE_PTYPE_TUNNEL_GRENAT |
3866                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3867                         RTE_PTYPE_INNER_L4_UDP,
3868                 /* [120] reserved */
3869                 [121] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3870                         RTE_PTYPE_TUNNEL_GRENAT |
3871                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3872                         RTE_PTYPE_INNER_L4_TCP,
3873                 [122] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3874                         RTE_PTYPE_TUNNEL_GRENAT |
3875                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3876                         RTE_PTYPE_INNER_L4_SCTP,
3877                 [123] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3878                         RTE_PTYPE_TUNNEL_GRENAT |
3879                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3880                         RTE_PTYPE_INNER_L4_ICMP,
3881
3882                 /* IPv6 --> GRE/Teredo/VXLAN --> MAC */
3883                 [124] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3884                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER,
3885
3886                 /* IPv6 --> GRE/Teredo/VXLAN --> MAC --> IPv4 */
3887                 [125] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3888                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3889                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3890                         RTE_PTYPE_INNER_L4_FRAG,
3891                 [126] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3892                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3893                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3894                         RTE_PTYPE_INNER_L4_NONFRAG,
3895                 [127] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3896                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3897                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3898                         RTE_PTYPE_INNER_L4_UDP,
3899                 /* [128] reserved */
3900                 [129] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3901                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3902                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3903                         RTE_PTYPE_INNER_L4_TCP,
3904                 [130] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3905                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3906                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3907                         RTE_PTYPE_INNER_L4_SCTP,
3908                 [131] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3909                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3910                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
3911                         RTE_PTYPE_INNER_L4_ICMP,
3912
3913                 /* IPv6 --> GRE/Teredo/VXLAN --> MAC --> IPv6 */
3914                 [132] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3915                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3916                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3917                         RTE_PTYPE_INNER_L4_FRAG,
3918                 [133] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3919                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3920                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3921                         RTE_PTYPE_INNER_L4_NONFRAG,
3922                 [134] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3923                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3924                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3925                         RTE_PTYPE_INNER_L4_UDP,
3926                 /* [135] reserved */
3927                 [136] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3928                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3929                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3930                         RTE_PTYPE_INNER_L4_TCP,
3931                 [137] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3932                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3933                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3934                         RTE_PTYPE_INNER_L4_SCTP,
3935                 [138] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3936                         RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
3937                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
3938                         RTE_PTYPE_INNER_L4_ICMP,
3939                 /* [139] - [299] reserved */
3940
3941                 /* PPPoE */
3942                 [300] = RTE_PTYPE_L2_ETHER_PPPOE,
3943                 [301] = RTE_PTYPE_L2_ETHER_PPPOE,
3944
3945                 /* PPPoE --> IPv4 */
3946                 [302] = RTE_PTYPE_L2_ETHER_PPPOE |
3947                         RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3948                         RTE_PTYPE_L4_FRAG,
3949                 [303] = RTE_PTYPE_L2_ETHER_PPPOE |
3950                         RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3951                         RTE_PTYPE_L4_NONFRAG,
3952                 [304] = RTE_PTYPE_L2_ETHER_PPPOE |
3953                         RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3954                         RTE_PTYPE_L4_UDP,
3955                 [305] = RTE_PTYPE_L2_ETHER_PPPOE |
3956                         RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3957                         RTE_PTYPE_L4_TCP,
3958                 [306] = RTE_PTYPE_L2_ETHER_PPPOE |
3959                         RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3960                         RTE_PTYPE_L4_SCTP,
3961                 [307] = RTE_PTYPE_L2_ETHER_PPPOE |
3962                         RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3963                         RTE_PTYPE_L4_ICMP,
3964
3965                 /* PPPoE --> IPv6 */
3966                 [308] = RTE_PTYPE_L2_ETHER_PPPOE |
3967                         RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3968                         RTE_PTYPE_L4_FRAG,
3969                 [309] = RTE_PTYPE_L2_ETHER_PPPOE |
3970                         RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3971                         RTE_PTYPE_L4_NONFRAG,
3972                 [310] = RTE_PTYPE_L2_ETHER_PPPOE |
3973                         RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3974                         RTE_PTYPE_L4_UDP,
3975                 [311] = RTE_PTYPE_L2_ETHER_PPPOE |
3976                         RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3977                         RTE_PTYPE_L4_TCP,
3978                 [312] = RTE_PTYPE_L2_ETHER_PPPOE |
3979                         RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3980                         RTE_PTYPE_L4_SCTP,
3981                 [313] = RTE_PTYPE_L2_ETHER_PPPOE |
3982                         RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3983                         RTE_PTYPE_L4_ICMP,
3984                 /* [314] - [324] reserved */
3985
3986                 /* IPv4/IPv6 --> GTPC/GTPU */
3987                 [325] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3988                         RTE_PTYPE_TUNNEL_GTPC,
3989                 [326] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3990                         RTE_PTYPE_TUNNEL_GTPC,
3991                 [327] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3992                         RTE_PTYPE_TUNNEL_GTPC,
3993                 [328] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3994                         RTE_PTYPE_TUNNEL_GTPC,
3995                 [329] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
3996                         RTE_PTYPE_TUNNEL_GTPU,
3997                 [330] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
3998                         RTE_PTYPE_TUNNEL_GTPU,
3999
4000                 /* IPv4 --> GTPU --> IPv4 */
4001                 [331] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4002                         RTE_PTYPE_TUNNEL_GTPU |
4003                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
4004                         RTE_PTYPE_INNER_L4_FRAG,
4005                 [332] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4006                         RTE_PTYPE_TUNNEL_GTPU |
4007                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
4008                         RTE_PTYPE_INNER_L4_NONFRAG,
4009                 [333] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4010                         RTE_PTYPE_TUNNEL_GTPU |
4011                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
4012                         RTE_PTYPE_INNER_L4_UDP,
4013                 [334] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4014                         RTE_PTYPE_TUNNEL_GTPU |
4015                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
4016                         RTE_PTYPE_INNER_L4_TCP,
4017                 [335] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4018                         RTE_PTYPE_TUNNEL_GTPU |
4019                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
4020                         RTE_PTYPE_INNER_L4_ICMP,
4021
4022                 /* IPv6 --> GTPU --> IPv4 */
4023                 [336] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4024                         RTE_PTYPE_TUNNEL_GTPU |
4025                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
4026                         RTE_PTYPE_INNER_L4_FRAG,
4027                 [337] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4028                         RTE_PTYPE_TUNNEL_GTPU |
4029                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
4030                         RTE_PTYPE_INNER_L4_NONFRAG,
4031                 [338] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4032                         RTE_PTYPE_TUNNEL_GTPU |
4033                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
4034                         RTE_PTYPE_INNER_L4_UDP,
4035                 [339] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4036                         RTE_PTYPE_TUNNEL_GTPU |
4037                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
4038                         RTE_PTYPE_INNER_L4_TCP,
4039                 [340] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4040                         RTE_PTYPE_TUNNEL_GTPU |
4041                         RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
4042                         RTE_PTYPE_INNER_L4_ICMP,
4043
4044                 /* IPv4 --> GTPU --> IPv6 */
4045                 [341] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4046                         RTE_PTYPE_TUNNEL_GTPU |
4047                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
4048                         RTE_PTYPE_INNER_L4_FRAG,
4049                 [342] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4050                         RTE_PTYPE_TUNNEL_GTPU |
4051                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
4052                         RTE_PTYPE_INNER_L4_NONFRAG,
4053                 [343] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4054                         RTE_PTYPE_TUNNEL_GTPU |
4055                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
4056                         RTE_PTYPE_INNER_L4_UDP,
4057                 [344] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4058                         RTE_PTYPE_TUNNEL_GTPU |
4059                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
4060                         RTE_PTYPE_INNER_L4_TCP,
4061                 [345] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4062                         RTE_PTYPE_TUNNEL_GTPU |
4063                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
4064                         RTE_PTYPE_INNER_L4_ICMP,
4065
4066                 /* IPv6 --> GTPU --> IPv6 */
4067                 [346] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4068                         RTE_PTYPE_TUNNEL_GTPU |
4069                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
4070                         RTE_PTYPE_INNER_L4_FRAG,
4071                 [347] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4072                         RTE_PTYPE_TUNNEL_GTPU |
4073                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
4074                         RTE_PTYPE_INNER_L4_NONFRAG,
4075                 [348] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4076                         RTE_PTYPE_TUNNEL_GTPU |
4077                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
4078                         RTE_PTYPE_INNER_L4_UDP,
4079                 [349] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4080                         RTE_PTYPE_TUNNEL_GTPU |
4081                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
4082                         RTE_PTYPE_INNER_L4_TCP,
4083                 [350] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4084                         RTE_PTYPE_TUNNEL_GTPU |
4085                         RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
4086                         RTE_PTYPE_INNER_L4_ICMP,
4087
4088                 /* IPv4 --> UDP ECPRI */
4089                 [372] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4090                         RTE_PTYPE_L4_UDP,
4091                 [373] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4092                         RTE_PTYPE_L4_UDP,
4093                 [374] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4094                         RTE_PTYPE_L4_UDP,
4095                 [375] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4096                         RTE_PTYPE_L4_UDP,
4097                 [376] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4098                         RTE_PTYPE_L4_UDP,
4099                 [377] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4100                         RTE_PTYPE_L4_UDP,
4101                 [378] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4102                         RTE_PTYPE_L4_UDP,
4103                 [379] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4104                         RTE_PTYPE_L4_UDP,
4105                 [380] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4106                         RTE_PTYPE_L4_UDP,
4107                 [381] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
4108                         RTE_PTYPE_L4_UDP,
4109
4110                 /* IPV6 --> UDP ECPRI */
4111                 [382] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4112                         RTE_PTYPE_L4_UDP,
4113                 [383] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4114                         RTE_PTYPE_L4_UDP,
4115                 [384] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4116                         RTE_PTYPE_L4_UDP,
4117                 [385] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4118                         RTE_PTYPE_L4_UDP,
4119                 [386] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4120                         RTE_PTYPE_L4_UDP,
4121                 [387] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4122                         RTE_PTYPE_L4_UDP,
4123                 [388] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4124                         RTE_PTYPE_L4_UDP,
4125                 [389] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4126                         RTE_PTYPE_L4_UDP,
4127                 [390] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4128                         RTE_PTYPE_L4_UDP,
4129                 [391] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
4130                         RTE_PTYPE_L4_UDP,
4131                 /* All others reserved */
4132         };
4133
4134         return type_table[ptype];
4135 }
4136
4137 void __rte_cold
4138 ice_set_default_ptype_table(struct rte_eth_dev *dev)
4139 {
4140         struct ice_adapter *ad =
4141                 ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
4142         int i;
4143
4144         for (i = 0; i < ICE_MAX_PKT_TYPE; i++)
4145                 ad->ptype_tbl[i] = ice_get_default_pkt_type(i);
4146 }
4147
4148 #define ICE_RX_PROG_STATUS_DESC_WB_QW1_PROGID_S 1
4149 #define ICE_RX_PROG_STATUS_DESC_WB_QW1_PROGID_M \
4150                         (0x3UL << ICE_RX_PROG_STATUS_DESC_WB_QW1_PROGID_S)
4151 #define ICE_RX_PROG_STATUS_DESC_WB_QW1_PROG_ADD 0
4152 #define ICE_RX_PROG_STATUS_DESC_WB_QW1_PROG_DEL 0x1
4153
4154 #define ICE_RX_PROG_STATUS_DESC_WB_QW1_FAIL_S   4
4155 #define ICE_RX_PROG_STATUS_DESC_WB_QW1_FAIL_M   \
4156         (1 << ICE_RX_PROG_STATUS_DESC_WB_QW1_FAIL_S)
4157 #define ICE_RX_PROG_STATUS_DESC_WB_QW1_FAIL_PROF_S      5
4158 #define ICE_RX_PROG_STATUS_DESC_WB_QW1_FAIL_PROF_M      \
4159         (1 << ICE_RX_PROG_STATUS_DESC_WB_QW1_FAIL_PROF_S)
4160
4161 /*
4162  * check the programming status descriptor in rx queue.
4163  * done after Programming Flow Director is programmed on
4164  * tx queue
4165  */
4166 static inline int
4167 ice_check_fdir_programming_status(struct ice_rx_queue *rxq)
4168 {
4169         volatile union ice_32byte_rx_desc *rxdp;
4170         uint64_t qword1;
4171         uint32_t rx_status;
4172         uint32_t error;
4173         uint32_t id;
4174         int ret = -EAGAIN;
4175
4176         rxdp = (volatile union ice_32byte_rx_desc *)
4177                 (&rxq->rx_ring[rxq->rx_tail]);
4178         qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len);
4179         rx_status = (qword1 & ICE_RXD_QW1_STATUS_M)
4180                         >> ICE_RXD_QW1_STATUS_S;
4181
4182         if (rx_status & (1 << ICE_RX_DESC_STATUS_DD_S)) {
4183                 ret = 0;
4184                 error = (qword1 & ICE_RX_PROG_STATUS_DESC_WB_QW1_FAIL_M) >>
4185                         ICE_RX_PROG_STATUS_DESC_WB_QW1_FAIL_S;
4186                 id = (qword1 & ICE_RX_PROG_STATUS_DESC_WB_QW1_PROGID_M) >>
4187                         ICE_RX_PROG_STATUS_DESC_WB_QW1_PROGID_S;
4188                 if (error) {
4189                         if (id == ICE_RX_PROG_STATUS_DESC_WB_QW1_PROG_ADD)
4190                                 PMD_DRV_LOG(ERR, "Failed to add FDIR rule.");
4191                         else if (id == ICE_RX_PROG_STATUS_DESC_WB_QW1_PROG_DEL)
4192                                 PMD_DRV_LOG(ERR, "Failed to remove FDIR rule.");
4193                         ret = -EINVAL;
4194                         goto err;
4195                 }
4196                 error = (qword1 & ICE_RX_PROG_STATUS_DESC_WB_QW1_FAIL_PROF_M) >>
4197                         ICE_RX_PROG_STATUS_DESC_WB_QW1_FAIL_PROF_S;
4198                 if (error) {
4199                         PMD_DRV_LOG(ERR, "Failed to create FDIR profile.");
4200                         ret = -EINVAL;
4201                 }
4202 err:
4203                 rxdp->wb.qword1.status_error_len = 0;
4204                 rxq->rx_tail++;
4205                 if (unlikely(rxq->rx_tail == rxq->nb_rx_desc))
4206                         rxq->rx_tail = 0;
4207                 if (rxq->rx_tail == 0)
4208                         ICE_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
4209                 else
4210                         ICE_PCI_REG_WRITE(rxq->qrx_tail, rxq->rx_tail - 1);
4211         }
4212
4213         return ret;
4214 }
4215
4216 #define ICE_FDIR_MAX_WAIT_US 10000
4217
4218 int
4219 ice_fdir_programming(struct ice_pf *pf, struct ice_fltr_desc *fdir_desc)
4220 {
4221         struct ice_tx_queue *txq = pf->fdir.txq;
4222         struct ice_rx_queue *rxq = pf->fdir.rxq;
4223         volatile struct ice_fltr_desc *fdirdp;
4224         volatile struct ice_tx_desc *txdp;
4225         uint32_t td_cmd;
4226         uint16_t i;
4227
4228         fdirdp = (volatile struct ice_fltr_desc *)
4229                 (&txq->tx_ring[txq->tx_tail]);
4230         fdirdp->qidx_compq_space_stat = fdir_desc->qidx_compq_space_stat;
4231         fdirdp->dtype_cmd_vsi_fdid = fdir_desc->dtype_cmd_vsi_fdid;
4232
4233         txdp = &txq->tx_ring[txq->tx_tail + 1];
4234         txdp->buf_addr = rte_cpu_to_le_64(pf->fdir.dma_addr);
4235         td_cmd = ICE_TX_DESC_CMD_EOP |
4236                 ICE_TX_DESC_CMD_RS  |
4237                 ICE_TX_DESC_CMD_DUMMY;
4238
4239         txdp->cmd_type_offset_bsz =
4240                 ice_build_ctob(td_cmd, 0, ICE_FDIR_PKT_LEN, 0);
4241
4242         txq->tx_tail += 2;
4243         if (txq->tx_tail >= txq->nb_tx_desc)
4244                 txq->tx_tail = 0;
4245         /* Update the tx tail register */
4246         ICE_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail);
4247         for (i = 0; i < ICE_FDIR_MAX_WAIT_US; i++) {
4248                 if ((txdp->cmd_type_offset_bsz &
4249                      rte_cpu_to_le_64(ICE_TXD_QW1_DTYPE_M)) ==
4250                     rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DESC_DONE))
4251                         break;
4252                 rte_delay_us(1);
4253         }
4254         if (i >= ICE_FDIR_MAX_WAIT_US) {
4255                 PMD_DRV_LOG(ERR,
4256                             "Failed to program FDIR filter: time out to get DD on tx queue.");
4257                 return -ETIMEDOUT;
4258         }
4259
4260         for (; i < ICE_FDIR_MAX_WAIT_US; i++) {
4261                 int ret;
4262
4263                 ret = ice_check_fdir_programming_status(rxq);
4264                 if (ret == -EAGAIN)
4265                         rte_delay_us(1);
4266                 else
4267                         return ret;
4268         }
4269
4270         PMD_DRV_LOG(ERR,
4271                     "Failed to program FDIR filter: programming status reported.");
4272         return -ETIMEDOUT;
4273
4274
4275 }