drivers/net/ice/ice_rxtx_vec_common.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019 Intel Corporation
   3  */
   4
   5 #ifndef _ICE_RXTX_VEC_COMMON_H_
   6 #define _ICE_RXTX_VEC_COMMON_H_
   7
   8 #include "ice_rxtx.h"
   9
  10 #ifndef __INTEL_COMPILER
  11 #pragma GCC diagnostic ignored "-Wcast-qual"
  12 #endif
  13
  14 static inline uint16_t
  15 ice_rx_reassemble_packets(struct ice_rx_queue *rxq, struct rte_mbuf **rx_bufs,
  16                           uint16_t nb_bufs, uint8_t *split_flags)
  17 {
  18         struct rte_mbuf *pkts[ICE_VPMD_RX_BURST] = {0}; /*finished pkts*/
  19         struct rte_mbuf *start = rxq->pkt_first_seg;
  20         struct rte_mbuf *end =  rxq->pkt_last_seg;
  21         unsigned int pkt_idx, buf_idx;
  22
  23         for (buf_idx = 0, pkt_idx = 0; buf_idx < nb_bufs; buf_idx++) {
  24                 if (end) {
  25                         /* processing a split packet */
  26                         end->next = rx_bufs[buf_idx];
  27                         rx_bufs[buf_idx]->data_len += rxq->crc_len;
  28
  29                         start->nb_segs++;
  30                         start->pkt_len += rx_bufs[buf_idx]->data_len;
  31                         end = end->next;
  32
  33                         if (!split_flags[buf_idx]) {
  34                                 /* it's the last packet of the set */
  35                                 start->hash = end->hash;
  36                                 start->vlan_tci = end->vlan_tci;
  37                                 start->ol_flags = end->ol_flags;
  38                                 /* we need to strip crc for the whole packet */
  39                                 start->pkt_len -= rxq->crc_len;
  40                                 if (end->data_len > rxq->crc_len) {
  41                                         end->data_len -= rxq->crc_len;
  42                                 } else {
  43                                         /* free up last mbuf */
  44                                         struct rte_mbuf *secondlast = start;
  45
  46                                         start->nb_segs--;
  47                                         while (secondlast->next != end)
  48                                                 secondlast = secondlast->next;
  49                                         secondlast->data_len -= (rxq->crc_len -
  50                                                         end->data_len);
  51                                         secondlast->next = NULL;
  52                                         rte_pktmbuf_free_seg(end);
  53                                 }
  54                                 pkts[pkt_idx++] = start;
  55                                 start = NULL;
  56                                 end = NULL;
  57                         }
  58                 } else {
  59                         /* not processing a split packet */
  60                         if (!split_flags[buf_idx]) {
  61                                 /* not a split packet, save and skip */
  62                                 pkts[pkt_idx++] = rx_bufs[buf_idx];
  63                                 continue;
  64                         }
  65                         start = rx_bufs[buf_idx];
  66                         end = start;
  67                         rx_bufs[buf_idx]->data_len += rxq->crc_len;
  68                         rx_bufs[buf_idx]->pkt_len += rxq->crc_len;
  69                 }
  70         }
  71
  72         /* save the partial packet for next time */
  73         rxq->pkt_first_seg = start;
  74         rxq->pkt_last_seg = end;
  75         rte_memcpy(rx_bufs, pkts, pkt_idx * (sizeof(*pkts)));
  76         return pkt_idx;
  77 }
  78
  79 static __rte_always_inline int
  80 ice_tx_free_bufs_vec(struct ice_tx_queue *txq)
  81 {
  82         struct ice_tx_entry *txep;
  83         uint32_t n;
  84         uint32_t i;
  85         int nb_free = 0;
  86         struct rte_mbuf *m, *free[ICE_TX_MAX_FREE_BUF_SZ];
  87
  88         /* check DD bits on threshold descriptor */
  89         if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &
  90                         rte_cpu_to_le_64(ICE_TXD_QW1_DTYPE_M)) !=
  91                         rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DESC_DONE))
  92                 return 0;
  93
  94         n = txq->tx_rs_thresh;
  95
  96          /* first buffer to free from S/W ring is at index
  97           * tx_next_dd - (tx_rs_thresh-1)
  98           */
  99         txep = &txq->sw_ring[txq->tx_next_dd - (n - 1)];
 100         m = rte_pktmbuf_prefree_seg(txep[0].mbuf);
 101         if (likely(m)) {
 102                 free[0] = m;
 103                 nb_free = 1;
 104                 for (i = 1; i < n; i++) {
 105                         m = rte_pktmbuf_prefree_seg(txep[i].mbuf);
 106                         if (likely(m)) {
 107                                 if (likely(m->pool == free[0]->pool)) {
 108                                         free[nb_free++] = m;
 109                                 } else {
 110                                         rte_mempool_put_bulk(free[0]->pool,
 111                                                              (void *)free,
 112                                                              nb_free);
 113                                         free[0] = m;
 114                                         nb_free = 1;
 115                                 }
 116                         }
 117                 }
 118                 rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free);
 119         } else {
 120                 for (i = 1; i < n; i++) {
 121                         m = rte_pktmbuf_prefree_seg(txep[i].mbuf);
 122                         if (m)
 123                                 rte_mempool_put(m->pool, m);
 124                 }
 125         }
 126
 127         /* buffers were freed, update counters */
 128         txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh);
 129         txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh);
 130         if (txq->tx_next_dd >= txq->nb_tx_desc)
 131                 txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
 132
 133         return txq->tx_rs_thresh;
 134 }
 135
 136 static __rte_always_inline void
 137 ice_tx_backlog_entry(struct ice_tx_entry *txep,
 138                      struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 139 {
 140         int i;
 141
 142         for (i = 0; i < (int)nb_pkts; ++i)
 143                 txep[i].mbuf = tx_pkts[i];
 144 }
 145
 146 static inline void
 147 _ice_rx_queue_release_mbufs_vec(struct ice_rx_queue *rxq)
 148 {
 149         const unsigned int mask = rxq->nb_rx_desc - 1;
 150         unsigned int i;
 151
 152         if (unlikely(!rxq->sw_ring)) {
 153                 PMD_DRV_LOG(DEBUG, "sw_ring is NULL");
 154                 return;
 155         }
 156
 157         if (rxq->rxrearm_nb >= rxq->nb_rx_desc)
 158                 return;
 159
 160         /* free all mbufs that are valid in the ring */
 161         if (rxq->rxrearm_nb == 0) {
 162                 for (i = 0; i < rxq->nb_rx_desc; i++) {
 163                         if (rxq->sw_ring[i].mbuf)
 164                                 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
 165                 }
 166         } else {
 167                 for (i = rxq->rx_tail;
 168                      i != rxq->rxrearm_start;
 169                      i = (i + 1) & mask) {
 170                         if (rxq->sw_ring[i].mbuf)
 171                                 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
 172                 }
 173         }
 174
 175         rxq->rxrearm_nb = rxq->nb_rx_desc;
 176
 177         /* set all entries to NULL */
 178         memset(rxq->sw_ring, 0, sizeof(rxq->sw_ring[0]) * rxq->nb_rx_desc);
 179 }
 180
 181 static inline void
 182 _ice_tx_queue_release_mbufs_vec(struct ice_tx_queue *txq)
 183 {
 184         uint16_t i;
 185
 186         if (unlikely(!txq || !txq->sw_ring)) {
 187                 PMD_DRV_LOG(DEBUG, "Pointer to rxq or sw_ring is NULL");
 188                 return;
 189         }
 190
 191         /**
 192          *  vPMD tx will not set sw_ring's mbuf to NULL after free,
 193          *  so need to free remains more carefully.
 194          */
 195         i = txq->tx_next_dd - txq->tx_rs_thresh + 1;
 196
 197 #ifdef CC_AVX512_SUPPORT
 198         struct rte_eth_dev *dev = txq->vsi->adapter->eth_dev;
 199
 200         if (dev->tx_pkt_burst == ice_xmit_pkts_vec_avx512 ||
 201             dev->tx_pkt_burst == ice_xmit_pkts_vec_avx512_offload) {
 202                 struct ice_vec_tx_entry *swr = (void *)txq->sw_ring;
 203
 204                 if (txq->tx_tail < i) {
 205                         for (; i < txq->nb_tx_desc; i++) {
 206                                 rte_pktmbuf_free_seg(swr[i].mbuf);
 207                                 swr[i].mbuf = NULL;
 208                         }
 209                         i = 0;
 210                 }
 211                 for (; i < txq->tx_tail; i++) {
 212                         rte_pktmbuf_free_seg(swr[i].mbuf);
 213                         swr[i].mbuf = NULL;
 214                 }
 215         } else
 216 #endif
 217         {
 218                 if (txq->tx_tail < i) {
 219                         for (; i < txq->nb_tx_desc; i++) {
 220                                 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
 221                                 txq->sw_ring[i].mbuf = NULL;
 222                         }
 223                         i = 0;
 224                 }
 225                 for (; i < txq->tx_tail; i++) {
 226                         rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
 227                         txq->sw_ring[i].mbuf = NULL;
 228                 }
 229         }
 230 }
 231
 232 static inline int
 233 ice_rxq_vec_setup_default(struct ice_rx_queue *rxq)
 234 {
 235         uintptr_t p;
 236         struct rte_mbuf mb_def = { .buf_addr = 0 }; /* zeroed mbuf */
 237
 238         mb_def.nb_segs = 1;
 239         mb_def.data_off = RTE_PKTMBUF_HEADROOM;
 240         mb_def.port = rxq->port_id;
 241         rte_mbuf_refcnt_set(&mb_def, 1);
 242
 243         /* prevent compiler reordering: rearm_data covers previous fields */
 244         rte_compiler_barrier();
 245         p = (uintptr_t)&mb_def.rearm_data;
 246         rxq->mbuf_initializer = *(uint64_t *)p;
 247         return 0;
 248 }
 249
 250 static inline int
 251 ice_rx_vec_queue_default(struct ice_rx_queue *rxq)
 252 {
 253         if (!rxq)
 254                 return -1;
 255
 256         if (!rte_is_power_of_2(rxq->nb_rx_desc))
 257                 return -1;
 258
 259         if (rxq->rx_free_thresh < ICE_VPMD_RX_BURST)
 260                 return -1;
 261
 262         if (rxq->nb_rx_desc % rxq->rx_free_thresh)
 263                 return -1;
 264
 265         if (rxq->proto_xtr != PROTO_XTR_NONE)
 266                 return -1;
 267
 268         return 0;
 269 }
 270
 271 #define ICE_TX_NO_VECTOR_FLAGS (                        \
 272                 DEV_TX_OFFLOAD_MULTI_SEGS |             \
 273                 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |       \
 274                 DEV_TX_OFFLOAD_TCP_TSO)
 275
 276 #define ICE_TX_VECTOR_OFFLOAD (                         \
 277                 DEV_TX_OFFLOAD_VLAN_INSERT |            \
 278                 DEV_TX_OFFLOAD_QINQ_INSERT |            \
 279                 DEV_TX_OFFLOAD_IPV4_CKSUM |             \
 280                 DEV_TX_OFFLOAD_SCTP_CKSUM |             \
 281                 DEV_TX_OFFLOAD_UDP_CKSUM |              \
 282                 DEV_TX_OFFLOAD_TCP_CKSUM)
 283
 284 #define ICE_VECTOR_PATH         0
 285 #define ICE_VECTOR_OFFLOAD_PATH 1
 286
 287 static inline int
 288 ice_tx_vec_queue_default(struct ice_tx_queue *txq)
 289 {
 290         if (!txq)
 291                 return -1;
 292
 293         if (txq->tx_rs_thresh < ICE_VPMD_TX_BURST ||
 294             txq->tx_rs_thresh > ICE_TX_MAX_FREE_BUF_SZ)
 295                 return -1;
 296
 297         if (txq->offloads & ICE_TX_NO_VECTOR_FLAGS)
 298                 return -1;
 299
 300         if (txq->offloads & ICE_TX_VECTOR_OFFLOAD)
 301                 return ICE_VECTOR_OFFLOAD_PATH;
 302
 303         return ICE_VECTOR_PATH;
 304 }
 305
 306 static inline int
 307 ice_rx_vec_dev_check_default(struct rte_eth_dev *dev)
 308 {
 309         int i;
 310         struct ice_rx_queue *rxq;
 311
 312         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 313                 rxq = dev->data->rx_queues[i];
 314                 if (ice_rx_vec_queue_default(rxq))
 315                         return -1;
 316         }
 317
 318         return 0;
 319 }
 320
 321 static inline int
 322 ice_tx_vec_dev_check_default(struct rte_eth_dev *dev)
 323 {
 324         int i;
 325         struct ice_tx_queue *txq;
 326         int ret = 0;
 327         int result = 0;
 328
 329         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 330                 txq = dev->data->tx_queues[i];
 331                 ret = ice_tx_vec_queue_default(txq);
 332                 if (ret < 0)
 333                         return -1;
 334                 if (ret == ICE_VECTOR_OFFLOAD_PATH)
 335                         result = ret;
 336         }
 337
 338         return result;
 339 }
 340
 341 #ifdef CC_AVX2_SUPPORT
 342 static __rte_always_inline void
 343 ice_rxq_rearm_common(struct ice_rx_queue *rxq, __rte_unused bool avx512)
 344 {
 345         int i;
 346         uint16_t rx_id;
 347         volatile union ice_rx_flex_desc *rxdp;
 348         struct ice_rx_entry *rxep = &rxq->sw_ring[rxq->rxrearm_start];
 349
 350         rxdp = rxq->rx_ring + rxq->rxrearm_start;
 351
 352         /* Pull 'n' more MBUFs into the software ring */
 353         if (rte_mempool_get_bulk(rxq->mp,
 354                                  (void *)rxep,
 355                                  ICE_RXQ_REARM_THRESH) < 0) {
 356                 if (rxq->rxrearm_nb + ICE_RXQ_REARM_THRESH >=
 357                     rxq->nb_rx_desc) {
 358                         __m128i dma_addr0;
 359
 360                         dma_addr0 = _mm_setzero_si128();
 361                         for (i = 0; i < ICE_DESCS_PER_LOOP; i++) {
 362                                 rxep[i].mbuf = &rxq->fake_mbuf;
 363                                 _mm_store_si128((__m128i *)&rxdp[i].read,
 364                                                 dma_addr0);
 365                         }
 366                 }
 367                 rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed +=
 368                         ICE_RXQ_REARM_THRESH;
 369                 return;
 370         }
 371
 372 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
 373         struct rte_mbuf *mb0, *mb1;
 374         __m128i dma_addr0, dma_addr1;
 375         __m128i hdr_room = _mm_set_epi64x(RTE_PKTMBUF_HEADROOM,
 376                         RTE_PKTMBUF_HEADROOM);
 377         /* Initialize the mbufs in vector, process 2 mbufs in one loop */
 378         for (i = 0; i < ICE_RXQ_REARM_THRESH; i += 2, rxep += 2) {
 379                 __m128i vaddr0, vaddr1;
 380
 381                 mb0 = rxep[0].mbuf;
 382                 mb1 = rxep[1].mbuf;
 383
 384                 /* load buf_addr(lo 64bit) and buf_iova(hi 64bit) */
 385                 RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) !=
 386                                 offsetof(struct rte_mbuf, buf_addr) + 8);
 387                 vaddr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr);
 388                 vaddr1 = _mm_loadu_si128((__m128i *)&mb1->buf_addr);
 389
 390                 /* convert pa to dma_addr hdr/data */
 391                 dma_addr0 = _mm_unpackhi_epi64(vaddr0, vaddr0);
 392                 dma_addr1 = _mm_unpackhi_epi64(vaddr1, vaddr1);
 393
 394                 /* add headroom to pa values */
 395                 dma_addr0 = _mm_add_epi64(dma_addr0, hdr_room);
 396                 dma_addr1 = _mm_add_epi64(dma_addr1, hdr_room);
 397
 398                 /* flush desc with pa dma_addr */
 399                 _mm_store_si128((__m128i *)&rxdp++->read, dma_addr0);
 400                 _mm_store_si128((__m128i *)&rxdp++->read, dma_addr1);
 401         }
 402 #else
 403 #ifdef CC_AVX512_SUPPORT
 404         if (avx512) {
 405                 struct rte_mbuf *mb0, *mb1, *mb2, *mb3;
 406                 struct rte_mbuf *mb4, *mb5, *mb6, *mb7;
 407                 __m512i dma_addr0_3, dma_addr4_7;
 408                 __m512i hdr_room = _mm512_set1_epi64(RTE_PKTMBUF_HEADROOM);
 409                 /* Initialize the mbufs in vector, process 8 mbufs in one loop */
 410                 for (i = 0; i < ICE_RXQ_REARM_THRESH;
 411                                 i += 8, rxep += 8, rxdp += 8) {
 412                         __m128i vaddr0, vaddr1, vaddr2, vaddr3;
 413                         __m128i vaddr4, vaddr5, vaddr6, vaddr7;
 414                         __m256i vaddr0_1, vaddr2_3;
 415                         __m256i vaddr4_5, vaddr6_7;
 416                         __m512i vaddr0_3, vaddr4_7;
 417
 418                         mb0 = rxep[0].mbuf;
 419                         mb1 = rxep[1].mbuf;
 420                         mb2 = rxep[2].mbuf;
 421                         mb3 = rxep[3].mbuf;
 422                         mb4 = rxep[4].mbuf;
 423                         mb5 = rxep[5].mbuf;
 424                         mb6 = rxep[6].mbuf;
 425                         mb7 = rxep[7].mbuf;
 426
 427                         /* load buf_addr(lo 64bit) and buf_iova(hi 64bit) */
 428                         RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) !=
 429                                         offsetof(struct rte_mbuf, buf_addr) + 8);
 430                         vaddr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr);
 431                         vaddr1 = _mm_loadu_si128((__m128i *)&mb1->buf_addr);
 432                         vaddr2 = _mm_loadu_si128((__m128i *)&mb2->buf_addr);
 433                         vaddr3 = _mm_loadu_si128((__m128i *)&mb3->buf_addr);
 434                         vaddr4 = _mm_loadu_si128((__m128i *)&mb4->buf_addr);
 435                         vaddr5 = _mm_loadu_si128((__m128i *)&mb5->buf_addr);
 436                         vaddr6 = _mm_loadu_si128((__m128i *)&mb6->buf_addr);
 437                         vaddr7 = _mm_loadu_si128((__m128i *)&mb7->buf_addr);
 438
 439                         /**
 440                          * merge 0 & 1, by casting 0 to 256-bit and inserting 1
 441                          * into the high lanes. Similarly for 2 & 3, and so on.
 442                          */
 443                         vaddr0_1 =
 444                                 _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr0),
 445                                                         vaddr1, 1);
 446                         vaddr2_3 =
 447                                 _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr2),
 448                                                         vaddr3, 1);
 449                         vaddr4_5 =
 450                                 _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr4),
 451                                                         vaddr5, 1);
 452                         vaddr6_7 =
 453                                 _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr6),
 454                                                         vaddr7, 1);
 455                         vaddr0_3 =
 456                                 _mm512_inserti64x4(_mm512_castsi256_si512(vaddr0_1),
 457                                                         vaddr2_3, 1);
 458                         vaddr4_7 =
 459                                 _mm512_inserti64x4(_mm512_castsi256_si512(vaddr4_5),
 460                                                         vaddr6_7, 1);
 461
 462                         /* convert pa to dma_addr hdr/data */
 463                         dma_addr0_3 = _mm512_unpackhi_epi64(vaddr0_3, vaddr0_3);
 464                         dma_addr4_7 = _mm512_unpackhi_epi64(vaddr4_7, vaddr4_7);
 465
 466                         /* add headroom to pa values */
 467                         dma_addr0_3 = _mm512_add_epi64(dma_addr0_3, hdr_room);
 468                         dma_addr4_7 = _mm512_add_epi64(dma_addr4_7, hdr_room);
 469
 470                         /* flush desc with pa dma_addr */
 471                         _mm512_store_si512((__m512i *)&rxdp->read, dma_addr0_3);
 472                         _mm512_store_si512((__m512i *)&(rxdp + 4)->read, dma_addr4_7);
 473                 }
 474         } else
 475 #endif
 476         {
 477                 struct rte_mbuf *mb0, *mb1, *mb2, *mb3;
 478                 __m256i dma_addr0_1, dma_addr2_3;
 479                 __m256i hdr_room = _mm256_set1_epi64x(RTE_PKTMBUF_HEADROOM);
 480                 /* Initialize the mbufs in vector, process 4 mbufs in one loop */
 481                 for (i = 0; i < ICE_RXQ_REARM_THRESH;
 482                                 i += 4, rxep += 4, rxdp += 4) {
 483                         __m128i vaddr0, vaddr1, vaddr2, vaddr3;
 484                         __m256i vaddr0_1, vaddr2_3;
 485
 486                         mb0 = rxep[0].mbuf;
 487                         mb1 = rxep[1].mbuf;
 488                         mb2 = rxep[2].mbuf;
 489                         mb3 = rxep[3].mbuf;
 490
 491                         /* load buf_addr(lo 64bit) and buf_iova(hi 64bit) */
 492                         RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) !=
 493                                         offsetof(struct rte_mbuf, buf_addr) + 8);
 494                         vaddr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr);
 495                         vaddr1 = _mm_loadu_si128((__m128i *)&mb1->buf_addr);
 496                         vaddr2 = _mm_loadu_si128((__m128i *)&mb2->buf_addr);
 497                         vaddr3 = _mm_loadu_si128((__m128i *)&mb3->buf_addr);
 498
 499                         /**
 500                          * merge 0 & 1, by casting 0 to 256-bit and inserting 1
 501                          * into the high lanes. Similarly for 2 & 3
 502                          */
 503                         vaddr0_1 =
 504                                 _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr0),
 505                                                         vaddr1, 1);
 506                         vaddr2_3 =
 507                                 _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr2),
 508                                                         vaddr3, 1);
 509
 510                         /* convert pa to dma_addr hdr/data */
 511                         dma_addr0_1 = _mm256_unpackhi_epi64(vaddr0_1, vaddr0_1);
 512                         dma_addr2_3 = _mm256_unpackhi_epi64(vaddr2_3, vaddr2_3);
 513
 514                         /* add headroom to pa values */
 515                         dma_addr0_1 = _mm256_add_epi64(dma_addr0_1, hdr_room);
 516                         dma_addr2_3 = _mm256_add_epi64(dma_addr2_3, hdr_room);
 517
 518                         /* flush desc with pa dma_addr */
 519                         _mm256_store_si256((__m256i *)&rxdp->read, dma_addr0_1);
 520                         _mm256_store_si256((__m256i *)&(rxdp + 2)->read, dma_addr2_3);
 521                 }
 522         }
 523
 524 #endif
 525
 526         rxq->rxrearm_start += ICE_RXQ_REARM_THRESH;
 527         if (rxq->rxrearm_start >= rxq->nb_rx_desc)
 528                 rxq->rxrearm_start = 0;
 529
 530         rxq->rxrearm_nb -= ICE_RXQ_REARM_THRESH;
 531
 532         rx_id = (uint16_t)((rxq->rxrearm_start == 0) ?
 533                              (rxq->nb_rx_desc - 1) : (rxq->rxrearm_start - 1));
 534
 535         /* Update the tail pointer on the NIC */
 536         ICE_PCI_REG_WC_WRITE(rxq->qrx_tail, rx_id);
 537 }
 538 #endif
 539
 540 static inline void
 541 ice_txd_enable_offload(struct rte_mbuf *tx_pkt,
 542                        uint64_t *txd_hi)
 543 {
 544         uint64_t ol_flags = tx_pkt->ol_flags;
 545         uint32_t td_cmd = 0;
 546         uint32_t td_offset = 0;
 547
 548         /* Tx Checksum Offload */
 549         /* SET MACLEN */
 550         td_offset |= (tx_pkt->l2_len >> 1) <<
 551                         ICE_TX_DESC_LEN_MACLEN_S;
 552
 553         /* Enable L3 checksum offload */
 554         if (ol_flags & PKT_TX_IP_CKSUM) {
 555                 td_cmd |= ICE_TX_DESC_CMD_IIPT_IPV4_CSUM;
 556                 td_offset |= (tx_pkt->l3_len >> 2) <<
 557                         ICE_TX_DESC_LEN_IPLEN_S;
 558         } else if (ol_flags & PKT_TX_IPV4) {
 559                 td_cmd |= ICE_TX_DESC_CMD_IIPT_IPV4;
 560                 td_offset |= (tx_pkt->l3_len >> 2) <<
 561                         ICE_TX_DESC_LEN_IPLEN_S;
 562         } else if (ol_flags & PKT_TX_IPV6) {
 563                 td_cmd |= ICE_TX_DESC_CMD_IIPT_IPV6;
 564                 td_offset |= (tx_pkt->l3_len >> 2) <<
 565                         ICE_TX_DESC_LEN_IPLEN_S;
 566         }
 567
 568         /* Enable L4 checksum offloads */
 569         switch (ol_flags & PKT_TX_L4_MASK) {
 570         case PKT_TX_TCP_CKSUM:
 571                 td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_TCP;
 572                 td_offset |= (sizeof(struct rte_tcp_hdr) >> 2) <<
 573                         ICE_TX_DESC_LEN_L4_LEN_S;
 574                 break;
 575         case PKT_TX_SCTP_CKSUM:
 576                 td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_SCTP;
 577                 td_offset |= (sizeof(struct rte_sctp_hdr) >> 2) <<
 578                         ICE_TX_DESC_LEN_L4_LEN_S;
 579                 break;
 580         case PKT_TX_UDP_CKSUM:
 581                 td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_UDP;
 582                 td_offset |= (sizeof(struct rte_udp_hdr) >> 2) <<
 583                         ICE_TX_DESC_LEN_L4_LEN_S;
 584                 break;
 585         default:
 586                 break;
 587         }
 588
 589         *txd_hi |= ((uint64_t)td_offset) << ICE_TXD_QW1_OFFSET_S;
 590
 591         /* Tx VLAN/QINQ insertion Offload */
 592         if (ol_flags & (PKT_TX_VLAN | PKT_TX_QINQ)) {
 593                 td_cmd |= ICE_TX_DESC_CMD_IL2TAG1;
 594                 *txd_hi |= ((uint64_t)tx_pkt->vlan_tci <<
 595                                 ICE_TXD_QW1_L2TAG1_S);
 596         }
 597
 598         *txd_hi |= ((uint64_t)td_cmd) << ICE_TXD_QW1_CMD_S;
 599 }
 600 #endif