drivers/net/ice/ice_rxtx_vec_common.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019 Intel Corporation
   3  */
   4
   5 #ifndef _ICE_RXTX_VEC_COMMON_H_
   6 #define _ICE_RXTX_VEC_COMMON_H_
   7
   8 #include "ice_rxtx.h"
   9
  10 #ifndef __INTEL_COMPILER
  11 #pragma GCC diagnostic ignored "-Wcast-qual"
  12 #endif
  13
  14 static inline uint16_t
  15 ice_rx_reassemble_packets(struct ice_rx_queue *rxq, struct rte_mbuf **rx_bufs,
  16                           uint16_t nb_bufs, uint8_t *split_flags)
  17 {
  18         struct rte_mbuf *pkts[ICE_VPMD_RX_BURST] = {0}; /*finished pkts*/
  19         struct rte_mbuf *start = rxq->pkt_first_seg;
  20         struct rte_mbuf *end =  rxq->pkt_last_seg;
  21         unsigned int pkt_idx, buf_idx;
  22
  23         for (buf_idx = 0, pkt_idx = 0; buf_idx < nb_bufs; buf_idx++) {
  24                 if (end) {
  25                         /* processing a split packet */
  26                         end->next = rx_bufs[buf_idx];
  27                         rx_bufs[buf_idx]->data_len += rxq->crc_len;
  28
  29                         start->nb_segs++;
  30                         start->pkt_len += rx_bufs[buf_idx]->data_len;
  31                         end = end->next;
  32
  33                         if (!split_flags[buf_idx]) {
  34                                 /* it's the last packet of the set */
  35                                 start->hash = end->hash;
  36                                 start->vlan_tci = end->vlan_tci;
  37                                 start->ol_flags = end->ol_flags;
  38                                 /* we need to strip crc for the whole packet */
  39                                 start->pkt_len -= rxq->crc_len;
  40                                 if (end->data_len > rxq->crc_len) {
  41                                         end->data_len -= rxq->crc_len;
  42                                 } else {
  43                                         /* free up last mbuf */
  44                                         struct rte_mbuf *secondlast = start;
  45
  46                                         start->nb_segs--;
  47                                         while (secondlast->next != end)
  48                                                 secondlast = secondlast->next;
  49                                         secondlast->data_len -= (rxq->crc_len -
  50                                                         end->data_len);
  51                                         secondlast->next = NULL;
  52                                         rte_pktmbuf_free_seg(end);
  53                                 }
  54                                 pkts[pkt_idx++] = start;
  55                                 start = NULL;
  56                                 end = NULL;
  57                         }
  58                 } else {
  59                         /* not processing a split packet */
  60                         if (!split_flags[buf_idx]) {
  61                                 /* not a split packet, save and skip */
  62                                 pkts[pkt_idx++] = rx_bufs[buf_idx];
  63                                 continue;
  64                         }
  65                         start = rx_bufs[buf_idx];
  66                         end = start;
  67                         rx_bufs[buf_idx]->data_len += rxq->crc_len;
  68                         rx_bufs[buf_idx]->pkt_len += rxq->crc_len;
  69                 }
  70         }
  71
  72         /* save the partial packet for next time */
  73         rxq->pkt_first_seg = start;
  74         rxq->pkt_last_seg = end;
  75         rte_memcpy(rx_bufs, pkts, pkt_idx * (sizeof(*pkts)));
  76         return pkt_idx;
  77 }
  78
  79 static __rte_always_inline int
  80 ice_tx_free_bufs_vec(struct ice_tx_queue *txq)
  81 {
  82         struct ice_tx_entry *txep;
  83         uint32_t n;
  84         uint32_t i;
  85         int nb_free = 0;
  86         struct rte_mbuf *m, *free[ICE_TX_MAX_FREE_BUF_SZ];
  87
  88         /* check DD bits on threshold descriptor */
  89         if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &
  90                         rte_cpu_to_le_64(ICE_TXD_QW1_DTYPE_M)) !=
  91                         rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DESC_DONE))
  92                 return 0;
  93
  94         n = txq->tx_rs_thresh;
  95
  96          /* first buffer to free from S/W ring is at index
  97           * tx_next_dd - (tx_rs_thresh-1)
  98           */
  99         txep = &txq->sw_ring[txq->tx_next_dd - (n - 1)];
 100         m = rte_pktmbuf_prefree_seg(txep[0].mbuf);
 101         if (likely(m)) {
 102                 free[0] = m;
 103                 nb_free = 1;
 104                 for (i = 1; i < n; i++) {
 105                         m = rte_pktmbuf_prefree_seg(txep[i].mbuf);
 106                         if (likely(m)) {
 107                                 if (likely(m->pool == free[0]->pool)) {
 108                                         free[nb_free++] = m;
 109                                 } else {
 110                                         rte_mempool_put_bulk(free[0]->pool,
 111                                                              (void *)free,
 112                                                              nb_free);
 113                                         free[0] = m;
 114                                         nb_free = 1;
 115                                 }
 116                         }
 117                 }
 118                 rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free);
 119         } else {
 120                 for (i = 1; i < n; i++) {
 121                         m = rte_pktmbuf_prefree_seg(txep[i].mbuf);
 122                         if (m)
 123                                 rte_mempool_put(m->pool, m);
 124                 }
 125         }
 126
 127         /* buffers were freed, update counters */
 128         txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh);
 129         txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh);
 130         if (txq->tx_next_dd >= txq->nb_tx_desc)
 131                 txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
 132
 133         return txq->tx_rs_thresh;
 134 }
 135
 136 static __rte_always_inline void
 137 ice_tx_backlog_entry(struct ice_tx_entry *txep,
 138                      struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 139 {
 140         int i;
 141
 142         for (i = 0; i < (int)nb_pkts; ++i)
 143                 txep[i].mbuf = tx_pkts[i];
 144 }
 145
 146 static inline void
 147 _ice_rx_queue_release_mbufs_vec(struct ice_rx_queue *rxq)
 148 {
 149         const unsigned int mask = rxq->nb_rx_desc - 1;
 150         unsigned int i;
 151
 152         if (unlikely(!rxq->sw_ring)) {
 153                 PMD_DRV_LOG(DEBUG, "sw_ring is NULL");
 154                 return;
 155         }
 156
 157         if (rxq->rxrearm_nb >= rxq->nb_rx_desc)
 158                 return;
 159
 160         /* free all mbufs that are valid in the ring */
 161         if (rxq->rxrearm_nb == 0) {
 162                 for (i = 0; i < rxq->nb_rx_desc; i++) {
 163                         if (rxq->sw_ring[i].mbuf)
 164                                 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
 165                 }
 166         } else {
 167                 for (i = rxq->rx_tail;
 168                      i != rxq->rxrearm_start;
 169                      i = (i + 1) & mask) {
 170                         if (rxq->sw_ring[i].mbuf)
 171                                 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
 172                 }
 173         }
 174
 175         rxq->rxrearm_nb = rxq->nb_rx_desc;
 176
 177         /* set all entries to NULL */
 178         memset(rxq->sw_ring, 0, sizeof(rxq->sw_ring[0]) * rxq->nb_rx_desc);
 179 }
 180
 181 static inline void
 182 _ice_tx_queue_release_mbufs_vec(struct ice_tx_queue *txq)
 183 {
 184         uint16_t i;
 185
 186         if (unlikely(!txq || !txq->sw_ring)) {
 187                 PMD_DRV_LOG(DEBUG, "Pointer to rxq or sw_ring is NULL");
 188                 return;
 189         }
 190
 191         /**
 192          *  vPMD tx will not set sw_ring's mbuf to NULL after free,
 193          *  so need to free remains more carefully.
 194          */
 195         i = txq->tx_next_dd - txq->tx_rs_thresh + 1;
 196
 197 #ifdef CC_AVX512_SUPPORT
 198         struct rte_eth_dev *dev = &rte_eth_devices[txq->vsi->adapter->pf.dev_data->port_id];
 199
 200         if (dev->tx_pkt_burst == ice_xmit_pkts_vec_avx512 ||
 201             dev->tx_pkt_burst == ice_xmit_pkts_vec_avx512_offload) {
 202                 struct ice_vec_tx_entry *swr = (void *)txq->sw_ring;
 203
 204                 if (txq->tx_tail < i) {
 205                         for (; i < txq->nb_tx_desc; i++) {
 206                                 rte_pktmbuf_free_seg(swr[i].mbuf);
 207                                 swr[i].mbuf = NULL;
 208                         }
 209                         i = 0;
 210                 }
 211                 for (; i < txq->tx_tail; i++) {
 212                         rte_pktmbuf_free_seg(swr[i].mbuf);
 213                         swr[i].mbuf = NULL;
 214                 }
 215         } else
 216 #endif
 217         {
 218                 if (txq->tx_tail < i) {
 219                         for (; i < txq->nb_tx_desc; i++) {
 220                                 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
 221                                 txq->sw_ring[i].mbuf = NULL;
 222                         }
 223                         i = 0;
 224                 }
 225                 for (; i < txq->tx_tail; i++) {
 226                         rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
 227                         txq->sw_ring[i].mbuf = NULL;
 228                 }
 229         }
 230 }
 231
 232 static inline int
 233 ice_rxq_vec_setup_default(struct ice_rx_queue *rxq)
 234 {
 235         uintptr_t p;
 236         struct rte_mbuf mb_def = { .buf_addr = 0 }; /* zeroed mbuf */
 237
 238         mb_def.nb_segs = 1;
 239         mb_def.data_off = RTE_PKTMBUF_HEADROOM;
 240         mb_def.port = rxq->port_id;
 241         rte_mbuf_refcnt_set(&mb_def, 1);
 242
 243         /* prevent compiler reordering: rearm_data covers previous fields */
 244         rte_compiler_barrier();
 245         p = (uintptr_t)&mb_def.rearm_data;
 246         rxq->mbuf_initializer = *(uint64_t *)p;
 247         return 0;
 248 }
 249
 250 #define ICE_TX_NO_VECTOR_FLAGS (                        \
 251                 DEV_TX_OFFLOAD_MULTI_SEGS |             \
 252                 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |       \
 253                 DEV_TX_OFFLOAD_TCP_TSO)
 254
 255 #define ICE_TX_VECTOR_OFFLOAD (                         \
 256                 DEV_TX_OFFLOAD_VLAN_INSERT |            \
 257                 DEV_TX_OFFLOAD_QINQ_INSERT |            \
 258                 DEV_TX_OFFLOAD_IPV4_CKSUM |             \
 259                 DEV_TX_OFFLOAD_SCTP_CKSUM |             \
 260                 DEV_TX_OFFLOAD_UDP_CKSUM |              \
 261                 DEV_TX_OFFLOAD_TCP_CKSUM)
 262
 263 #define ICE_RX_VECTOR_OFFLOAD (                         \
 264                 DEV_RX_OFFLOAD_CHECKSUM |               \
 265                 DEV_RX_OFFLOAD_SCTP_CKSUM |             \
 266                 DEV_RX_OFFLOAD_VLAN |                   \
 267                 DEV_RX_OFFLOAD_RSS_HASH)
 268
 269 #define ICE_VECTOR_PATH         0
 270 #define ICE_VECTOR_OFFLOAD_PATH 1
 271
 272 static inline int
 273 ice_rx_vec_queue_default(struct ice_rx_queue *rxq)
 274 {
 275         if (!rxq)
 276                 return -1;
 277
 278         if (!rte_is_power_of_2(rxq->nb_rx_desc))
 279                 return -1;
 280
 281         if (rxq->rx_free_thresh < ICE_VPMD_RX_BURST)
 282                 return -1;
 283
 284         if (rxq->nb_rx_desc % rxq->rx_free_thresh)
 285                 return -1;
 286
 287         if (rxq->proto_xtr != PROTO_XTR_NONE)
 288                 return -1;
 289
 290         if (rxq->offloads & ICE_RX_VECTOR_OFFLOAD)
 291                 return ICE_VECTOR_OFFLOAD_PATH;
 292
 293         return ICE_VECTOR_PATH;
 294 }
 295
 296 static inline int
 297 ice_tx_vec_queue_default(struct ice_tx_queue *txq)
 298 {
 299         if (!txq)
 300                 return -1;
 301
 302         if (txq->tx_rs_thresh < ICE_VPMD_TX_BURST ||
 303             txq->tx_rs_thresh > ICE_TX_MAX_FREE_BUF_SZ)
 304                 return -1;
 305
 306         if (txq->offloads & ICE_TX_NO_VECTOR_FLAGS)
 307                 return -1;
 308
 309         if (txq->offloads & ICE_TX_VECTOR_OFFLOAD)
 310                 return ICE_VECTOR_OFFLOAD_PATH;
 311
 312         return ICE_VECTOR_PATH;
 313 }
 314
 315 static inline int
 316 ice_rx_vec_dev_check_default(struct rte_eth_dev *dev)
 317 {
 318         int i;
 319         struct ice_rx_queue *rxq;
 320         int ret = 0;
 321         int result = 0;
 322
 323         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 324                 rxq = dev->data->rx_queues[i];
 325                 ret = (ice_rx_vec_queue_default(rxq));
 326                 if (ret < 0)
 327                         return -1;
 328                 if (ret == ICE_VECTOR_OFFLOAD_PATH)
 329                         result = ret;
 330         }
 331
 332         return result;
 333 }
 334
 335 static inline int
 336 ice_tx_vec_dev_check_default(struct rte_eth_dev *dev)
 337 {
 338         int i;
 339         struct ice_tx_queue *txq;
 340         int ret = 0;
 341         int result = 0;
 342
 343         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 344                 txq = dev->data->tx_queues[i];
 345                 ret = ice_tx_vec_queue_default(txq);
 346                 if (ret < 0)
 347                         return -1;
 348                 if (ret == ICE_VECTOR_OFFLOAD_PATH)
 349                         result = ret;
 350         }
 351
 352         return result;
 353 }
 354
 355 #ifdef CC_AVX2_SUPPORT
 356 static __rte_always_inline void
 357 ice_rxq_rearm_common(struct ice_rx_queue *rxq, __rte_unused bool avx512)
 358 {
 359         int i;
 360         uint16_t rx_id;
 361         volatile union ice_rx_flex_desc *rxdp;
 362         struct ice_rx_entry *rxep = &rxq->sw_ring[rxq->rxrearm_start];
 363
 364         rxdp = rxq->rx_ring + rxq->rxrearm_start;
 365
 366         /* Pull 'n' more MBUFs into the software ring */
 367         if (rte_mempool_get_bulk(rxq->mp,
 368                                  (void *)rxep,
 369                                  ICE_RXQ_REARM_THRESH) < 0) {
 370                 if (rxq->rxrearm_nb + ICE_RXQ_REARM_THRESH >=
 371                     rxq->nb_rx_desc) {
 372                         __m128i dma_addr0;
 373
 374                         dma_addr0 = _mm_setzero_si128();
 375                         for (i = 0; i < ICE_DESCS_PER_LOOP; i++) {
 376                                 rxep[i].mbuf = &rxq->fake_mbuf;
 377                                 _mm_store_si128((__m128i *)&rxdp[i].read,
 378                                                 dma_addr0);
 379                         }
 380                 }
 381                 rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed +=
 382                         ICE_RXQ_REARM_THRESH;
 383                 return;
 384         }
 385
 386 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
 387         struct rte_mbuf *mb0, *mb1;
 388         __m128i dma_addr0, dma_addr1;
 389         __m128i hdr_room = _mm_set_epi64x(RTE_PKTMBUF_HEADROOM,
 390                         RTE_PKTMBUF_HEADROOM);
 391         /* Initialize the mbufs in vector, process 2 mbufs in one loop */
 392         for (i = 0; i < ICE_RXQ_REARM_THRESH; i += 2, rxep += 2) {
 393                 __m128i vaddr0, vaddr1;
 394
 395                 mb0 = rxep[0].mbuf;
 396                 mb1 = rxep[1].mbuf;
 397
 398                 /* load buf_addr(lo 64bit) and buf_iova(hi 64bit) */
 399                 RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) !=
 400                                 offsetof(struct rte_mbuf, buf_addr) + 8);
 401                 vaddr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr);
 402                 vaddr1 = _mm_loadu_si128((__m128i *)&mb1->buf_addr);
 403
 404                 /* convert pa to dma_addr hdr/data */
 405                 dma_addr0 = _mm_unpackhi_epi64(vaddr0, vaddr0);
 406                 dma_addr1 = _mm_unpackhi_epi64(vaddr1, vaddr1);
 407
 408                 /* add headroom to pa values */
 409                 dma_addr0 = _mm_add_epi64(dma_addr0, hdr_room);
 410                 dma_addr1 = _mm_add_epi64(dma_addr1, hdr_room);
 411
 412                 /* flush desc with pa dma_addr */
 413                 _mm_store_si128((__m128i *)&rxdp++->read, dma_addr0);
 414                 _mm_store_si128((__m128i *)&rxdp++->read, dma_addr1);
 415         }
 416 #else
 417 #ifdef CC_AVX512_SUPPORT
 418         if (avx512) {
 419                 struct rte_mbuf *mb0, *mb1, *mb2, *mb3;
 420                 struct rte_mbuf *mb4, *mb5, *mb6, *mb7;
 421                 __m512i dma_addr0_3, dma_addr4_7;
 422                 __m512i hdr_room = _mm512_set1_epi64(RTE_PKTMBUF_HEADROOM);
 423                 /* Initialize the mbufs in vector, process 8 mbufs in one loop */
 424                 for (i = 0; i < ICE_RXQ_REARM_THRESH;
 425                                 i += 8, rxep += 8, rxdp += 8) {
 426                         __m128i vaddr0, vaddr1, vaddr2, vaddr3;
 427                         __m128i vaddr4, vaddr5, vaddr6, vaddr7;
 428                         __m256i vaddr0_1, vaddr2_3;
 429                         __m256i vaddr4_5, vaddr6_7;
 430                         __m512i vaddr0_3, vaddr4_7;
 431
 432                         mb0 = rxep[0].mbuf;
 433                         mb1 = rxep[1].mbuf;
 434                         mb2 = rxep[2].mbuf;
 435                         mb3 = rxep[3].mbuf;
 436                         mb4 = rxep[4].mbuf;
 437                         mb5 = rxep[5].mbuf;
 438                         mb6 = rxep[6].mbuf;
 439                         mb7 = rxep[7].mbuf;
 440
 441                         /* load buf_addr(lo 64bit) and buf_iova(hi 64bit) */
 442                         RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) !=
 443                                         offsetof(struct rte_mbuf, buf_addr) + 8);
 444                         vaddr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr);
 445                         vaddr1 = _mm_loadu_si128((__m128i *)&mb1->buf_addr);
 446                         vaddr2 = _mm_loadu_si128((__m128i *)&mb2->buf_addr);
 447                         vaddr3 = _mm_loadu_si128((__m128i *)&mb3->buf_addr);
 448                         vaddr4 = _mm_loadu_si128((__m128i *)&mb4->buf_addr);
 449                         vaddr5 = _mm_loadu_si128((__m128i *)&mb5->buf_addr);
 450                         vaddr6 = _mm_loadu_si128((__m128i *)&mb6->buf_addr);
 451                         vaddr7 = _mm_loadu_si128((__m128i *)&mb7->buf_addr);
 452
 453                         /**
 454                          * merge 0 & 1, by casting 0 to 256-bit and inserting 1
 455                          * into the high lanes. Similarly for 2 & 3, and so on.
 456                          */
 457                         vaddr0_1 =
 458                                 _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr0),
 459                                                         vaddr1, 1);
 460                         vaddr2_3 =
 461                                 _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr2),
 462                                                         vaddr3, 1);
 463                         vaddr4_5 =
 464                                 _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr4),
 465                                                         vaddr5, 1);
 466                         vaddr6_7 =
 467                                 _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr6),
 468                                                         vaddr7, 1);
 469                         vaddr0_3 =
 470                                 _mm512_inserti64x4(_mm512_castsi256_si512(vaddr0_1),
 471                                                         vaddr2_3, 1);
 472                         vaddr4_7 =
 473                                 _mm512_inserti64x4(_mm512_castsi256_si512(vaddr4_5),
 474                                                         vaddr6_7, 1);
 475
 476                         /* convert pa to dma_addr hdr/data */
 477                         dma_addr0_3 = _mm512_unpackhi_epi64(vaddr0_3, vaddr0_3);
 478                         dma_addr4_7 = _mm512_unpackhi_epi64(vaddr4_7, vaddr4_7);
 479
 480                         /* add headroom to pa values */
 481                         dma_addr0_3 = _mm512_add_epi64(dma_addr0_3, hdr_room);
 482                         dma_addr4_7 = _mm512_add_epi64(dma_addr4_7, hdr_room);
 483
 484                         /* flush desc with pa dma_addr */
 485                         _mm512_store_si512((__m512i *)&rxdp->read, dma_addr0_3);
 486                         _mm512_store_si512((__m512i *)&(rxdp + 4)->read, dma_addr4_7);
 487                 }
 488         } else
 489 #endif
 490         {
 491                 struct rte_mbuf *mb0, *mb1, *mb2, *mb3;
 492                 __m256i dma_addr0_1, dma_addr2_3;
 493                 __m256i hdr_room = _mm256_set1_epi64x(RTE_PKTMBUF_HEADROOM);
 494                 /* Initialize the mbufs in vector, process 4 mbufs in one loop */
 495                 for (i = 0; i < ICE_RXQ_REARM_THRESH;
 496                                 i += 4, rxep += 4, rxdp += 4) {
 497                         __m128i vaddr0, vaddr1, vaddr2, vaddr3;
 498                         __m256i vaddr0_1, vaddr2_3;
 499
 500                         mb0 = rxep[0].mbuf;
 501                         mb1 = rxep[1].mbuf;
 502                         mb2 = rxep[2].mbuf;
 503                         mb3 = rxep[3].mbuf;
 504
 505                         /* load buf_addr(lo 64bit) and buf_iova(hi 64bit) */
 506                         RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) !=
 507                                         offsetof(struct rte_mbuf, buf_addr) + 8);
 508                         vaddr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr);
 509                         vaddr1 = _mm_loadu_si128((__m128i *)&mb1->buf_addr);
 510                         vaddr2 = _mm_loadu_si128((__m128i *)&mb2->buf_addr);
 511                         vaddr3 = _mm_loadu_si128((__m128i *)&mb3->buf_addr);
 512
 513                         /**
 514                          * merge 0 & 1, by casting 0 to 256-bit and inserting 1
 515                          * into the high lanes. Similarly for 2 & 3
 516                          */
 517                         vaddr0_1 =
 518                                 _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr0),
 519                                                         vaddr1, 1);
 520                         vaddr2_3 =
 521                                 _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr2),
 522                                                         vaddr3, 1);
 523
 524                         /* convert pa to dma_addr hdr/data */
 525                         dma_addr0_1 = _mm256_unpackhi_epi64(vaddr0_1, vaddr0_1);
 526                         dma_addr2_3 = _mm256_unpackhi_epi64(vaddr2_3, vaddr2_3);
 527
 528                         /* add headroom to pa values */
 529                         dma_addr0_1 = _mm256_add_epi64(dma_addr0_1, hdr_room);
 530                         dma_addr2_3 = _mm256_add_epi64(dma_addr2_3, hdr_room);
 531
 532                         /* flush desc with pa dma_addr */
 533                         _mm256_store_si256((__m256i *)&rxdp->read, dma_addr0_1);
 534                         _mm256_store_si256((__m256i *)&(rxdp + 2)->read, dma_addr2_3);
 535                 }
 536         }
 537
 538 #endif
 539
 540         rxq->rxrearm_start += ICE_RXQ_REARM_THRESH;
 541         if (rxq->rxrearm_start >= rxq->nb_rx_desc)
 542                 rxq->rxrearm_start = 0;
 543
 544         rxq->rxrearm_nb -= ICE_RXQ_REARM_THRESH;
 545
 546         rx_id = (uint16_t)((rxq->rxrearm_start == 0) ?
 547                              (rxq->nb_rx_desc - 1) : (rxq->rxrearm_start - 1));
 548
 549         /* Update the tail pointer on the NIC */
 550         ICE_PCI_REG_WC_WRITE(rxq->qrx_tail, rx_id);
 551 }
 552 #endif
 553
 554 static inline void
 555 ice_txd_enable_offload(struct rte_mbuf *tx_pkt,
 556                        uint64_t *txd_hi)
 557 {
 558         uint64_t ol_flags = tx_pkt->ol_flags;
 559         uint32_t td_cmd = 0;
 560         uint32_t td_offset = 0;
 561
 562         /* Tx Checksum Offload */
 563         /* SET MACLEN */
 564         td_offset |= (tx_pkt->l2_len >> 1) <<
 565                         ICE_TX_DESC_LEN_MACLEN_S;
 566
 567         /* Enable L3 checksum offload */
 568         if (ol_flags & PKT_TX_IP_CKSUM) {
 569                 td_cmd |= ICE_TX_DESC_CMD_IIPT_IPV4_CSUM;
 570                 td_offset |= (tx_pkt->l3_len >> 2) <<
 571                         ICE_TX_DESC_LEN_IPLEN_S;
 572         } else if (ol_flags & PKT_TX_IPV4) {
 573                 td_cmd |= ICE_TX_DESC_CMD_IIPT_IPV4;
 574                 td_offset |= (tx_pkt->l3_len >> 2) <<
 575                         ICE_TX_DESC_LEN_IPLEN_S;
 576         } else if (ol_flags & PKT_TX_IPV6) {
 577                 td_cmd |= ICE_TX_DESC_CMD_IIPT_IPV6;
 578                 td_offset |= (tx_pkt->l3_len >> 2) <<
 579                         ICE_TX_DESC_LEN_IPLEN_S;
 580         }
 581
 582         /* Enable L4 checksum offloads */
 583         switch (ol_flags & PKT_TX_L4_MASK) {
 584         case PKT_TX_TCP_CKSUM:
 585                 td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_TCP;
 586                 td_offset |= (sizeof(struct rte_tcp_hdr) >> 2) <<
 587                         ICE_TX_DESC_LEN_L4_LEN_S;
 588                 break;
 589         case PKT_TX_SCTP_CKSUM:
 590                 td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_SCTP;
 591                 td_offset |= (sizeof(struct rte_sctp_hdr) >> 2) <<
 592                         ICE_TX_DESC_LEN_L4_LEN_S;
 593                 break;
 594         case PKT_TX_UDP_CKSUM:
 595                 td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_UDP;
 596                 td_offset |= (sizeof(struct rte_udp_hdr) >> 2) <<
 597                         ICE_TX_DESC_LEN_L4_LEN_S;
 598                 break;
 599         default:
 600                 break;
 601         }
 602
 603         *txd_hi |= ((uint64_t)td_offset) << ICE_TXD_QW1_OFFSET_S;
 604
 605         /* Tx VLAN/QINQ insertion Offload */
 606         if (ol_flags & (PKT_TX_VLAN | PKT_TX_QINQ)) {
 607                 td_cmd |= ICE_TX_DESC_CMD_IL2TAG1;
 608                 *txd_hi |= ((uint64_t)tx_pkt->vlan_tci <<
 609                                 ICE_TXD_QW1_L2TAG1_S);
 610         }
 611
 612         *txd_hi |= ((uint64_t)td_cmd) << ICE_TXD_QW1_CMD_S;
 613 }
 614 #endif