drivers/net/ice/ice_rxtx_vec_common.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019 Intel Corporation
   3  */
   4
   5 #ifndef _ICE_RXTX_VEC_COMMON_H_
   6 #define _ICE_RXTX_VEC_COMMON_H_
   7
   8 #include "ice_rxtx.h"
   9
  10 #ifndef __INTEL_COMPILER
  11 #pragma GCC diagnostic ignored "-Wcast-qual"
  12 #endif
  13
  14 static inline uint16_t
  15 ice_rx_reassemble_packets(struct ice_rx_queue *rxq, struct rte_mbuf **rx_bufs,
  16                           uint16_t nb_bufs, uint8_t *split_flags)
  17 {
  18         struct rte_mbuf *pkts[ICE_VPMD_RX_BURST] = {0}; /*finished pkts*/
  19         struct rte_mbuf *start = rxq->pkt_first_seg;
  20         struct rte_mbuf *end =  rxq->pkt_last_seg;
  21         unsigned int pkt_idx, buf_idx;
  22
  23         for (buf_idx = 0, pkt_idx = 0; buf_idx < nb_bufs; buf_idx++) {
  24                 if (end) {
  25                         /* processing a split packet */
  26                         end->next = rx_bufs[buf_idx];
  27                         rx_bufs[buf_idx]->data_len += rxq->crc_len;
  28
  29                         start->nb_segs++;
  30                         start->pkt_len += rx_bufs[buf_idx]->data_len;
  31                         end = end->next;
  32
  33                         if (!split_flags[buf_idx]) {
  34                                 /* it's the last packet of the set */
  35                                 start->hash = end->hash;
  36                                 start->vlan_tci = end->vlan_tci;
  37                                 start->ol_flags = end->ol_flags;
  38                                 /* we need to strip crc for the whole packet */
  39                                 start->pkt_len -= rxq->crc_len;
  40                                 if (end->data_len > rxq->crc_len) {
  41                                         end->data_len -= rxq->crc_len;
  42                                 } else {
  43                                         /* free up last mbuf */
  44                                         struct rte_mbuf *secondlast = start;
  45
  46                                         start->nb_segs--;
  47                                         while (secondlast->next != end)
  48                                                 secondlast = secondlast->next;
  49                                         secondlast->data_len -= (rxq->crc_len -
  50                                                         end->data_len);
  51                                         secondlast->next = NULL;
  52                                         rte_pktmbuf_free_seg(end);
  53                                 }
  54                                 pkts[pkt_idx++] = start;
  55                                 start = NULL;
  56                                 end = NULL;
  57                         }
  58                 } else {
  59                         /* not processing a split packet */
  60                         if (!split_flags[buf_idx]) {
  61                                 /* not a split packet, save and skip */
  62                                 pkts[pkt_idx++] = rx_bufs[buf_idx];
  63                                 continue;
  64                         }
  65                         start = rx_bufs[buf_idx];
  66                         end = start;
  67                         rx_bufs[buf_idx]->data_len += rxq->crc_len;
  68                         rx_bufs[buf_idx]->pkt_len += rxq->crc_len;
  69                 }
  70         }
  71
  72         /* save the partial packet for next time */
  73         rxq->pkt_first_seg = start;
  74         rxq->pkt_last_seg = end;
  75         rte_memcpy(rx_bufs, pkts, pkt_idx * (sizeof(*pkts)));
  76         return pkt_idx;
  77 }
  78
  79 static __rte_always_inline int
  80 ice_tx_free_bufs_vec(struct ice_tx_queue *txq)
  81 {
  82         struct ice_tx_entry *txep;
  83         uint32_t n;
  84         uint32_t i;
  85         int nb_free = 0;
  86         struct rte_mbuf *m, *free[ICE_TX_MAX_FREE_BUF_SZ];
  87
  88         /* check DD bits on threshold descriptor */
  89         if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &
  90                         rte_cpu_to_le_64(ICE_TXD_QW1_DTYPE_M)) !=
  91                         rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DESC_DONE))
  92                 return 0;
  93
  94         n = txq->tx_rs_thresh;
  95
  96          /* first buffer to free from S/W ring is at index
  97           * tx_next_dd - (tx_rs_thresh-1)
  98           */
  99         txep = &txq->sw_ring[txq->tx_next_dd - (n - 1)];
 100         m = rte_pktmbuf_prefree_seg(txep[0].mbuf);
 101         if (likely(m)) {
 102                 free[0] = m;
 103                 nb_free = 1;
 104                 for (i = 1; i < n; i++) {
 105                         m = rte_pktmbuf_prefree_seg(txep[i].mbuf);
 106                         if (likely(m)) {
 107                                 if (likely(m->pool == free[0]->pool)) {
 108                                         free[nb_free++] = m;
 109                                 } else {
 110                                         rte_mempool_put_bulk(free[0]->pool,
 111                                                              (void *)free,
 112                                                              nb_free);
 113                                         free[0] = m;
 114                                         nb_free = 1;
 115                                 }
 116                         }
 117                 }
 118                 rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free);
 119         } else {
 120                 for (i = 1; i < n; i++) {
 121                         m = rte_pktmbuf_prefree_seg(txep[i].mbuf);
 122                         if (m)
 123                                 rte_mempool_put(m->pool, m);
 124                 }
 125         }
 126
 127         /* buffers were freed, update counters */
 128         txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh);
 129         txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh);
 130         if (txq->tx_next_dd >= txq->nb_tx_desc)
 131                 txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
 132
 133         return txq->tx_rs_thresh;
 134 }
 135
 136 static __rte_always_inline void
 137 ice_tx_backlog_entry(struct ice_tx_entry *txep,
 138                      struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 139 {
 140         int i;
 141
 142         for (i = 0; i < (int)nb_pkts; ++i)
 143                 txep[i].mbuf = tx_pkts[i];
 144 }
 145
 146 static inline void
 147 _ice_rx_queue_release_mbufs_vec(struct ice_rx_queue *rxq)
 148 {
 149         const unsigned int mask = rxq->nb_rx_desc - 1;
 150         unsigned int i;
 151
 152         if (unlikely(!rxq->sw_ring)) {
 153                 PMD_DRV_LOG(DEBUG, "sw_ring is NULL");
 154                 return;
 155         }
 156
 157         if (rxq->rxrearm_nb >= rxq->nb_rx_desc)
 158                 return;
 159
 160         /* free all mbufs that are valid in the ring */
 161         if (rxq->rxrearm_nb == 0) {
 162                 for (i = 0; i < rxq->nb_rx_desc; i++) {
 163                         if (rxq->sw_ring[i].mbuf)
 164                                 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
 165                 }
 166         } else {
 167                 for (i = rxq->rx_tail;
 168                      i != rxq->rxrearm_start;
 169                      i = (i + 1) & mask) {
 170                         if (rxq->sw_ring[i].mbuf)
 171                                 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
 172                 }
 173         }
 174
 175         rxq->rxrearm_nb = rxq->nb_rx_desc;
 176
 177         /* set all entries to NULL */
 178         memset(rxq->sw_ring, 0, sizeof(rxq->sw_ring[0]) * rxq->nb_rx_desc);
 179 }
 180
 181 static inline void
 182 _ice_tx_queue_release_mbufs_vec(struct ice_tx_queue *txq)
 183 {
 184         uint16_t i;
 185
 186         if (unlikely(!txq || !txq->sw_ring)) {
 187                 PMD_DRV_LOG(DEBUG, "Pointer to rxq or sw_ring is NULL");
 188                 return;
 189         }
 190
 191         /**
 192          *  vPMD tx will not set sw_ring's mbuf to NULL after free,
 193          *  so need to free remains more carefully.
 194          */
 195         i = txq->tx_next_dd - txq->tx_rs_thresh + 1;
 196
 197 #ifdef CC_AVX512_SUPPORT
 198         struct rte_eth_dev *dev = &rte_eth_devices[txq->vsi->adapter->pf.dev_data->port_id];
 199
 200         if (dev->tx_pkt_burst == ice_xmit_pkts_vec_avx512 ||
 201             dev->tx_pkt_burst == ice_xmit_pkts_vec_avx512_offload) {
 202                 struct ice_vec_tx_entry *swr = (void *)txq->sw_ring;
 203
 204                 if (txq->tx_tail < i) {
 205                         for (; i < txq->nb_tx_desc; i++) {
 206                                 rte_pktmbuf_free_seg(swr[i].mbuf);
 207                                 swr[i].mbuf = NULL;
 208                         }
 209                         i = 0;
 210                 }
 211                 for (; i < txq->tx_tail; i++) {
 212                         rte_pktmbuf_free_seg(swr[i].mbuf);
 213                         swr[i].mbuf = NULL;
 214                 }
 215         } else
 216 #endif
 217         {
 218                 if (txq->tx_tail < i) {
 219                         for (; i < txq->nb_tx_desc; i++) {
 220                                 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
 221                                 txq->sw_ring[i].mbuf = NULL;
 222                         }
 223                         i = 0;
 224                 }
 225                 for (; i < txq->tx_tail; i++) {
 226                         rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
 227                         txq->sw_ring[i].mbuf = NULL;
 228                 }
 229         }
 230 }
 231
 232 static inline int
 233 ice_rxq_vec_setup_default(struct ice_rx_queue *rxq)
 234 {
 235         uintptr_t p;
 236         struct rte_mbuf mb_def = { .buf_addr = 0 }; /* zeroed mbuf */
 237
 238         mb_def.nb_segs = 1;
 239         mb_def.data_off = RTE_PKTMBUF_HEADROOM;
 240         mb_def.port = rxq->port_id;
 241         rte_mbuf_refcnt_set(&mb_def, 1);
 242
 243         /* prevent compiler reordering: rearm_data covers previous fields */
 244         rte_compiler_barrier();
 245         p = (uintptr_t)&mb_def.rearm_data;
 246         rxq->mbuf_initializer = *(uint64_t *)p;
 247         return 0;
 248 }
 249
 250 #define ICE_TX_NO_VECTOR_FLAGS (                        \
 251                 DEV_TX_OFFLOAD_MULTI_SEGS |             \
 252                 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |       \
 253                 DEV_TX_OFFLOAD_TCP_TSO)
 254
 255 #define ICE_TX_VECTOR_OFFLOAD (                         \
 256                 DEV_TX_OFFLOAD_VLAN_INSERT |            \
 257                 DEV_TX_OFFLOAD_QINQ_INSERT |            \
 258                 DEV_TX_OFFLOAD_IPV4_CKSUM |             \
 259                 DEV_TX_OFFLOAD_SCTP_CKSUM |             \
 260                 DEV_TX_OFFLOAD_UDP_CKSUM |              \
 261                 DEV_TX_OFFLOAD_TCP_CKSUM)
 262
 263 #define ICE_RX_VECTOR_OFFLOAD (                         \
 264                 DEV_RX_OFFLOAD_CHECKSUM |               \
 265                 DEV_RX_OFFLOAD_SCTP_CKSUM |             \
 266                 DEV_RX_OFFLOAD_VLAN |                   \
 267                 DEV_RX_OFFLOAD_RSS_HASH)
 268
 269 #define ICE_VECTOR_PATH         0
 270 #define ICE_VECTOR_OFFLOAD_PATH 1
 271
 272 static inline int
 273 ice_rx_vec_queue_default(struct ice_rx_queue *rxq)
 274 {
 275         if (!rxq)
 276                 return -1;
 277
 278         if (!rte_is_power_of_2(rxq->nb_rx_desc))
 279                 return -1;
 280
 281         if (rxq->rx_free_thresh < ICE_VPMD_RX_BURST)
 282                 return -1;
 283
 284         if (rxq->nb_rx_desc % rxq->rx_free_thresh)
 285                 return -1;
 286
 287         if (rxq->proto_xtr != PROTO_XTR_NONE)
 288                 return -1;
 289
 290         if (rxq->offloads & DEV_RX_OFFLOAD_TIMESTAMP)
 291                 return -1;
 292
 293         if (rxq->offloads & ICE_RX_VECTOR_OFFLOAD)
 294                 return ICE_VECTOR_OFFLOAD_PATH;
 295
 296         return ICE_VECTOR_PATH;
 297 }
 298
 299 static inline int
 300 ice_tx_vec_queue_default(struct ice_tx_queue *txq)
 301 {
 302         if (!txq)
 303                 return -1;
 304
 305         if (txq->tx_rs_thresh < ICE_VPMD_TX_BURST ||
 306             txq->tx_rs_thresh > ICE_TX_MAX_FREE_BUF_SZ)
 307                 return -1;
 308
 309         if (txq->offloads & ICE_TX_NO_VECTOR_FLAGS)
 310                 return -1;
 311
 312         if (txq->offloads & ICE_TX_VECTOR_OFFLOAD)
 313                 return ICE_VECTOR_OFFLOAD_PATH;
 314
 315         return ICE_VECTOR_PATH;
 316 }
 317
 318 static inline int
 319 ice_rx_vec_dev_check_default(struct rte_eth_dev *dev)
 320 {
 321         int i;
 322         struct ice_rx_queue *rxq;
 323         int ret = 0;
 324         int result = 0;
 325
 326         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 327                 rxq = dev->data->rx_queues[i];
 328                 ret = (ice_rx_vec_queue_default(rxq));
 329                 if (ret < 0)
 330                         return -1;
 331                 if (ret == ICE_VECTOR_OFFLOAD_PATH)
 332                         result = ret;
 333         }
 334
 335         return result;
 336 }
 337
 338 static inline int
 339 ice_tx_vec_dev_check_default(struct rte_eth_dev *dev)
 340 {
 341         int i;
 342         struct ice_tx_queue *txq;
 343         int ret = 0;
 344         int result = 0;
 345
 346         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 347                 txq = dev->data->tx_queues[i];
 348                 ret = ice_tx_vec_queue_default(txq);
 349                 if (ret < 0)
 350                         return -1;
 351                 if (ret == ICE_VECTOR_OFFLOAD_PATH)
 352                         result = ret;
 353         }
 354
 355         return result;
 356 }
 357
 358 #ifdef CC_AVX2_SUPPORT
 359 static __rte_always_inline void
 360 ice_rxq_rearm_common(struct ice_rx_queue *rxq, __rte_unused bool avx512)
 361 {
 362         int i;
 363         uint16_t rx_id;
 364         volatile union ice_rx_flex_desc *rxdp;
 365         struct ice_rx_entry *rxep = &rxq->sw_ring[rxq->rxrearm_start];
 366
 367         rxdp = rxq->rx_ring + rxq->rxrearm_start;
 368
 369         /* Pull 'n' more MBUFs into the software ring */
 370         if (rte_mempool_get_bulk(rxq->mp,
 371                                  (void *)rxep,
 372                                  ICE_RXQ_REARM_THRESH) < 0) {
 373                 if (rxq->rxrearm_nb + ICE_RXQ_REARM_THRESH >=
 374                     rxq->nb_rx_desc) {
 375                         __m128i dma_addr0;
 376
 377                         dma_addr0 = _mm_setzero_si128();
 378                         for (i = 0; i < ICE_DESCS_PER_LOOP; i++) {
 379                                 rxep[i].mbuf = &rxq->fake_mbuf;
 380                                 _mm_store_si128((__m128i *)&rxdp[i].read,
 381                                                 dma_addr0);
 382                         }
 383                 }
 384                 rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed +=
 385                         ICE_RXQ_REARM_THRESH;
 386                 return;
 387         }
 388
 389 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
 390         struct rte_mbuf *mb0, *mb1;
 391         __m128i dma_addr0, dma_addr1;
 392         __m128i hdr_room = _mm_set_epi64x(RTE_PKTMBUF_HEADROOM,
 393                         RTE_PKTMBUF_HEADROOM);
 394         /* Initialize the mbufs in vector, process 2 mbufs in one loop */
 395         for (i = 0; i < ICE_RXQ_REARM_THRESH; i += 2, rxep += 2) {
 396                 __m128i vaddr0, vaddr1;
 397
 398                 mb0 = rxep[0].mbuf;
 399                 mb1 = rxep[1].mbuf;
 400
 401                 /* load buf_addr(lo 64bit) and buf_iova(hi 64bit) */
 402                 RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) !=
 403                                 offsetof(struct rte_mbuf, buf_addr) + 8);
 404                 vaddr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr);
 405                 vaddr1 = _mm_loadu_si128((__m128i *)&mb1->buf_addr);
 406
 407                 /* convert pa to dma_addr hdr/data */
 408                 dma_addr0 = _mm_unpackhi_epi64(vaddr0, vaddr0);
 409                 dma_addr1 = _mm_unpackhi_epi64(vaddr1, vaddr1);
 410
 411                 /* add headroom to pa values */
 412                 dma_addr0 = _mm_add_epi64(dma_addr0, hdr_room);
 413                 dma_addr1 = _mm_add_epi64(dma_addr1, hdr_room);
 414
 415                 /* flush desc with pa dma_addr */
 416                 _mm_store_si128((__m128i *)&rxdp++->read, dma_addr0);
 417                 _mm_store_si128((__m128i *)&rxdp++->read, dma_addr1);
 418         }
 419 #else
 420 #ifdef CC_AVX512_SUPPORT
 421         if (avx512) {
 422                 struct rte_mbuf *mb0, *mb1, *mb2, *mb3;
 423                 struct rte_mbuf *mb4, *mb5, *mb6, *mb7;
 424                 __m512i dma_addr0_3, dma_addr4_7;
 425                 __m512i hdr_room = _mm512_set1_epi64(RTE_PKTMBUF_HEADROOM);
 426                 /* Initialize the mbufs in vector, process 8 mbufs in one loop */
 427                 for (i = 0; i < ICE_RXQ_REARM_THRESH;
 428                                 i += 8, rxep += 8, rxdp += 8) {
 429                         __m128i vaddr0, vaddr1, vaddr2, vaddr3;
 430                         __m128i vaddr4, vaddr5, vaddr6, vaddr7;
 431                         __m256i vaddr0_1, vaddr2_3;
 432                         __m256i vaddr4_5, vaddr6_7;
 433                         __m512i vaddr0_3, vaddr4_7;
 434
 435                         mb0 = rxep[0].mbuf;
 436                         mb1 = rxep[1].mbuf;
 437                         mb2 = rxep[2].mbuf;
 438                         mb3 = rxep[3].mbuf;
 439                         mb4 = rxep[4].mbuf;
 440                         mb5 = rxep[5].mbuf;
 441                         mb6 = rxep[6].mbuf;
 442                         mb7 = rxep[7].mbuf;
 443
 444                         /* load buf_addr(lo 64bit) and buf_iova(hi 64bit) */
 445                         RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) !=
 446                                         offsetof(struct rte_mbuf, buf_addr) + 8);
 447                         vaddr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr);
 448                         vaddr1 = _mm_loadu_si128((__m128i *)&mb1->buf_addr);
 449                         vaddr2 = _mm_loadu_si128((__m128i *)&mb2->buf_addr);
 450                         vaddr3 = _mm_loadu_si128((__m128i *)&mb3->buf_addr);
 451                         vaddr4 = _mm_loadu_si128((__m128i *)&mb4->buf_addr);
 452                         vaddr5 = _mm_loadu_si128((__m128i *)&mb5->buf_addr);
 453                         vaddr6 = _mm_loadu_si128((__m128i *)&mb6->buf_addr);
 454                         vaddr7 = _mm_loadu_si128((__m128i *)&mb7->buf_addr);
 455
 456                         /**
 457                          * merge 0 & 1, by casting 0 to 256-bit and inserting 1
 458                          * into the high lanes. Similarly for 2 & 3, and so on.
 459                          */
 460                         vaddr0_1 =
 461                                 _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr0),
 462                                                         vaddr1, 1);
 463                         vaddr2_3 =
 464                                 _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr2),
 465                                                         vaddr3, 1);
 466                         vaddr4_5 =
 467                                 _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr4),
 468                                                         vaddr5, 1);
 469                         vaddr6_7 =
 470                                 _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr6),
 471                                                         vaddr7, 1);
 472                         vaddr0_3 =
 473                                 _mm512_inserti64x4(_mm512_castsi256_si512(vaddr0_1),
 474                                                         vaddr2_3, 1);
 475                         vaddr4_7 =
 476                                 _mm512_inserti64x4(_mm512_castsi256_si512(vaddr4_5),
 477                                                         vaddr6_7, 1);
 478
 479                         /* convert pa to dma_addr hdr/data */
 480                         dma_addr0_3 = _mm512_unpackhi_epi64(vaddr0_3, vaddr0_3);
 481                         dma_addr4_7 = _mm512_unpackhi_epi64(vaddr4_7, vaddr4_7);
 482
 483                         /* add headroom to pa values */
 484                         dma_addr0_3 = _mm512_add_epi64(dma_addr0_3, hdr_room);
 485                         dma_addr4_7 = _mm512_add_epi64(dma_addr4_7, hdr_room);
 486
 487                         /* flush desc with pa dma_addr */
 488                         _mm512_store_si512((__m512i *)&rxdp->read, dma_addr0_3);
 489                         _mm512_store_si512((__m512i *)&(rxdp + 4)->read, dma_addr4_7);
 490                 }
 491         } else
 492 #endif
 493         {
 494                 struct rte_mbuf *mb0, *mb1, *mb2, *mb3;
 495                 __m256i dma_addr0_1, dma_addr2_3;
 496                 __m256i hdr_room = _mm256_set1_epi64x(RTE_PKTMBUF_HEADROOM);
 497                 /* Initialize the mbufs in vector, process 4 mbufs in one loop */
 498                 for (i = 0; i < ICE_RXQ_REARM_THRESH;
 499                                 i += 4, rxep += 4, rxdp += 4) {
 500                         __m128i vaddr0, vaddr1, vaddr2, vaddr3;
 501                         __m256i vaddr0_1, vaddr2_3;
 502
 503                         mb0 = rxep[0].mbuf;
 504                         mb1 = rxep[1].mbuf;
 505                         mb2 = rxep[2].mbuf;
 506                         mb3 = rxep[3].mbuf;
 507
 508                         /* load buf_addr(lo 64bit) and buf_iova(hi 64bit) */
 509                         RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) !=
 510                                         offsetof(struct rte_mbuf, buf_addr) + 8);
 511                         vaddr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr);
 512                         vaddr1 = _mm_loadu_si128((__m128i *)&mb1->buf_addr);
 513                         vaddr2 = _mm_loadu_si128((__m128i *)&mb2->buf_addr);
 514                         vaddr3 = _mm_loadu_si128((__m128i *)&mb3->buf_addr);
 515
 516                         /**
 517                          * merge 0 & 1, by casting 0 to 256-bit and inserting 1
 518                          * into the high lanes. Similarly for 2 & 3
 519                          */
 520                         vaddr0_1 =
 521                                 _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr0),
 522                                                         vaddr1, 1);
 523                         vaddr2_3 =
 524                                 _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr2),
 525                                                         vaddr3, 1);
 526
 527                         /* convert pa to dma_addr hdr/data */
 528                         dma_addr0_1 = _mm256_unpackhi_epi64(vaddr0_1, vaddr0_1);
 529                         dma_addr2_3 = _mm256_unpackhi_epi64(vaddr2_3, vaddr2_3);
 530
 531                         /* add headroom to pa values */
 532                         dma_addr0_1 = _mm256_add_epi64(dma_addr0_1, hdr_room);
 533                         dma_addr2_3 = _mm256_add_epi64(dma_addr2_3, hdr_room);
 534
 535                         /* flush desc with pa dma_addr */
 536                         _mm256_store_si256((__m256i *)&rxdp->read, dma_addr0_1);
 537                         _mm256_store_si256((__m256i *)&(rxdp + 2)->read, dma_addr2_3);
 538                 }
 539         }
 540
 541 #endif
 542
 543         rxq->rxrearm_start += ICE_RXQ_REARM_THRESH;
 544         if (rxq->rxrearm_start >= rxq->nb_rx_desc)
 545                 rxq->rxrearm_start = 0;
 546
 547         rxq->rxrearm_nb -= ICE_RXQ_REARM_THRESH;
 548
 549         rx_id = (uint16_t)((rxq->rxrearm_start == 0) ?
 550                              (rxq->nb_rx_desc - 1) : (rxq->rxrearm_start - 1));
 551
 552         /* Update the tail pointer on the NIC */
 553         ICE_PCI_REG_WC_WRITE(rxq->qrx_tail, rx_id);
 554 }
 555 #endif
 556
 557 static inline void
 558 ice_txd_enable_offload(struct rte_mbuf *tx_pkt,
 559                        uint64_t *txd_hi)
 560 {
 561         uint64_t ol_flags = tx_pkt->ol_flags;
 562         uint32_t td_cmd = 0;
 563         uint32_t td_offset = 0;
 564
 565         /* Tx Checksum Offload */
 566         /* SET MACLEN */
 567         td_offset |= (tx_pkt->l2_len >> 1) <<
 568                         ICE_TX_DESC_LEN_MACLEN_S;
 569
 570         /* Enable L3 checksum offload */
 571         if (ol_flags & PKT_TX_IP_CKSUM) {
 572                 td_cmd |= ICE_TX_DESC_CMD_IIPT_IPV4_CSUM;
 573                 td_offset |= (tx_pkt->l3_len >> 2) <<
 574                         ICE_TX_DESC_LEN_IPLEN_S;
 575         } else if (ol_flags & PKT_TX_IPV4) {
 576                 td_cmd |= ICE_TX_DESC_CMD_IIPT_IPV4;
 577                 td_offset |= (tx_pkt->l3_len >> 2) <<
 578                         ICE_TX_DESC_LEN_IPLEN_S;
 579         } else if (ol_flags & PKT_TX_IPV6) {
 580                 td_cmd |= ICE_TX_DESC_CMD_IIPT_IPV6;
 581                 td_offset |= (tx_pkt->l3_len >> 2) <<
 582                         ICE_TX_DESC_LEN_IPLEN_S;
 583         }
 584
 585         /* Enable L4 checksum offloads */
 586         switch (ol_flags & PKT_TX_L4_MASK) {
 587         case PKT_TX_TCP_CKSUM:
 588                 td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_TCP;
 589                 td_offset |= (sizeof(struct rte_tcp_hdr) >> 2) <<
 590                         ICE_TX_DESC_LEN_L4_LEN_S;
 591                 break;
 592         case PKT_TX_SCTP_CKSUM:
 593                 td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_SCTP;
 594                 td_offset |= (sizeof(struct rte_sctp_hdr) >> 2) <<
 595                         ICE_TX_DESC_LEN_L4_LEN_S;
 596                 break;
 597         case PKT_TX_UDP_CKSUM:
 598                 td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_UDP;
 599                 td_offset |= (sizeof(struct rte_udp_hdr) >> 2) <<
 600                         ICE_TX_DESC_LEN_L4_LEN_S;
 601                 break;
 602         default:
 603                 break;
 604         }
 605
 606         *txd_hi |= ((uint64_t)td_offset) << ICE_TXD_QW1_OFFSET_S;
 607
 608         /* Tx VLAN/QINQ insertion Offload */
 609         if (ol_flags & (PKT_TX_VLAN | PKT_TX_QINQ)) {
 610                 td_cmd |= ICE_TX_DESC_CMD_IL2TAG1;
 611                 *txd_hi |= ((uint64_t)tx_pkt->vlan_tci <<
 612                                 ICE_TXD_QW1_L2TAG1_S);
 613         }
 614
 615         *txd_hi |= ((uint64_t)td_cmd) << ICE_TXD_QW1_CMD_S;
 616 }
 617 #endif