drivers/net/ice/ice_rxtx_vec_common.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019 Intel Corporation
   3  */
   4
   5 #ifndef _ICE_RXTX_VEC_COMMON_H_
   6 #define _ICE_RXTX_VEC_COMMON_H_
   7
   8 #include "ice_rxtx.h"
   9
  10 #ifndef __INTEL_COMPILER
  11 #pragma GCC diagnostic ignored "-Wcast-qual"
  12 #endif
  13
  14 static inline uint16_t
  15 ice_rx_reassemble_packets(struct ice_rx_queue *rxq, struct rte_mbuf **rx_bufs,
  16                           uint16_t nb_bufs, uint8_t *split_flags)
  17 {
  18         struct rte_mbuf *pkts[ICE_VPMD_RX_BURST] = {0}; /*finished pkts*/
  19         struct rte_mbuf *start = rxq->pkt_first_seg;
  20         struct rte_mbuf *end =  rxq->pkt_last_seg;
  21         unsigned int pkt_idx, buf_idx;
  22
  23         for (buf_idx = 0, pkt_idx = 0; buf_idx < nb_bufs; buf_idx++) {
  24                 if (end) {
  25                         /* processing a split packet */
  26                         end->next = rx_bufs[buf_idx];
  27                         rx_bufs[buf_idx]->data_len += rxq->crc_len;
  28
  29                         start->nb_segs++;
  30                         start->pkt_len += rx_bufs[buf_idx]->data_len;
  31                         end = end->next;
  32
  33                         if (!split_flags[buf_idx]) {
  34                                 /* it's the last packet of the set */
  35                                 start->hash = end->hash;
  36                                 start->vlan_tci = end->vlan_tci;
  37                                 start->ol_flags = end->ol_flags;
  38                                 /* we need to strip crc for the whole packet */
  39                                 start->pkt_len -= rxq->crc_len;
  40                                 if (end->data_len > rxq->crc_len) {
  41                                         end->data_len -= rxq->crc_len;
  42                                 } else {
  43                                         /* free up last mbuf */
  44                                         struct rte_mbuf *secondlast = start;
  45
  46                                         start->nb_segs--;
  47                                         while (secondlast->next != end)
  48                                                 secondlast = secondlast->next;
  49                                         secondlast->data_len -= (rxq->crc_len -
  50                                                         end->data_len);
  51                                         secondlast->next = NULL;
  52                                         rte_pktmbuf_free_seg(end);
  53                                 }
  54                                 pkts[pkt_idx++] = start;
  55                                 start = NULL;
  56                                 end = NULL;
  57                         }
  58                 } else {
  59                         /* not processing a split packet */
  60                         if (!split_flags[buf_idx]) {
  61                                 /* not a split packet, save and skip */
  62                                 pkts[pkt_idx++] = rx_bufs[buf_idx];
  63                                 continue;
  64                         }
  65                         start = rx_bufs[buf_idx];
  66                         end = start;
  67                         rx_bufs[buf_idx]->data_len += rxq->crc_len;
  68                         rx_bufs[buf_idx]->pkt_len += rxq->crc_len;
  69                 }
  70         }
  71
  72         /* save the partial packet for next time */
  73         rxq->pkt_first_seg = start;
  74         rxq->pkt_last_seg = end;
  75         rte_memcpy(rx_bufs, pkts, pkt_idx * (sizeof(*pkts)));
  76         return pkt_idx;
  77 }
  78
  79 static __rte_always_inline int
  80 ice_tx_free_bufs(struct ice_tx_queue *txq)
  81 {
  82         struct ice_tx_entry *txep;
  83         uint32_t n;
  84         uint32_t i;
  85         int nb_free = 0;
  86         struct rte_mbuf *m, *free[ICE_TX_MAX_FREE_BUF_SZ];
  87
  88         /* check DD bits on threshold descriptor */
  89         if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &
  90                         rte_cpu_to_le_64(ICE_TXD_QW1_DTYPE_M)) !=
  91                         rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DESC_DONE))
  92                 return 0;
  93
  94         n = txq->tx_rs_thresh;
  95
  96          /* first buffer to free from S/W ring is at index
  97           * tx_next_dd - (tx_rs_thresh-1)
  98           */
  99         txep = &txq->sw_ring[txq->tx_next_dd - (n - 1)];
 100         m = rte_pktmbuf_prefree_seg(txep[0].mbuf);
 101         if (likely(m)) {
 102                 free[0] = m;
 103                 nb_free = 1;
 104                 for (i = 1; i < n; i++) {
 105                         m = rte_pktmbuf_prefree_seg(txep[i].mbuf);
 106                         if (likely(m)) {
 107                                 if (likely(m->pool == free[0]->pool)) {
 108                                         free[nb_free++] = m;
 109                                 } else {
 110                                         rte_mempool_put_bulk(free[0]->pool,
 111                                                              (void *)free,
 112                                                              nb_free);
 113                                         free[0] = m;
 114                                         nb_free = 1;
 115                                 }
 116                         }
 117                 }
 118                 rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free);
 119         } else {
 120                 for (i = 1; i < n; i++) {
 121                         m = rte_pktmbuf_prefree_seg(txep[i].mbuf);
 122                         if (m)
 123                                 rte_mempool_put(m->pool, m);
 124                 }
 125         }
 126
 127         /* buffers were freed, update counters */
 128         txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh);
 129         txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh);
 130         if (txq->tx_next_dd >= txq->nb_tx_desc)
 131                 txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
 132
 133         return txq->tx_rs_thresh;
 134 }
 135
 136 static __rte_always_inline void
 137 ice_tx_backlog_entry(struct ice_tx_entry *txep,
 138                      struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 139 {
 140         int i;
 141
 142         for (i = 0; i < (int)nb_pkts; ++i)
 143                 txep[i].mbuf = tx_pkts[i];
 144 }
 145
 146 static inline void
 147 _ice_rx_queue_release_mbufs_vec(struct ice_rx_queue *rxq)
 148 {
 149         const unsigned int mask = rxq->nb_rx_desc - 1;
 150         unsigned int i;
 151
 152         if (unlikely(!rxq->sw_ring)) {
 153                 PMD_DRV_LOG(DEBUG, "sw_ring is NULL");
 154                 return;
 155         }
 156
 157         if (rxq->rxrearm_nb >= rxq->nb_rx_desc)
 158                 return;
 159
 160         /* free all mbufs that are valid in the ring */
 161         if (rxq->rxrearm_nb == 0) {
 162                 for (i = 0; i < rxq->nb_rx_desc; i++) {
 163                         if (rxq->sw_ring[i].mbuf)
 164                                 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
 165                 }
 166         } else {
 167                 for (i = rxq->rx_tail;
 168                      i != rxq->rxrearm_start;
 169                      i = (i + 1) & mask) {
 170                         if (rxq->sw_ring[i].mbuf)
 171                                 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
 172                 }
 173         }
 174
 175         rxq->rxrearm_nb = rxq->nb_rx_desc;
 176
 177         /* set all entries to NULL */
 178         memset(rxq->sw_ring, 0, sizeof(rxq->sw_ring[0]) * rxq->nb_rx_desc);
 179 }
 180
 181 static inline void
 182 _ice_tx_queue_release_mbufs_vec(struct ice_tx_queue *txq)
 183 {
 184         uint16_t i;
 185
 186         if (unlikely(!txq || !txq->sw_ring)) {
 187                 PMD_DRV_LOG(DEBUG, "Pointer to rxq or sw_ring is NULL");
 188                 return;
 189         }
 190
 191         /**
 192          *  vPMD tx will not set sw_ring's mbuf to NULL after free,
 193          *  so need to free remains more carefully.
 194          */
 195         i = txq->tx_next_dd - txq->tx_rs_thresh + 1;
 196
 197 #ifdef CC_AVX512_SUPPORT
 198         struct rte_eth_dev *dev = txq->vsi->adapter->eth_dev;
 199
 200         if (dev->tx_pkt_burst == ice_xmit_pkts_vec_avx512) {
 201                 struct ice_vec_tx_entry *swr = (void *)txq->sw_ring;
 202
 203                 if (txq->tx_tail < i) {
 204                         for (; i < txq->nb_tx_desc; i++) {
 205                                 rte_pktmbuf_free_seg(swr[i].mbuf);
 206                                 swr[i].mbuf = NULL;
 207                         }
 208                         i = 0;
 209                 }
 210                 for (; i < txq->tx_tail; i++) {
 211                         rte_pktmbuf_free_seg(swr[i].mbuf);
 212                         swr[i].mbuf = NULL;
 213                 }
 214         } else
 215 #endif
 216         {
 217                 if (txq->tx_tail < i) {
 218                         for (; i < txq->nb_tx_desc; i++) {
 219                                 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
 220                                 txq->sw_ring[i].mbuf = NULL;
 221                         }
 222                         i = 0;
 223                 }
 224                 for (; i < txq->tx_tail; i++) {
 225                         rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
 226                         txq->sw_ring[i].mbuf = NULL;
 227                 }
 228         }
 229 }
 230
 231 static inline int
 232 ice_rxq_vec_setup_default(struct ice_rx_queue *rxq)
 233 {
 234         uintptr_t p;
 235         struct rte_mbuf mb_def = { .buf_addr = 0 }; /* zeroed mbuf */
 236
 237         mb_def.nb_segs = 1;
 238         mb_def.data_off = RTE_PKTMBUF_HEADROOM;
 239         mb_def.port = rxq->port_id;
 240         rte_mbuf_refcnt_set(&mb_def, 1);
 241
 242         /* prevent compiler reordering: rearm_data covers previous fields */
 243         rte_compiler_barrier();
 244         p = (uintptr_t)&mb_def.rearm_data;
 245         rxq->mbuf_initializer = *(uint64_t *)p;
 246         return 0;
 247 }
 248
 249 static inline int
 250 ice_rx_vec_queue_default(struct ice_rx_queue *rxq)
 251 {
 252         if (!rxq)
 253                 return -1;
 254
 255         if (!rte_is_power_of_2(rxq->nb_rx_desc))
 256                 return -1;
 257
 258         if (rxq->rx_free_thresh < ICE_VPMD_RX_BURST)
 259                 return -1;
 260
 261         if (rxq->nb_rx_desc % rxq->rx_free_thresh)
 262                 return -1;
 263
 264         if (rxq->proto_xtr != PROTO_XTR_NONE)
 265                 return -1;
 266
 267         return 0;
 268 }
 269
 270 #define ICE_NO_VECTOR_FLAGS (                            \
 271                 DEV_TX_OFFLOAD_MULTI_SEGS |              \
 272                 DEV_TX_OFFLOAD_VLAN_INSERT |             \
 273                 DEV_TX_OFFLOAD_IPV4_CKSUM |              \
 274                 DEV_TX_OFFLOAD_SCTP_CKSUM |              \
 275                 DEV_TX_OFFLOAD_UDP_CKSUM |               \
 276                 DEV_TX_OFFLOAD_TCP_TSO |                 \
 277                 DEV_TX_OFFLOAD_TCP_CKSUM)
 278
 279 static inline int
 280 ice_tx_vec_queue_default(struct ice_tx_queue *txq)
 281 {
 282         if (!txq)
 283                 return -1;
 284
 285         if (txq->offloads & ICE_NO_VECTOR_FLAGS)
 286                 return -1;
 287
 288         if (txq->tx_rs_thresh < ICE_VPMD_TX_BURST ||
 289             txq->tx_rs_thresh > ICE_TX_MAX_FREE_BUF_SZ)
 290                 return -1;
 291
 292         return 0;
 293 }
 294
 295 static inline int
 296 ice_rx_vec_dev_check_default(struct rte_eth_dev *dev)
 297 {
 298         int i;
 299         struct ice_rx_queue *rxq;
 300
 301         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 302                 rxq = dev->data->rx_queues[i];
 303                 if (ice_rx_vec_queue_default(rxq))
 304                         return -1;
 305         }
 306
 307         return 0;
 308 }
 309
 310 static inline int
 311 ice_tx_vec_dev_check_default(struct rte_eth_dev *dev)
 312 {
 313         int i;
 314         struct ice_tx_queue *txq;
 315
 316         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 317                 txq = dev->data->tx_queues[i];
 318                 if (ice_tx_vec_queue_default(txq))
 319                         return -1;
 320         }
 321
 322         return 0;
 323 }
 324
 325 #ifdef CC_AVX2_SUPPORT
 326 static __rte_always_inline void
 327 ice_rxq_rearm_common(struct ice_rx_queue *rxq, __rte_unused bool avx512)
 328 {
 329         int i;
 330         uint16_t rx_id;
 331         volatile union ice_rx_flex_desc *rxdp;
 332         struct ice_rx_entry *rxep = &rxq->sw_ring[rxq->rxrearm_start];
 333
 334         rxdp = rxq->rx_ring + rxq->rxrearm_start;
 335
 336         /* Pull 'n' more MBUFs into the software ring */
 337         if (rte_mempool_get_bulk(rxq->mp,
 338                                  (void *)rxep,
 339                                  ICE_RXQ_REARM_THRESH) < 0) {
 340                 if (rxq->rxrearm_nb + ICE_RXQ_REARM_THRESH >=
 341                     rxq->nb_rx_desc) {
 342                         __m128i dma_addr0;
 343
 344                         dma_addr0 = _mm_setzero_si128();
 345                         for (i = 0; i < ICE_DESCS_PER_LOOP; i++) {
 346                                 rxep[i].mbuf = &rxq->fake_mbuf;
 347                                 _mm_store_si128((__m128i *)&rxdp[i].read,
 348                                                 dma_addr0);
 349                         }
 350                 }
 351                 rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed +=
 352                         ICE_RXQ_REARM_THRESH;
 353                 return;
 354         }
 355
 356 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
 357         struct rte_mbuf *mb0, *mb1;
 358         __m128i dma_addr0, dma_addr1;
 359         __m128i hdr_room = _mm_set_epi64x(RTE_PKTMBUF_HEADROOM,
 360                         RTE_PKTMBUF_HEADROOM);
 361         /* Initialize the mbufs in vector, process 2 mbufs in one loop */
 362         for (i = 0; i < ICE_RXQ_REARM_THRESH; i += 2, rxep += 2) {
 363                 __m128i vaddr0, vaddr1;
 364
 365                 mb0 = rxep[0].mbuf;
 366                 mb1 = rxep[1].mbuf;
 367
 368                 /* load buf_addr(lo 64bit) and buf_iova(hi 64bit) */
 369                 RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) !=
 370                                 offsetof(struct rte_mbuf, buf_addr) + 8);
 371                 vaddr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr);
 372                 vaddr1 = _mm_loadu_si128((__m128i *)&mb1->buf_addr);
 373
 374                 /* convert pa to dma_addr hdr/data */
 375                 dma_addr0 = _mm_unpackhi_epi64(vaddr0, vaddr0);
 376                 dma_addr1 = _mm_unpackhi_epi64(vaddr1, vaddr1);
 377
 378                 /* add headroom to pa values */
 379                 dma_addr0 = _mm_add_epi64(dma_addr0, hdr_room);
 380                 dma_addr1 = _mm_add_epi64(dma_addr1, hdr_room);
 381
 382                 /* flush desc with pa dma_addr */
 383                 _mm_store_si128((__m128i *)&rxdp++->read, dma_addr0);
 384                 _mm_store_si128((__m128i *)&rxdp++->read, dma_addr1);
 385         }
 386 #else
 387 #ifdef CC_AVX512_SUPPORT
 388         if (avx512) {
 389                 struct rte_mbuf *mb0, *mb1, *mb2, *mb3;
 390                 struct rte_mbuf *mb4, *mb5, *mb6, *mb7;
 391                 __m512i dma_addr0_3, dma_addr4_7;
 392                 __m512i hdr_room = _mm512_set1_epi64(RTE_PKTMBUF_HEADROOM);
 393                 /* Initialize the mbufs in vector, process 8 mbufs in one loop */
 394                 for (i = 0; i < ICE_RXQ_REARM_THRESH;
 395                                 i += 8, rxep += 8, rxdp += 8) {
 396                         __m128i vaddr0, vaddr1, vaddr2, vaddr3;
 397                         __m128i vaddr4, vaddr5, vaddr6, vaddr7;
 398                         __m256i vaddr0_1, vaddr2_3;
 399                         __m256i vaddr4_5, vaddr6_7;
 400                         __m512i vaddr0_3, vaddr4_7;
 401
 402                         mb0 = rxep[0].mbuf;
 403                         mb1 = rxep[1].mbuf;
 404                         mb2 = rxep[2].mbuf;
 405                         mb3 = rxep[3].mbuf;
 406                         mb4 = rxep[4].mbuf;
 407                         mb5 = rxep[5].mbuf;
 408                         mb6 = rxep[6].mbuf;
 409                         mb7 = rxep[7].mbuf;
 410
 411                         /* load buf_addr(lo 64bit) and buf_iova(hi 64bit) */
 412                         RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) !=
 413                                         offsetof(struct rte_mbuf, buf_addr) + 8);
 414                         vaddr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr);
 415                         vaddr1 = _mm_loadu_si128((__m128i *)&mb1->buf_addr);
 416                         vaddr2 = _mm_loadu_si128((__m128i *)&mb2->buf_addr);
 417                         vaddr3 = _mm_loadu_si128((__m128i *)&mb3->buf_addr);
 418                         vaddr4 = _mm_loadu_si128((__m128i *)&mb4->buf_addr);
 419                         vaddr5 = _mm_loadu_si128((__m128i *)&mb5->buf_addr);
 420                         vaddr6 = _mm_loadu_si128((__m128i *)&mb6->buf_addr);
 421                         vaddr7 = _mm_loadu_si128((__m128i *)&mb7->buf_addr);
 422
 423                         /**
 424                          * merge 0 & 1, by casting 0 to 256-bit and inserting 1
 425                          * into the high lanes. Similarly for 2 & 3, and so on.
 426                          */
 427                         vaddr0_1 =
 428                                 _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr0),
 429                                                         vaddr1, 1);
 430                         vaddr2_3 =
 431                                 _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr2),
 432                                                         vaddr3, 1);
 433                         vaddr4_5 =
 434                                 _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr4),
 435                                                         vaddr5, 1);
 436                         vaddr6_7 =
 437                                 _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr6),
 438                                                         vaddr7, 1);
 439                         vaddr0_3 =
 440                                 _mm512_inserti64x4(_mm512_castsi256_si512(vaddr0_1),
 441                                                         vaddr2_3, 1);
 442                         vaddr4_7 =
 443                                 _mm512_inserti64x4(_mm512_castsi256_si512(vaddr4_5),
 444                                                         vaddr6_7, 1);
 445
 446                         /* convert pa to dma_addr hdr/data */
 447                         dma_addr0_3 = _mm512_unpackhi_epi64(vaddr0_3, vaddr0_3);
 448                         dma_addr4_7 = _mm512_unpackhi_epi64(vaddr4_7, vaddr4_7);
 449
 450                         /* add headroom to pa values */
 451                         dma_addr0_3 = _mm512_add_epi64(dma_addr0_3, hdr_room);
 452                         dma_addr4_7 = _mm512_add_epi64(dma_addr4_7, hdr_room);
 453
 454                         /* flush desc with pa dma_addr */
 455                         _mm512_store_si512((__m512i *)&rxdp->read, dma_addr0_3);
 456                         _mm512_store_si512((__m512i *)&(rxdp + 4)->read, dma_addr4_7);
 457                 }
 458         } else
 459 #endif
 460         {
 461                 struct rte_mbuf *mb0, *mb1, *mb2, *mb3;
 462                 __m256i dma_addr0_1, dma_addr2_3;
 463                 __m256i hdr_room = _mm256_set1_epi64x(RTE_PKTMBUF_HEADROOM);
 464                 /* Initialize the mbufs in vector, process 4 mbufs in one loop */
 465                 for (i = 0; i < ICE_RXQ_REARM_THRESH;
 466                                 i += 4, rxep += 4, rxdp += 4) {
 467                         __m128i vaddr0, vaddr1, vaddr2, vaddr3;
 468                         __m256i vaddr0_1, vaddr2_3;
 469
 470                         mb0 = rxep[0].mbuf;
 471                         mb1 = rxep[1].mbuf;
 472                         mb2 = rxep[2].mbuf;
 473                         mb3 = rxep[3].mbuf;
 474
 475                         /* load buf_addr(lo 64bit) and buf_iova(hi 64bit) */
 476                         RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) !=
 477                                         offsetof(struct rte_mbuf, buf_addr) + 8);
 478                         vaddr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr);
 479                         vaddr1 = _mm_loadu_si128((__m128i *)&mb1->buf_addr);
 480                         vaddr2 = _mm_loadu_si128((__m128i *)&mb2->buf_addr);
 481                         vaddr3 = _mm_loadu_si128((__m128i *)&mb3->buf_addr);
 482
 483                         /**
 484                          * merge 0 & 1, by casting 0 to 256-bit and inserting 1
 485                          * into the high lanes. Similarly for 2 & 3
 486                          */
 487                         vaddr0_1 =
 488                                 _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr0),
 489                                                         vaddr1, 1);
 490                         vaddr2_3 =
 491                                 _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr2),
 492                                                         vaddr3, 1);
 493
 494                         /* convert pa to dma_addr hdr/data */
 495                         dma_addr0_1 = _mm256_unpackhi_epi64(vaddr0_1, vaddr0_1);
 496                         dma_addr2_3 = _mm256_unpackhi_epi64(vaddr2_3, vaddr2_3);
 497
 498                         /* add headroom to pa values */
 499                         dma_addr0_1 = _mm256_add_epi64(dma_addr0_1, hdr_room);
 500                         dma_addr2_3 = _mm256_add_epi64(dma_addr2_3, hdr_room);
 501
 502                         /* flush desc with pa dma_addr */
 503                         _mm256_store_si256((__m256i *)&rxdp->read, dma_addr0_1);
 504                         _mm256_store_si256((__m256i *)&(rxdp + 2)->read, dma_addr2_3);
 505                 }
 506         }
 507
 508 #endif
 509
 510         rxq->rxrearm_start += ICE_RXQ_REARM_THRESH;
 511         if (rxq->rxrearm_start >= rxq->nb_rx_desc)
 512                 rxq->rxrearm_start = 0;
 513
 514         rxq->rxrearm_nb -= ICE_RXQ_REARM_THRESH;
 515
 516         rx_id = (uint16_t)((rxq->rxrearm_start == 0) ?
 517                              (rxq->nb_rx_desc - 1) : (rxq->rxrearm_start - 1));
 518
 519         /* Update the tail pointer on the NIC */
 520         ICE_PCI_REG_WC_WRITE(rxq->qrx_tail, rx_id);
 521 }
 522 #endif
 523
 524 #endif