drivers/net/ice/ice_rxtx_vec_avx2.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019 Intel Corporation
   3  */
   4
   5 #include "ice_rxtx_vec_common.h"
   6
   7 #include <rte_vect.h>
   8
   9 #ifndef __INTEL_COMPILER
  10 #pragma GCC diagnostic ignored "-Wcast-qual"
  11 #endif
  12
  13 static __rte_always_inline void
  14 ice_rxq_rearm(struct ice_rx_queue *rxq)
  15 {
  16         return ice_rxq_rearm_common(rxq, false);
  17 }
  18
  19 static __rte_always_inline __m256i
  20 ice_flex_rxd_to_fdir_flags_vec_avx2(const __m256i fdir_id0_7)
  21 {
  22 #define FDID_MIS_MAGIC 0xFFFFFFFF
  23         RTE_BUILD_BUG_ON(RTE_MBUF_F_RX_FDIR != (1 << 2));
  24         RTE_BUILD_BUG_ON(RTE_MBUF_F_RX_FDIR_ID != (1 << 13));
  25         const __m256i pkt_fdir_bit = _mm256_set1_epi32(RTE_MBUF_F_RX_FDIR |
  26                         RTE_MBUF_F_RX_FDIR_ID);
  27         /* desc->flow_id field == 0xFFFFFFFF means fdir mismatch */
  28         const __m256i fdir_mis_mask = _mm256_set1_epi32(FDID_MIS_MAGIC);
  29         __m256i fdir_mask = _mm256_cmpeq_epi32(fdir_id0_7,
  30                         fdir_mis_mask);
  31         /* this XOR op results to bit-reverse the fdir_mask */
  32         fdir_mask = _mm256_xor_si256(fdir_mask, fdir_mis_mask);
  33         const __m256i fdir_flags = _mm256_and_si256(fdir_mask, pkt_fdir_bit);
  34
  35         return fdir_flags;
  36 }
  37
  38 static __rte_always_inline uint16_t
  39 _ice_recv_raw_pkts_vec_avx2(struct ice_rx_queue *rxq, struct rte_mbuf **rx_pkts,
  40                             uint16_t nb_pkts, uint8_t *split_packet,
  41                             bool offload)
  42 {
  43 #define ICE_DESCS_PER_LOOP_AVX 8
  44
  45         const uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl;
  46         const __m256i mbuf_init = _mm256_set_epi64x(0, 0,
  47                         0, rxq->mbuf_initializer);
  48         struct ice_rx_entry *sw_ring = &rxq->sw_ring[rxq->rx_tail];
  49         volatile union ice_rx_flex_desc *rxdp = rxq->rx_ring + rxq->rx_tail;
  50         const int avx_aligned = ((rxq->rx_tail & 1) == 0);
  51
  52         rte_prefetch0(rxdp);
  53
  54         /* nb_pkts has to be floor-aligned to ICE_DESCS_PER_LOOP_AVX */
  55         nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, ICE_DESCS_PER_LOOP_AVX);
  56
  57         /* See if we need to rearm the RX queue - gives the prefetch a bit
  58          * of time to act
  59          */
  60         if (rxq->rxrearm_nb > ICE_RXQ_REARM_THRESH)
  61                 ice_rxq_rearm(rxq);
  62
  63         /* Before we start moving massive data around, check to see if
  64          * there is actually a packet available
  65          */
  66         if (!(rxdp->wb.status_error0 &
  67                         rte_cpu_to_le_32(1 << ICE_RX_FLEX_DESC_STATUS0_DD_S)))
  68                 return 0;
  69
  70         /* constants used in processing loop */
  71         const __m256i crc_adjust =
  72                 _mm256_set_epi16
  73                         (/* first descriptor */
  74                          0, 0, 0,       /* ignore non-length fields */
  75                          -rxq->crc_len, /* sub crc on data_len */
  76                          0,             /* ignore high-16bits of pkt_len */
  77                          -rxq->crc_len, /* sub crc on pkt_len */
  78                          0, 0,          /* ignore pkt_type field */
  79                          /* second descriptor */
  80                          0, 0, 0,       /* ignore non-length fields */
  81                          -rxq->crc_len, /* sub crc on data_len */
  82                          0,             /* ignore high-16bits of pkt_len */
  83                          -rxq->crc_len, /* sub crc on pkt_len */
  84                          0, 0           /* ignore pkt_type field */
  85                         );
  86
  87         /* 8 packets DD mask, LSB in each 32-bit value */
  88         const __m256i dd_check = _mm256_set1_epi32(1);
  89
  90         /* 8 packets EOP mask, second-LSB in each 32-bit value */
  91         const __m256i eop_check = _mm256_slli_epi32(dd_check,
  92                         ICE_RX_DESC_STATUS_EOF_S);
  93
  94         /* mask to shuffle from desc. to mbuf (2 descriptors)*/
  95         const __m256i shuf_msk =
  96                 _mm256_set_epi8
  97                         (/* first descriptor */
  98                          0xFF, 0xFF,
  99                          0xFF, 0xFF,    /* rss hash parsed separately */
 100                          11, 10,        /* octet 10~11, 16 bits vlan_macip */
 101                          5, 4,          /* octet 4~5, 16 bits data_len */
 102                          0xFF, 0xFF,    /* skip hi 16 bits pkt_len, zero out */
 103                          5, 4,          /* octet 4~5, 16 bits pkt_len */
 104                          0xFF, 0xFF,    /* pkt_type set as unknown */
 105                          0xFF, 0xFF,    /*pkt_type set as unknown */
 106                          /* second descriptor */
 107                          0xFF, 0xFF,
 108                          0xFF, 0xFF,    /* rss hash parsed separately */
 109                          11, 10,        /* octet 10~11, 16 bits vlan_macip */
 110                          5, 4,          /* octet 4~5, 16 bits data_len */
 111                          0xFF, 0xFF,    /* skip hi 16 bits pkt_len, zero out */
 112                          5, 4,          /* octet 4~5, 16 bits pkt_len */
 113                          0xFF, 0xFF,    /* pkt_type set as unknown */
 114                          0xFF, 0xFF     /*pkt_type set as unknown */
 115                         );
 116         /**
 117          * compile-time check the above crc and shuffle layout is correct.
 118          * NOTE: the first field (lowest address) is given last in set_epi
 119          * calls above.
 120          */
 121         RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) !=
 122                         offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4);
 123         RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) !=
 124                         offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8);
 125         RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, vlan_tci) !=
 126                         offsetof(struct rte_mbuf, rx_descriptor_fields1) + 10);
 127         RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, hash) !=
 128                         offsetof(struct rte_mbuf, rx_descriptor_fields1) + 12);
 129
 130         /* Status/Error flag masks */
 131         /**
 132          * mask everything except Checksum Reports, RSS indication
 133          * and VLAN indication.
 134          * bit6:4 for IP/L4 checksum errors.
 135          * bit12 is for RSS indication.
 136          * bit13 is for VLAN indication.
 137          */
 138         const __m256i flags_mask =
 139                  _mm256_set1_epi32((0xF << 4) | (1 << 12) | (1 << 13));
 140         /**
 141          * data to be shuffled by the result of the flags mask shifted by 4
 142          * bits.  This gives use the l3_l4 flags.
 143          */
 144         const __m256i l3_l4_flags_shuf =
 145                 _mm256_set_epi8((RTE_MBUF_F_RX_OUTER_L4_CKSUM_BAD >> 20 |
 146                  RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD |
 147                   RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1,
 148                 (RTE_MBUF_F_RX_OUTER_L4_CKSUM_BAD >> 20 | RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
 149                  RTE_MBUF_F_RX_L4_CKSUM_BAD | RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1,
 150                 (RTE_MBUF_F_RX_OUTER_L4_CKSUM_BAD >> 20 | RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
 151                  RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1,
 152                 (RTE_MBUF_F_RX_OUTER_L4_CKSUM_BAD >> 20 | RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
 153                  RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1,
 154                 (RTE_MBUF_F_RX_OUTER_L4_CKSUM_BAD >> 20 | RTE_MBUF_F_RX_L4_CKSUM_BAD  |
 155                  RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1,
 156                 (RTE_MBUF_F_RX_OUTER_L4_CKSUM_BAD >> 20 | RTE_MBUF_F_RX_L4_CKSUM_BAD  |
 157                  RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1,
 158                 (RTE_MBUF_F_RX_OUTER_L4_CKSUM_BAD >> 20 | RTE_MBUF_F_RX_L4_CKSUM_GOOD |
 159                  RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1,
 160                 (RTE_MBUF_F_RX_OUTER_L4_CKSUM_BAD >> 20 | RTE_MBUF_F_RX_L4_CKSUM_GOOD |
 161                  RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1,
 162                 (RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD >> 20 | RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
 163                  RTE_MBUF_F_RX_L4_CKSUM_BAD | RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1,
 164                 (RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD >> 20 | RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
 165                  RTE_MBUF_F_RX_L4_CKSUM_BAD | RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1,
 166                 (RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD >> 20 | RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
 167                  RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1,
 168                 (RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD >> 20 | RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
 169                  RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1,
 170                 (RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD >> 20 | RTE_MBUF_F_RX_L4_CKSUM_BAD |
 171                  RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1,
 172                 (RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD >> 20 | RTE_MBUF_F_RX_L4_CKSUM_BAD |
 173                  RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1,
 174                 (RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD >> 20 | RTE_MBUF_F_RX_L4_CKSUM_GOOD |
 175                  RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1,
 176                 (RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD >> 20 | RTE_MBUF_F_RX_L4_CKSUM_GOOD |
 177                  RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1,
 178                 /**
 179                  * second 128-bits
 180                  * shift right 20 bits to use the low two bits to indicate
 181                  * outer checksum status
 182                  * shift right 1 bit to make sure it not exceed 255
 183                  */
 184                 (RTE_MBUF_F_RX_OUTER_L4_CKSUM_BAD >> 20 | RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
 185                  RTE_MBUF_F_RX_L4_CKSUM_BAD | RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1,
 186                 (RTE_MBUF_F_RX_OUTER_L4_CKSUM_BAD >> 20 | RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
 187                  RTE_MBUF_F_RX_L4_CKSUM_BAD | RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1,
 188                 (RTE_MBUF_F_RX_OUTER_L4_CKSUM_BAD >> 20 | RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
 189                  RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1,
 190                 (RTE_MBUF_F_RX_OUTER_L4_CKSUM_BAD >> 20 | RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
 191                  RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1,
 192                 (RTE_MBUF_F_RX_OUTER_L4_CKSUM_BAD >> 20 | RTE_MBUF_F_RX_L4_CKSUM_BAD  |
 193                  RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1,
 194                 (RTE_MBUF_F_RX_OUTER_L4_CKSUM_BAD >> 20 | RTE_MBUF_F_RX_L4_CKSUM_BAD  |
 195                  RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1,
 196                 (RTE_MBUF_F_RX_OUTER_L4_CKSUM_BAD >> 20 | RTE_MBUF_F_RX_L4_CKSUM_GOOD |
 197                  RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1,
 198                 (RTE_MBUF_F_RX_OUTER_L4_CKSUM_BAD >> 20 | RTE_MBUF_F_RX_L4_CKSUM_GOOD |
 199                  RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1,
 200                 (RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD >> 20 | RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
 201                  RTE_MBUF_F_RX_L4_CKSUM_BAD | RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1,
 202                 (RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD >> 20 | RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
 203                  RTE_MBUF_F_RX_L4_CKSUM_BAD | RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1,
 204                 (RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD >> 20 | RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
 205                  RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1,
 206                 (RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD >> 20 | RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
 207                  RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1,
 208                 (RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD >> 20 | RTE_MBUF_F_RX_L4_CKSUM_BAD |
 209                  RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1,
 210                 (RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD >> 20 | RTE_MBUF_F_RX_L4_CKSUM_BAD |
 211                  RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1,
 212                 (RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD >> 20 | RTE_MBUF_F_RX_L4_CKSUM_GOOD |
 213                  RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1,
 214                 (RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD >> 20 | RTE_MBUF_F_RX_L4_CKSUM_GOOD |
 215                  RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1);
 216         const __m256i cksum_mask =
 217                  _mm256_set1_epi32(RTE_MBUF_F_RX_IP_CKSUM_MASK |
 218                                    RTE_MBUF_F_RX_L4_CKSUM_MASK |
 219                                    RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
 220                                    RTE_MBUF_F_RX_OUTER_L4_CKSUM_MASK);
 221         /**
 222          * data to be shuffled by result of flag mask, shifted down 12.
 223          * If RSS(bit12)/VLAN(bit13) are set,
 224          * shuffle moves appropriate flags in place.
 225          */
 226         const __m256i rss_vlan_flags_shuf = _mm256_set_epi8(0, 0, 0, 0,
 227                         0, 0, 0, 0,
 228                         0, 0, 0, 0,
 229                         RTE_MBUF_F_RX_RSS_HASH | RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED,
 230                         RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED,
 231                         RTE_MBUF_F_RX_RSS_HASH, 0,
 232                         /* end up 128-bits */
 233                         0, 0, 0, 0,
 234                         0, 0, 0, 0,
 235                         0, 0, 0, 0,
 236                         RTE_MBUF_F_RX_RSS_HASH | RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED,
 237                         RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED,
 238                         RTE_MBUF_F_RX_RSS_HASH, 0);
 239
 240         RTE_SET_USED(avx_aligned); /* for 32B descriptors we don't use this */
 241
 242         uint16_t i, received;
 243
 244         for (i = 0, received = 0; i < nb_pkts;
 245              i += ICE_DESCS_PER_LOOP_AVX,
 246              rxdp += ICE_DESCS_PER_LOOP_AVX) {
 247                 /* step 1, copy over 8 mbuf pointers to rx_pkts array */
 248                 _mm256_storeu_si256((void *)&rx_pkts[i],
 249                                     _mm256_loadu_si256((void *)&sw_ring[i]));
 250 #ifdef RTE_ARCH_X86_64
 251                 _mm256_storeu_si256
 252                         ((void *)&rx_pkts[i + 4],
 253                          _mm256_loadu_si256((void *)&sw_ring[i + 4]));
 254 #endif
 255
 256                 __m256i raw_desc0_1, raw_desc2_3, raw_desc4_5, raw_desc6_7;
 257 #ifdef RTE_LIBRTE_ICE_16BYTE_RX_DESC
 258                 /* for AVX we need alignment otherwise loads are not atomic */
 259                 if (avx_aligned) {
 260                         /* load in descriptors, 2 at a time, in reverse order */
 261                         raw_desc6_7 = _mm256_load_si256((void *)(rxdp + 6));
 262                         rte_compiler_barrier();
 263                         raw_desc4_5 = _mm256_load_si256((void *)(rxdp + 4));
 264                         rte_compiler_barrier();
 265                         raw_desc2_3 = _mm256_load_si256((void *)(rxdp + 2));
 266                         rte_compiler_barrier();
 267                         raw_desc0_1 = _mm256_load_si256((void *)(rxdp + 0));
 268                 } else
 269 #endif
 270                 {
 271                         const __m128i raw_desc7 =
 272                                 _mm_load_si128((void *)(rxdp + 7));
 273                         rte_compiler_barrier();
 274                         const __m128i raw_desc6 =
 275                                 _mm_load_si128((void *)(rxdp + 6));
 276                         rte_compiler_barrier();
 277                         const __m128i raw_desc5 =
 278                                 _mm_load_si128((void *)(rxdp + 5));
 279                         rte_compiler_barrier();
 280                         const __m128i raw_desc4 =
 281                                 _mm_load_si128((void *)(rxdp + 4));
 282                         rte_compiler_barrier();
 283                         const __m128i raw_desc3 =
 284                                 _mm_load_si128((void *)(rxdp + 3));
 285                         rte_compiler_barrier();
 286                         const __m128i raw_desc2 =
 287                                 _mm_load_si128((void *)(rxdp + 2));
 288                         rte_compiler_barrier();
 289                         const __m128i raw_desc1 =
 290                                 _mm_load_si128((void *)(rxdp + 1));
 291                         rte_compiler_barrier();
 292                         const __m128i raw_desc0 =
 293                                 _mm_load_si128((void *)(rxdp + 0));
 294
 295                         raw_desc6_7 =
 296                                 _mm256_inserti128_si256
 297                                         (_mm256_castsi128_si256(raw_desc6),
 298                                          raw_desc7, 1);
 299                         raw_desc4_5 =
 300                                 _mm256_inserti128_si256
 301                                         (_mm256_castsi128_si256(raw_desc4),
 302                                          raw_desc5, 1);
 303                         raw_desc2_3 =
 304                                 _mm256_inserti128_si256
 305                                         (_mm256_castsi128_si256(raw_desc2),
 306                                          raw_desc3, 1);
 307                         raw_desc0_1 =
 308                                 _mm256_inserti128_si256
 309                                         (_mm256_castsi128_si256(raw_desc0),
 310                                          raw_desc1, 1);
 311                 }
 312
 313                 if (split_packet) {
 314                         int j;
 315
 316                         for (j = 0; j < ICE_DESCS_PER_LOOP_AVX; j++)
 317                                 rte_mbuf_prefetch_part2(rx_pkts[i + j]);
 318                 }
 319
 320                 /**
 321                  * convert descriptors 4-7 into mbufs, re-arrange fields.
 322                  * Then write into the mbuf.
 323                  */
 324                 __m256i mb6_7 = _mm256_shuffle_epi8(raw_desc6_7, shuf_msk);
 325                 __m256i mb4_5 = _mm256_shuffle_epi8(raw_desc4_5, shuf_msk);
 326
 327                 mb6_7 = _mm256_add_epi16(mb6_7, crc_adjust);
 328                 mb4_5 = _mm256_add_epi16(mb4_5, crc_adjust);
 329                 /**
 330                  * to get packet types, ptype is located in bit16-25
 331                  * of each 128bits
 332                  */
 333                 const __m256i ptype_mask =
 334                         _mm256_set1_epi16(ICE_RX_FLEX_DESC_PTYPE_M);
 335                 const __m256i ptypes6_7 =
 336                         _mm256_and_si256(raw_desc6_7, ptype_mask);
 337                 const __m256i ptypes4_5 =
 338                         _mm256_and_si256(raw_desc4_5, ptype_mask);
 339                 const uint16_t ptype7 = _mm256_extract_epi16(ptypes6_7, 9);
 340                 const uint16_t ptype6 = _mm256_extract_epi16(ptypes6_7, 1);
 341                 const uint16_t ptype5 = _mm256_extract_epi16(ptypes4_5, 9);
 342                 const uint16_t ptype4 = _mm256_extract_epi16(ptypes4_5, 1);
 343
 344                 mb6_7 = _mm256_insert_epi32(mb6_7, ptype_tbl[ptype7], 4);
 345                 mb6_7 = _mm256_insert_epi32(mb6_7, ptype_tbl[ptype6], 0);
 346                 mb4_5 = _mm256_insert_epi32(mb4_5, ptype_tbl[ptype5], 4);
 347                 mb4_5 = _mm256_insert_epi32(mb4_5, ptype_tbl[ptype4], 0);
 348                 /* merge the status bits into one register */
 349                 const __m256i status4_7 = _mm256_unpackhi_epi32(raw_desc6_7,
 350                                 raw_desc4_5);
 351
 352                 /**
 353                  * convert descriptors 0-3 into mbufs, re-arrange fields.
 354                  * Then write into the mbuf.
 355                  */
 356                 __m256i mb2_3 = _mm256_shuffle_epi8(raw_desc2_3, shuf_msk);
 357                 __m256i mb0_1 = _mm256_shuffle_epi8(raw_desc0_1, shuf_msk);
 358
 359                 mb2_3 = _mm256_add_epi16(mb2_3, crc_adjust);
 360                 mb0_1 = _mm256_add_epi16(mb0_1, crc_adjust);
 361                 /**
 362                  * to get packet types, ptype is located in bit16-25
 363                  * of each 128bits
 364                  */
 365                 const __m256i ptypes2_3 =
 366                         _mm256_and_si256(raw_desc2_3, ptype_mask);
 367                 const __m256i ptypes0_1 =
 368                         _mm256_and_si256(raw_desc0_1, ptype_mask);
 369                 const uint16_t ptype3 = _mm256_extract_epi16(ptypes2_3, 9);
 370                 const uint16_t ptype2 = _mm256_extract_epi16(ptypes2_3, 1);
 371                 const uint16_t ptype1 = _mm256_extract_epi16(ptypes0_1, 9);
 372                 const uint16_t ptype0 = _mm256_extract_epi16(ptypes0_1, 1);
 373
 374                 mb2_3 = _mm256_insert_epi32(mb2_3, ptype_tbl[ptype3], 4);
 375                 mb2_3 = _mm256_insert_epi32(mb2_3, ptype_tbl[ptype2], 0);
 376                 mb0_1 = _mm256_insert_epi32(mb0_1, ptype_tbl[ptype1], 4);
 377                 mb0_1 = _mm256_insert_epi32(mb0_1, ptype_tbl[ptype0], 0);
 378                 /* merge the status bits into one register */
 379                 const __m256i status0_3 = _mm256_unpackhi_epi32(raw_desc2_3,
 380                                                                 raw_desc0_1);
 381
 382                 /**
 383                  * take the two sets of status bits and merge to one
 384                  * After merge, the packets status flags are in the
 385                  * order (hi->lo): [1, 3, 5, 7, 0, 2, 4, 6]
 386                  */
 387                 __m256i status0_7 = _mm256_unpacklo_epi64(status4_7,
 388                                                           status0_3);
 389                 __m256i mbuf_flags = _mm256_set1_epi32(0);
 390
 391                 if (offload) {
 392                         /* now do flag manipulation */
 393
 394                         /* get only flag/error bits we want */
 395                         const __m256i flag_bits =
 396                                 _mm256_and_si256(status0_7, flags_mask);
 397                         /**
 398                          * l3_l4_error flags, shuffle, then shift to correct adjustment
 399                          * of flags in flags_shuf, and finally mask out extra bits
 400                          */
 401                         __m256i l3_l4_flags = _mm256_shuffle_epi8(l3_l4_flags_shuf,
 402                                         _mm256_srli_epi32(flag_bits, 4));
 403                         l3_l4_flags = _mm256_slli_epi32(l3_l4_flags, 1);
 404
 405                         __m256i l4_outer_mask = _mm256_set1_epi32(0x6);
 406                         __m256i l4_outer_flags =
 407                                         _mm256_and_si256(l3_l4_flags, l4_outer_mask);
 408                         l4_outer_flags = _mm256_slli_epi32(l4_outer_flags, 20);
 409
 410                         __m256i l3_l4_mask = _mm256_set1_epi32(~0x6);
 411
 412                         l3_l4_flags = _mm256_and_si256(l3_l4_flags, l3_l4_mask);
 413                         l3_l4_flags = _mm256_or_si256(l3_l4_flags, l4_outer_flags);
 414                         l3_l4_flags = _mm256_and_si256(l3_l4_flags, cksum_mask);
 415                         /* set rss and vlan flags */
 416                         const __m256i rss_vlan_flag_bits =
 417                                 _mm256_srli_epi32(flag_bits, 12);
 418                         const __m256i rss_vlan_flags =
 419                                 _mm256_shuffle_epi8(rss_vlan_flags_shuf,
 420                                                     rss_vlan_flag_bits);
 421
 422                         /* merge flags */
 423                         mbuf_flags = _mm256_or_si256(l3_l4_flags,
 424                                                      rss_vlan_flags);
 425                 }
 426
 427                 if (rxq->fdir_enabled) {
 428                         const __m256i fdir_id4_7 =
 429                                 _mm256_unpackhi_epi32(raw_desc6_7, raw_desc4_5);
 430
 431                         const __m256i fdir_id0_3 =
 432                                 _mm256_unpackhi_epi32(raw_desc2_3, raw_desc0_1);
 433
 434                         const __m256i fdir_id0_7 =
 435                                 _mm256_unpackhi_epi64(fdir_id4_7, fdir_id0_3);
 436
 437                         const __m256i fdir_flags =
 438                                 ice_flex_rxd_to_fdir_flags_vec_avx2(fdir_id0_7);
 439
 440                         /* merge with fdir_flags */
 441                         mbuf_flags = _mm256_or_si256(mbuf_flags, fdir_flags);
 442
 443                         /* write to mbuf: have to use scalar store here */
 444                         rx_pkts[i + 0]->hash.fdir.hi =
 445                                 _mm256_extract_epi32(fdir_id0_7, 3);
 446
 447                         rx_pkts[i + 1]->hash.fdir.hi =
 448                                 _mm256_extract_epi32(fdir_id0_7, 7);
 449
 450                         rx_pkts[i + 2]->hash.fdir.hi =
 451                                 _mm256_extract_epi32(fdir_id0_7, 2);
 452
 453                         rx_pkts[i + 3]->hash.fdir.hi =
 454                                 _mm256_extract_epi32(fdir_id0_7, 6);
 455
 456                         rx_pkts[i + 4]->hash.fdir.hi =
 457                                 _mm256_extract_epi32(fdir_id0_7, 1);
 458
 459                         rx_pkts[i + 5]->hash.fdir.hi =
 460                                 _mm256_extract_epi32(fdir_id0_7, 5);
 461
 462                         rx_pkts[i + 6]->hash.fdir.hi =
 463                                 _mm256_extract_epi32(fdir_id0_7, 0);
 464
 465                         rx_pkts[i + 7]->hash.fdir.hi =
 466                                 _mm256_extract_epi32(fdir_id0_7, 4);
 467                 } /* if() on fdir_enabled */
 468
 469                 if (offload) {
 470 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
 471                         /**
 472                          * needs to load 2nd 16B of each desc for RSS hash parsing,
 473                          * will cause performance drop to get into this context.
 474                          */
 475                         if (rxq->vsi->adapter->pf.dev_data->dev_conf.rxmode.offloads &
 476                                         DEV_RX_OFFLOAD_RSS_HASH) {
 477                                 /* load bottom half of every 32B desc */
 478                                 const __m128i raw_desc_bh7 =
 479                                         _mm_load_si128
 480                                                 ((void *)(&rxdp[7].wb.status_error1));
 481                                 rte_compiler_barrier();
 482                                 const __m128i raw_desc_bh6 =
 483                                         _mm_load_si128
 484                                                 ((void *)(&rxdp[6].wb.status_error1));
 485                                 rte_compiler_barrier();
 486                                 const __m128i raw_desc_bh5 =
 487                                         _mm_load_si128
 488                                                 ((void *)(&rxdp[5].wb.status_error1));
 489                                 rte_compiler_barrier();
 490                                 const __m128i raw_desc_bh4 =
 491                                         _mm_load_si128
 492                                                 ((void *)(&rxdp[4].wb.status_error1));
 493                                 rte_compiler_barrier();
 494                                 const __m128i raw_desc_bh3 =
 495                                         _mm_load_si128
 496                                                 ((void *)(&rxdp[3].wb.status_error1));
 497                                 rte_compiler_barrier();
 498                                 const __m128i raw_desc_bh2 =
 499                                         _mm_load_si128
 500                                                 ((void *)(&rxdp[2].wb.status_error1));
 501                                 rte_compiler_barrier();
 502                                 const __m128i raw_desc_bh1 =
 503                                         _mm_load_si128
 504                                                 ((void *)(&rxdp[1].wb.status_error1));
 505                                 rte_compiler_barrier();
 506                                 const __m128i raw_desc_bh0 =
 507                                         _mm_load_si128
 508                                                 ((void *)(&rxdp[0].wb.status_error1));
 509
 510                                 __m256i raw_desc_bh6_7 =
 511                                         _mm256_inserti128_si256
 512                                                 (_mm256_castsi128_si256(raw_desc_bh6),
 513                                                 raw_desc_bh7, 1);
 514                                 __m256i raw_desc_bh4_5 =
 515                                         _mm256_inserti128_si256
 516                                                 (_mm256_castsi128_si256(raw_desc_bh4),
 517                                                 raw_desc_bh5, 1);
 518                                 __m256i raw_desc_bh2_3 =
 519                                         _mm256_inserti128_si256
 520                                                 (_mm256_castsi128_si256(raw_desc_bh2),
 521                                                 raw_desc_bh3, 1);
 522                                 __m256i raw_desc_bh0_1 =
 523                                         _mm256_inserti128_si256
 524                                                 (_mm256_castsi128_si256(raw_desc_bh0),
 525                                                 raw_desc_bh1, 1);
 526
 527                                 /**
 528                                  * to shift the 32b RSS hash value to the
 529                                  * highest 32b of each 128b before mask
 530                                  */
 531                                 __m256i rss_hash6_7 =
 532                                         _mm256_slli_epi64(raw_desc_bh6_7, 32);
 533                                 __m256i rss_hash4_5 =
 534                                         _mm256_slli_epi64(raw_desc_bh4_5, 32);
 535                                 __m256i rss_hash2_3 =
 536                                         _mm256_slli_epi64(raw_desc_bh2_3, 32);
 537                                 __m256i rss_hash0_1 =
 538                                         _mm256_slli_epi64(raw_desc_bh0_1, 32);
 539
 540                                 __m256i rss_hash_msk =
 541                                         _mm256_set_epi32(0xFFFFFFFF, 0, 0, 0,
 542                                                          0xFFFFFFFF, 0, 0, 0);
 543
 544                                 rss_hash6_7 = _mm256_and_si256
 545                                                 (rss_hash6_7, rss_hash_msk);
 546                                 rss_hash4_5 = _mm256_and_si256
 547                                                 (rss_hash4_5, rss_hash_msk);
 548                                 rss_hash2_3 = _mm256_and_si256
 549                                                 (rss_hash2_3, rss_hash_msk);
 550                                 rss_hash0_1 = _mm256_and_si256
 551                                                 (rss_hash0_1, rss_hash_msk);
 552
 553                                 mb6_7 = _mm256_or_si256(mb6_7, rss_hash6_7);
 554                                 mb4_5 = _mm256_or_si256(mb4_5, rss_hash4_5);
 555                                 mb2_3 = _mm256_or_si256(mb2_3, rss_hash2_3);
 556                                 mb0_1 = _mm256_or_si256(mb0_1, rss_hash0_1);
 557                         } /* if() on RSS hash parsing */
 558 #endif
 559                 }
 560
 561                 /**
 562                  * At this point, we have the 8 sets of flags in the low 16-bits
 563                  * of each 32-bit value in vlan0.
 564                  * We want to extract these, and merge them with the mbuf init
 565                  * data so we can do a single write to the mbuf to set the flags
 566                  * and all the other initialization fields. Extracting the
 567                  * appropriate flags means that we have to do a shift and blend
 568                  * for each mbuf before we do the write. However, we can also
 569                  * add in the previously computed rx_descriptor fields to
 570                  * make a single 256-bit write per mbuf
 571                  */
 572                 /* check the structure matches expectations */
 573                 RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, ol_flags) !=
 574                                  offsetof(struct rte_mbuf, rearm_data) + 8);
 575                 RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rearm_data) !=
 576                                  RTE_ALIGN(offsetof(struct rte_mbuf,
 577                                                     rearm_data),
 578                                            16));
 579                 /* build up data and do writes */
 580                 __m256i rearm0, rearm1, rearm2, rearm3, rearm4, rearm5,
 581                         rearm6, rearm7;
 582                 rearm6 = _mm256_blend_epi32(mbuf_init,
 583                                             _mm256_slli_si256(mbuf_flags, 8),
 584                                             0x04);
 585                 rearm4 = _mm256_blend_epi32(mbuf_init,
 586                                             _mm256_slli_si256(mbuf_flags, 4),
 587                                             0x04);
 588                 rearm2 = _mm256_blend_epi32(mbuf_init, mbuf_flags, 0x04);
 589                 rearm0 = _mm256_blend_epi32(mbuf_init,
 590                                             _mm256_srli_si256(mbuf_flags, 4),
 591                                             0x04);
 592                 /* permute to add in the rx_descriptor e.g. rss fields */
 593                 rearm6 = _mm256_permute2f128_si256(rearm6, mb6_7, 0x20);
 594                 rearm4 = _mm256_permute2f128_si256(rearm4, mb4_5, 0x20);
 595                 rearm2 = _mm256_permute2f128_si256(rearm2, mb2_3, 0x20);
 596                 rearm0 = _mm256_permute2f128_si256(rearm0, mb0_1, 0x20);
 597                 /* write to mbuf */
 598                 _mm256_storeu_si256((__m256i *)&rx_pkts[i + 6]->rearm_data,
 599                                     rearm6);
 600                 _mm256_storeu_si256((__m256i *)&rx_pkts[i + 4]->rearm_data,
 601                                     rearm4);
 602                 _mm256_storeu_si256((__m256i *)&rx_pkts[i + 2]->rearm_data,
 603                                     rearm2);
 604                 _mm256_storeu_si256((__m256i *)&rx_pkts[i + 0]->rearm_data,
 605                                     rearm0);
 606
 607                 /* repeat for the odd mbufs */
 608                 const __m256i odd_flags =
 609                         _mm256_castsi128_si256
 610                                 (_mm256_extracti128_si256(mbuf_flags, 1));
 611                 rearm7 = _mm256_blend_epi32(mbuf_init,
 612                                             _mm256_slli_si256(odd_flags, 8),
 613                                             0x04);
 614                 rearm5 = _mm256_blend_epi32(mbuf_init,
 615                                             _mm256_slli_si256(odd_flags, 4),
 616                                             0x04);
 617                 rearm3 = _mm256_blend_epi32(mbuf_init, odd_flags, 0x04);
 618                 rearm1 = _mm256_blend_epi32(mbuf_init,
 619                                             _mm256_srli_si256(odd_flags, 4),
 620                                             0x04);
 621                 /* since odd mbufs are already in hi 128-bits use blend */
 622                 rearm7 = _mm256_blend_epi32(rearm7, mb6_7, 0xF0);
 623                 rearm5 = _mm256_blend_epi32(rearm5, mb4_5, 0xF0);
 624                 rearm3 = _mm256_blend_epi32(rearm3, mb2_3, 0xF0);
 625                 rearm1 = _mm256_blend_epi32(rearm1, mb0_1, 0xF0);
 626                 /* again write to mbufs */
 627                 _mm256_storeu_si256((__m256i *)&rx_pkts[i + 7]->rearm_data,
 628                                     rearm7);
 629                 _mm256_storeu_si256((__m256i *)&rx_pkts[i + 5]->rearm_data,
 630                                     rearm5);
 631                 _mm256_storeu_si256((__m256i *)&rx_pkts[i + 3]->rearm_data,
 632                                     rearm3);
 633                 _mm256_storeu_si256((__m256i *)&rx_pkts[i + 1]->rearm_data,
 634                                     rearm1);
 635
 636                 /* extract and record EOP bit */
 637                 if (split_packet) {
 638                         const __m128i eop_mask =
 639                                 _mm_set1_epi16(1 << ICE_RX_DESC_STATUS_EOF_S);
 640                         const __m256i eop_bits256 = _mm256_and_si256(status0_7,
 641                                                                      eop_check);
 642                         /* pack status bits into a single 128-bit register */
 643                         const __m128i eop_bits =
 644                                 _mm_packus_epi32
 645                                         (_mm256_castsi256_si128(eop_bits256),
 646                                          _mm256_extractf128_si256(eop_bits256,
 647                                                                   1));
 648                         /**
 649                          * flip bits, and mask out the EOP bit, which is now
 650                          * a split-packet bit i.e. !EOP, rather than EOP one.
 651                          */
 652                         __m128i split_bits = _mm_andnot_si128(eop_bits,
 653                                         eop_mask);
 654                         /**
 655                          * eop bits are out of order, so we need to shuffle them
 656                          * back into order again. In doing so, only use low 8
 657                          * bits, which acts like another pack instruction
 658                          * The original order is (hi->lo): 1,3,5,7,0,2,4,6
 659                          * [Since we use epi8, the 16-bit positions are
 660                          * multiplied by 2 in the eop_shuffle value.]
 661                          */
 662                         __m128i eop_shuffle =
 663                                 _mm_set_epi8(/* zero hi 64b */
 664                                              0xFF, 0xFF, 0xFF, 0xFF,
 665                                              0xFF, 0xFF, 0xFF, 0xFF,
 666                                              /* move values to lo 64b */
 667                                              8, 0, 10, 2,
 668                                              12, 4, 14, 6);
 669                         split_bits = _mm_shuffle_epi8(split_bits, eop_shuffle);
 670                         *(uint64_t *)split_packet =
 671                                 _mm_cvtsi128_si64(split_bits);
 672                         split_packet += ICE_DESCS_PER_LOOP_AVX;
 673                 }
 674
 675                 /* perform dd_check */
 676                 status0_7 = _mm256_and_si256(status0_7, dd_check);
 677                 status0_7 = _mm256_packs_epi32(status0_7,
 678                                                _mm256_setzero_si256());
 679
 680                 uint64_t burst = __builtin_popcountll
 681                                         (_mm_cvtsi128_si64
 682                                                 (_mm256_extracti128_si256
 683                                                         (status0_7, 1)));
 684                 burst += __builtin_popcountll
 685                                 (_mm_cvtsi128_si64
 686                                         (_mm256_castsi256_si128(status0_7)));
 687                 received += burst;
 688                 if (burst != ICE_DESCS_PER_LOOP_AVX)
 689                         break;
 690         }
 691
 692         /* update tail pointers */
 693         rxq->rx_tail += received;
 694         rxq->rx_tail &= (rxq->nb_rx_desc - 1);
 695         if ((rxq->rx_tail & 1) == 1 && received > 1) { /* keep avx2 aligned */
 696                 rxq->rx_tail--;
 697                 received--;
 698         }
 699         rxq->rxrearm_nb += received;
 700         return received;
 701 }
 702
 703 /**
 704  * Notice:
 705  * - nb_pkts < ICE_DESCS_PER_LOOP, just return no packet
 706  */
 707 uint16_t
 708 ice_recv_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,
 709                        uint16_t nb_pkts)
 710 {
 711         return _ice_recv_raw_pkts_vec_avx2(rx_queue, rx_pkts,
 712                                            nb_pkts, NULL, false);
 713 }
 714
 715 uint16_t
 716 ice_recv_pkts_vec_avx2_offload(void *rx_queue, struct rte_mbuf **rx_pkts,
 717                                uint16_t nb_pkts)
 718 {
 719         return _ice_recv_raw_pkts_vec_avx2(rx_queue, rx_pkts,
 720                                            nb_pkts, NULL, true);
 721 }
 722
 723 /**
 724  * vPMD receive routine that reassembles single burst of 32 scattered packets
 725  * Notice:
 726  * - nb_pkts < ICE_DESCS_PER_LOOP, just return no packet
 727  */
 728 static __rte_always_inline uint16_t
 729 ice_recv_scattered_burst_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,
 730                                   uint16_t nb_pkts, bool offload)
 731 {
 732         struct ice_rx_queue *rxq = rx_queue;
 733         uint8_t split_flags[ICE_VPMD_RX_BURST] = {0};
 734
 735         /* get some new buffers */
 736         uint16_t nb_bufs = _ice_recv_raw_pkts_vec_avx2(rxq, rx_pkts, nb_pkts,
 737                                                        split_flags, offload);
 738         if (nb_bufs == 0)
 739                 return 0;
 740
 741         /* happy day case, full burst + no packets to be joined */
 742         const uint64_t *split_fl64 = (uint64_t *)split_flags;
 743
 744         if (!rxq->pkt_first_seg &&
 745             split_fl64[0] == 0 && split_fl64[1] == 0 &&
 746             split_fl64[2] == 0 && split_fl64[3] == 0)
 747                 return nb_bufs;
 748
 749         /* reassemble any packets that need reassembly*/
 750         unsigned int i = 0;
 751
 752         if (!rxq->pkt_first_seg) {
 753                 /* find the first split flag, and only reassemble then*/
 754                 while (i < nb_bufs && !split_flags[i])
 755                         i++;
 756                 if (i == nb_bufs)
 757                         return nb_bufs;
 758                 rxq->pkt_first_seg = rx_pkts[i];
 759         }
 760         return i + ice_rx_reassemble_packets(rxq, &rx_pkts[i], nb_bufs - i,
 761                                              &split_flags[i]);
 762 }
 763
 764 /**
 765  * vPMD receive routine that reassembles scattered packets.
 766  * Main receive routine that can handle arbitrary burst sizes
 767  * Notice:
 768  * - nb_pkts < ICE_DESCS_PER_LOOP, just return no packet
 769  */
 770 static __rte_always_inline uint16_t
 771 ice_recv_scattered_pkts_vec_avx2_common(void *rx_queue,
 772                                         struct rte_mbuf **rx_pkts,
 773                                         uint16_t nb_pkts,
 774                                         bool offload)
 775 {
 776         uint16_t retval = 0;
 777
 778         while (nb_pkts > ICE_VPMD_RX_BURST) {
 779                 uint16_t burst = ice_recv_scattered_burst_vec_avx2(rx_queue,
 780                                 rx_pkts + retval, ICE_VPMD_RX_BURST, offload);
 781                 retval += burst;
 782                 nb_pkts -= burst;
 783                 if (burst < ICE_VPMD_RX_BURST)
 784                         return retval;
 785         }
 786         return retval + ice_recv_scattered_burst_vec_avx2(rx_queue,
 787                                 rx_pkts + retval, nb_pkts, offload);
 788 }
 789
 790 uint16_t
 791 ice_recv_scattered_pkts_vec_avx2(void *rx_queue,
 792                                  struct rte_mbuf **rx_pkts,
 793                                  uint16_t nb_pkts)
 794 {
 795         return ice_recv_scattered_pkts_vec_avx2_common(rx_queue,
 796                                                        rx_pkts,
 797                                                        nb_pkts,
 798                                                        false);
 799 }
 800
 801 uint16_t
 802 ice_recv_scattered_pkts_vec_avx2_offload(void *rx_queue,
 803                                          struct rte_mbuf **rx_pkts,
 804                                          uint16_t nb_pkts)
 805 {
 806         return ice_recv_scattered_pkts_vec_avx2_common(rx_queue,
 807                                                        rx_pkts,
 808                                                        nb_pkts,
 809                                                        true);
 810 }
 811
 812 static __rte_always_inline void
 813 ice_vtx1(volatile struct ice_tx_desc *txdp,
 814          struct rte_mbuf *pkt, uint64_t flags, bool offload)
 815 {
 816         uint64_t high_qw =
 817                 (ICE_TX_DESC_DTYPE_DATA |
 818                  ((uint64_t)flags  << ICE_TXD_QW1_CMD_S) |
 819                  ((uint64_t)pkt->data_len << ICE_TXD_QW1_TX_BUF_SZ_S));
 820         if (offload)
 821                 ice_txd_enable_offload(pkt, &high_qw);
 822
 823         __m128i descriptor = _mm_set_epi64x(high_qw,
 824                                 pkt->buf_iova + pkt->data_off);
 825         _mm_store_si128((__m128i *)txdp, descriptor);
 826 }
 827
 828 static __rte_always_inline void
 829 ice_vtx(volatile struct ice_tx_desc *txdp,
 830         struct rte_mbuf **pkt, uint16_t nb_pkts,  uint64_t flags, bool offload)
 831 {
 832         const uint64_t hi_qw_tmpl = (ICE_TX_DESC_DTYPE_DATA |
 833                         ((uint64_t)flags  << ICE_TXD_QW1_CMD_S));
 834
 835         /* if unaligned on 32-bit boundary, do one to align */
 836         if (((uintptr_t)txdp & 0x1F) != 0 && nb_pkts != 0) {
 837                 ice_vtx1(txdp, *pkt, flags, offload);
 838                 nb_pkts--, txdp++, pkt++;
 839         }
 840
 841         /* do two at a time while possible, in bursts */
 842         for (; nb_pkts > 3; txdp += 4, pkt += 4, nb_pkts -= 4) {
 843                 uint64_t hi_qw3 =
 844                         hi_qw_tmpl |
 845                         ((uint64_t)pkt[3]->data_len <<
 846                          ICE_TXD_QW1_TX_BUF_SZ_S);
 847                 if (offload)
 848                         ice_txd_enable_offload(pkt[3], &hi_qw3);
 849                 uint64_t hi_qw2 =
 850                         hi_qw_tmpl |
 851                         ((uint64_t)pkt[2]->data_len <<
 852                          ICE_TXD_QW1_TX_BUF_SZ_S);
 853                 if (offload)
 854                         ice_txd_enable_offload(pkt[2], &hi_qw2);
 855                 uint64_t hi_qw1 =
 856                         hi_qw_tmpl |
 857                         ((uint64_t)pkt[1]->data_len <<
 858                          ICE_TXD_QW1_TX_BUF_SZ_S);
 859                 if (offload)
 860                         ice_txd_enable_offload(pkt[1], &hi_qw1);
 861                 uint64_t hi_qw0 =
 862                         hi_qw_tmpl |
 863                         ((uint64_t)pkt[0]->data_len <<
 864                          ICE_TXD_QW1_TX_BUF_SZ_S);
 865                 if (offload)
 866                         ice_txd_enable_offload(pkt[0], &hi_qw0);
 867
 868                 __m256i desc2_3 =
 869                         _mm256_set_epi64x
 870                                 (hi_qw3,
 871                                  pkt[3]->buf_iova + pkt[3]->data_off,
 872                                  hi_qw2,
 873                                  pkt[2]->buf_iova + pkt[2]->data_off);
 874                 __m256i desc0_1 =
 875                         _mm256_set_epi64x
 876                                 (hi_qw1,
 877                                  pkt[1]->buf_iova + pkt[1]->data_off,
 878                                  hi_qw0,
 879                                  pkt[0]->buf_iova + pkt[0]->data_off);
 880                 _mm256_store_si256((void *)(txdp + 2), desc2_3);
 881                 _mm256_store_si256((void *)txdp, desc0_1);
 882         }
 883
 884         /* do any last ones */
 885         while (nb_pkts) {
 886                 ice_vtx1(txdp, *pkt, flags, offload);
 887                 txdp++, pkt++, nb_pkts--;
 888         }
 889 }
 890
 891 static __rte_always_inline uint16_t
 892 ice_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,
 893                               uint16_t nb_pkts, bool offload)
 894 {
 895         struct ice_tx_queue *txq = (struct ice_tx_queue *)tx_queue;
 896         volatile struct ice_tx_desc *txdp;
 897         struct ice_tx_entry *txep;
 898         uint16_t n, nb_commit, tx_id;
 899         uint64_t flags = ICE_TD_CMD;
 900         uint64_t rs = ICE_TX_DESC_CMD_RS | ICE_TD_CMD;
 901
 902         /* cross rx_thresh boundary is not allowed */
 903         nb_pkts = RTE_MIN(nb_pkts, txq->tx_rs_thresh);
 904
 905         if (txq->nb_tx_free < txq->tx_free_thresh)
 906                 ice_tx_free_bufs_vec(txq);
 907
 908         nb_commit = nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts);
 909         if (unlikely(nb_pkts == 0))
 910                 return 0;
 911
 912         tx_id = txq->tx_tail;
 913         txdp = &txq->tx_ring[tx_id];
 914         txep = &txq->sw_ring[tx_id];
 915
 916         txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts);
 917
 918         n = (uint16_t)(txq->nb_tx_desc - tx_id);
 919         if (nb_commit >= n) {
 920                 ice_tx_backlog_entry(txep, tx_pkts, n);
 921
 922                 ice_vtx(txdp, tx_pkts, n - 1, flags, offload);
 923                 tx_pkts += (n - 1);
 924                 txdp += (n - 1);
 925
 926                 ice_vtx1(txdp, *tx_pkts++, rs, offload);
 927
 928                 nb_commit = (uint16_t)(nb_commit - n);
 929
 930                 tx_id = 0;
 931                 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
 932
 933                 /* avoid reach the end of ring */
 934                 txdp = &txq->tx_ring[tx_id];
 935                 txep = &txq->sw_ring[tx_id];
 936         }
 937
 938         ice_tx_backlog_entry(txep, tx_pkts, nb_commit);
 939
 940         ice_vtx(txdp, tx_pkts, nb_commit, flags, offload);
 941
 942         tx_id = (uint16_t)(tx_id + nb_commit);
 943         if (tx_id > txq->tx_next_rs) {
 944                 txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |=
 945                         rte_cpu_to_le_64(((uint64_t)ICE_TX_DESC_CMD_RS) <<
 946                                          ICE_TXD_QW1_CMD_S);
 947                 txq->tx_next_rs =
 948                         (uint16_t)(txq->tx_next_rs + txq->tx_rs_thresh);
 949         }
 950
 951         txq->tx_tail = tx_id;
 952
 953         ICE_PCI_REG_WC_WRITE(txq->qtx_tail, txq->tx_tail);
 954
 955         return nb_pkts;
 956 }
 957
 958 static __rte_always_inline uint16_t
 959 ice_xmit_pkts_vec_avx2_common(void *tx_queue, struct rte_mbuf **tx_pkts,
 960                               uint16_t nb_pkts, bool offload)
 961 {
 962         uint16_t nb_tx = 0;
 963         struct ice_tx_queue *txq = (struct ice_tx_queue *)tx_queue;
 964
 965         while (nb_pkts) {
 966                 uint16_t ret, num;
 967
 968                 num = (uint16_t)RTE_MIN(nb_pkts, txq->tx_rs_thresh);
 969                 ret = ice_xmit_fixed_burst_vec_avx2(tx_queue, &tx_pkts[nb_tx],
 970                                                     num, offload);
 971                 nb_tx += ret;
 972                 nb_pkts -= ret;
 973                 if (ret < num)
 974                         break;
 975         }
 976
 977         return nb_tx;
 978 }
 979
 980 uint16_t
 981 ice_xmit_pkts_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,
 982                        uint16_t nb_pkts)
 983 {
 984         return ice_xmit_pkts_vec_avx2_common(tx_queue, tx_pkts, nb_pkts, false);
 985 }
 986
 987 uint16_t
 988 ice_xmit_pkts_vec_avx2_offload(void *tx_queue, struct rte_mbuf **tx_pkts,
 989                                uint16_t nb_pkts)
 990 {
 991         return ice_xmit_pkts_vec_avx2_common(tx_queue, tx_pkts, nb_pkts, true);
 992 }