drivers/net/mlx5/mlx5_rxtx_vec_altivec.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2017 6WIND S.A.
   3  * Copyright 2017 Mellanox Technologies, Ltd
   4  */
   5
   6 #ifndef RTE_PMD_MLX5_RXTX_VEC_ALTIVEC_H_
   7 #define RTE_PMD_MLX5_RXTX_VEC_ALTIVEC_H_
   8
   9 #include <stdint.h>
  10 #include <string.h>
  11 #include <stdlib.h>
  12
  13 #include <rte_altivec.h>
  14
  15 #include <rte_mbuf.h>
  16 #include <rte_mempool.h>
  17 #include <rte_prefetch.h>
  18
  19 #include <mlx5_prm.h>
  20
  21 #include "mlx5_defs.h"
  22 #include "mlx5.h"
  23 #include "mlx5_utils.h"
  24 #include "mlx5_rxtx.h"
  25 #include "mlx5_rxtx_vec.h"
  26 #include "mlx5_autoconf.h"
  27
  28 #ifndef __INTEL_COMPILER
  29 #pragma GCC diagnostic ignored "-Wcast-qual"
  30 #pragma GCC diagnostic ignored "-Wstrict-aliasing"
  31 #endif
  32
  33 /**
  34  * Store free buffers to RX SW ring.
  35  *
  36  * @param elts
  37  *   Pointer to SW ring to be filled.
  38  * @param pkts
  39  *   Pointer to array of packets to be stored.
  40  * @param pkts_n
  41  *   Number of packets to be stored.
  42  */
  43 static inline void
  44 rxq_copy_mbuf_v(struct rte_mbuf **elts, struct rte_mbuf **pkts, uint16_t n)
  45 {
  46         unsigned int pos;
  47         uint16_t p = n & -2;
  48
  49         for (pos = 0; pos < p; pos += 2) {
  50                 vector unsigned char mbp;
  51
  52                 mbp = (vector unsigned char)vec_vsx_ld(0,
  53                                 (signed int const *)&elts[pos]);
  54                 *(vector unsigned char *)&pkts[pos] = mbp;
  55         }
  56         if (n & 1)
  57                 pkts[pos] = elts[pos];
  58 }
  59
  60 /**
  61  * Decompress a compressed completion and fill in mbufs in RX SW ring with data
  62  * extracted from the title completion descriptor.
  63  *
  64  * @param rxq
  65  *   Pointer to RX queue structure.
  66  * @param cq
  67  *   Pointer to completion array having a compressed completion at first.
  68  * @param elts
  69  *   Pointer to SW ring to be filled. The first mbuf has to be pre-built from
  70  *   the title completion descriptor to be copied to the rest of mbufs.
  71  *
  72  * @return
  73  *   Number of mini-CQEs successfully decompressed.
  74  */
  75 static inline uint16_t
  76 rxq_cq_decompress_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq,
  77                     struct rte_mbuf **elts)
  78 {
  79         volatile struct mlx5_mini_cqe8 *mcq = (void *)&(cq + 1)->pkt_info;
  80         struct rte_mbuf *t_pkt = elts[0]; /* Title packet is pre-built. */
  81         const vector unsigned char zero = (vector unsigned char){0};
  82         /* Mask to shuffle from extracted mini CQE to mbuf. */
  83         const vector unsigned char shuf_mask1 = (vector unsigned char){
  84                         -1, -1, -1, -1,   /* skip packet_type */
  85                          7,  6, -1, -1,   /* bswap16, pkt_len */
  86                          7,  6,           /* bswap16, data_len */
  87                         -1, -1,           /* skip vlan_tci */
  88                          3,  2,  1,  0};  /* bswap32, rss */
  89         const vector unsigned char shuf_mask2 = (vector unsigned char){
  90                         -1, -1, -1, -1,   /* skip packet_type */
  91                         15, 14, -1, -1,   /* bswap16, pkt_len */
  92                         15, 14,           /* data_len, bswap16 */
  93                         -1, -1,           /* skip vlan_tci */
  94                         11, 10,  9,  8};  /* bswap32, rss */
  95         /* Restore the compressed count. Must be 16 bits. */
  96         const uint16_t mcqe_n = t_pkt->data_len +
  97                 (rxq->crc_present * RTE_ETHER_CRC_LEN);
  98         const vector unsigned char rearm =
  99                 (vector unsigned char)vec_vsx_ld(0,
 100                 (signed int const *)&t_pkt->rearm_data);
 101         const vector unsigned char rxdf =
 102                 (vector unsigned char)vec_vsx_ld(0,
 103                 (signed int const *)&t_pkt->rx_descriptor_fields1);
 104         const vector unsigned char crc_adj =
 105                 (vector unsigned char)(vector unsigned short){
 106                         0, 0, rxq->crc_present * RTE_ETHER_CRC_LEN, 0,
 107                         rxq->crc_present * RTE_ETHER_CRC_LEN, 0, 0, 0};
 108         const vector unsigned short rxdf_sel_mask =
 109                 (vector unsigned short){
 110                         0xffff, 0xffff, 0, 0, 0, 0xffff, 0, 0};
 111         const uint32_t flow_tag = t_pkt->hash.fdir.hi;
 112         unsigned int pos;
 113         unsigned int i;
 114         unsigned int inv = 0;
 115
 116 #ifdef MLX5_PMD_SOFT_COUNTERS
 117         const vector unsigned char ones = vec_splat_u8(-1);
 118         uint32_t rcvd_byte = 0;
 119         /* Mask to shuffle byte_cnt to add up stats. Do bswap16 for all. */
 120         const vector unsigned char len_shuf_mask = (vector unsigned char){
 121                  3,  2, 11, 10,
 122                  7,  6, 15, 14,
 123                 -1, -1, -1, -1,
 124                 -1, -1, -1, -1};
 125 #endif
 126
 127         /*
 128          * A. load mCQEs into a 128bit register.
 129          * B. store rearm data to mbuf.
 130          * C. combine data from mCQEs with rx_descriptor_fields1.
 131          * D. store rx_descriptor_fields1.
 132          * E. store flow tag (rte_flow mark).
 133          */
 134         for (pos = 0; pos < mcqe_n; ) {
 135                 vector unsigned char mcqe1, mcqe2;
 136                 vector unsigned char rxdf1, rxdf2;
 137 #ifdef MLX5_PMD_SOFT_COUNTERS
 138                 const vector unsigned short mcqe_sel_mask =
 139                         (vector unsigned short){0, 0, 0xffff, 0xffff,
 140                         0, 0, 0xfff, 0xffff};
 141                 const vector unsigned char lower_half = {
 142                         0, 1, 4, 5, 8, 9, 12, 13, 16,
 143                         17, 20, 21, 24, 25, 28, 29};
 144                 const vector unsigned char upper_half = {
 145                         2, 3, 6, 7, 10, 11, 14, 15,
 146                         18, 19, 22, 23, 26, 27, 30, 31};
 147                 vector unsigned short left, right;
 148                 vector unsigned char byte_cnt, invalid_mask;
 149                 vector unsigned long lshift;
 150                 __attribute__((altivec(vector__)))
 151                         __attribute__((altivec(bool__)))
 152                         unsigned long long shmask;
 153                 const vector unsigned long shmax = {64, 64};
 154 #endif
 155
 156                 for (i = 0; i < MLX5_VPMD_DESCS_PER_LOOP; ++i)
 157                         if (likely(pos + i < mcqe_n))
 158                                 rte_prefetch0((void *)(cq + pos + i));
 159                 /* A.1 load mCQEs into a 128bit register. */
 160                 mcqe1 = (vector unsigned char)vec_vsx_ld(0,
 161                         (signed int const *)&mcq[pos % 8]);
 162                 mcqe2 = (vector unsigned char)vec_vsx_ld(0,
 163                         (signed int const *)&mcq[pos % 8 + 2]);
 164
 165                 /* B.1 store rearm data to mbuf. */
 166                 *(vector unsigned char *)
 167                         &elts[pos]->rearm_data = rearm;
 168                 *(vector unsigned char *)
 169                         &elts[pos + 1]->rearm_data = rearm;
 170
 171                 /* C.1 combine data from mCQEs with rx_descriptor_fields1. */
 172                 rxdf1 = vec_perm(mcqe1, zero, shuf_mask1);
 173                 rxdf2 = vec_perm(mcqe1, zero, shuf_mask2);
 174                 rxdf1 = (vector unsigned char)
 175                         ((vector unsigned short)rxdf1 -
 176                         (vector unsigned short)crc_adj);
 177                 rxdf2 = (vector unsigned char)
 178                         ((vector unsigned short)rxdf2 -
 179                         (vector unsigned short)crc_adj);
 180                 rxdf1 = (vector unsigned char)
 181                         vec_sel((vector unsigned short)rxdf1,
 182                         (vector unsigned short)rxdf, rxdf_sel_mask);
 183                 rxdf2 = (vector unsigned char)
 184                         vec_sel((vector unsigned short)rxdf2,
 185                         (vector unsigned short)rxdf, rxdf_sel_mask);
 186
 187                 /* D.1 store rx_descriptor_fields1. */
 188                 *(vector unsigned char *)
 189                         &elts[pos]->rx_descriptor_fields1 = rxdf1;
 190                 *(vector unsigned char *)
 191                         &elts[pos + 1]->rx_descriptor_fields1 = rxdf2;
 192
 193                 /* B.1 store rearm data to mbuf. */
 194                 *(vector unsigned char *)
 195                         &elts[pos + 2]->rearm_data = rearm;
 196                 *(vector unsigned char *)
 197                         &elts[pos + 3]->rearm_data = rearm;
 198
 199                 /* C.1 combine data from mCQEs with rx_descriptor_fields1. */
 200                 rxdf1 = vec_perm(mcqe2, zero, shuf_mask1);
 201                 rxdf2 = vec_perm(mcqe2, zero, shuf_mask2);
 202                 rxdf1 = (vector unsigned char)
 203                         ((vector unsigned short)rxdf1 -
 204                         (vector unsigned short)crc_adj);
 205                 rxdf2 = (vector unsigned char)
 206                         ((vector unsigned short)rxdf2 -
 207                         (vector unsigned short)crc_adj);
 208                 rxdf1 = (vector unsigned char)
 209                         vec_sel((vector unsigned short)rxdf1,
 210                         (vector unsigned short)rxdf, rxdf_sel_mask);
 211                 rxdf2 = (vector unsigned char)
 212                         vec_sel((vector unsigned short)rxdf2,
 213                         (vector unsigned short)rxdf, rxdf_sel_mask);
 214
 215                 /* D.1 store rx_descriptor_fields1. */
 216                 *(vector unsigned char *)
 217                         &elts[pos + 2]->rx_descriptor_fields1 = rxdf1;
 218                 *(vector unsigned char *)
 219                         &elts[pos + 3]->rx_descriptor_fields1 = rxdf2;
 220
 221 #ifdef MLX5_PMD_SOFT_COUNTERS
 222                 invalid_mask = (vector unsigned char)(vector unsigned long){
 223                         (mcqe_n - pos) * sizeof(uint16_t) * 8, 0};
 224
 225                 lshift =
 226                         vec_splat((vector unsigned long)invalid_mask, 0);
 227                 shmask = vec_cmpgt(shmax, lshift);
 228                 invalid_mask = (vector unsigned char)
 229                         vec_sl((vector unsigned long)ones, lshift);
 230                 invalid_mask = (vector unsigned char)
 231                         vec_sel((vector unsigned long)shmask,
 232                         (vector unsigned long)invalid_mask, shmask);
 233
 234                 mcqe1 = (vector unsigned char)
 235                         vec_sro((vector unsigned short)mcqe1,
 236                         (vector unsigned char){32}),
 237                 byte_cnt = (vector unsigned char)
 238                         vec_sel((vector unsigned short)mcqe1,
 239                         (vector unsigned short)mcqe2, mcqe_sel_mask);
 240                 byte_cnt = vec_perm(byte_cnt, zero, len_shuf_mask);
 241                 byte_cnt = (vector unsigned char)
 242                         vec_andc((vector unsigned long)byte_cnt,
 243                         (vector unsigned long)invalid_mask);
 244                 left = vec_perm((vector unsigned short)byte_cnt,
 245                         (vector unsigned short)zero, lower_half);
 246                 right = vec_perm((vector unsigned short)byte_cnt,
 247                         (vector unsigned short)zero, upper_half);
 248                 byte_cnt = (vector unsigned char)vec_add(left, right);
 249                 left = vec_perm((vector unsigned short)byte_cnt,
 250                         (vector unsigned short)zero, lower_half);
 251                 right = vec_perm((vector unsigned short)byte_cnt,
 252                         (vector unsigned short)zero, upper_half);
 253                 byte_cnt = (vector unsigned char)vec_add(left, right);
 254                 rcvd_byte += ((vector unsigned long)byte_cnt)[0];
 255 #endif
 256
 257                 if (rxq->mark) {
 258                         /* E.1 store flow tag (rte_flow mark). */
 259                         elts[pos]->hash.fdir.hi = flow_tag;
 260                         elts[pos + 1]->hash.fdir.hi = flow_tag;
 261                         elts[pos + 2]->hash.fdir.hi = flow_tag;
 262                         elts[pos + 3]->hash.fdir.hi = flow_tag;
 263                 }
 264                 if (rxq->dynf_meta) {
 265                         int32_t offs = rxq->flow_meta_offset;
 266                         const uint32_t meta =
 267                                 *RTE_MBUF_DYNFIELD(t_pkt, offs, uint32_t *);
 268
 269                         /* Check if title packet has valid metadata. */
 270                         if (meta) {
 271                                 MLX5_ASSERT(t_pkt->ol_flags &
 272                                             rxq->flow_meta_mask);
 273                                 *RTE_MBUF_DYNFIELD(elts[pos], offs,
 274                                                         uint32_t *) = meta;
 275                                 *RTE_MBUF_DYNFIELD(elts[pos + 1], offs,
 276                                                         uint32_t *) = meta;
 277                                 *RTE_MBUF_DYNFIELD(elts[pos + 2], offs,
 278                                                         uint32_t *) = meta;
 279                                 *RTE_MBUF_DYNFIELD(elts[pos + 3], offs,
 280                                                         uint32_t *) = meta;
 281                         }
 282                 }
 283
 284                 pos += MLX5_VPMD_DESCS_PER_LOOP;
 285                 /* Move to next CQE and invalidate consumed CQEs. */
 286                 if (!(pos & 0x7) && pos < mcqe_n) {
 287                         if (pos + 8 < mcqe_n)
 288                                 rte_prefetch0((void *)(cq + pos + 8));
 289                         mcq = (void *)&(cq + pos)->pkt_info;
 290                         for (i = 0; i < 8; ++i)
 291                                 cq[inv++].op_own = MLX5_CQE_INVALIDATE;
 292                 }
 293         }
 294
 295         /* Invalidate the rest of CQEs. */
 296         for (; inv < mcqe_n; ++inv)
 297                 cq[inv].op_own = MLX5_CQE_INVALIDATE;
 298
 299 #ifdef MLX5_PMD_SOFT_COUNTERS
 300         rxq->stats.ipackets += mcqe_n;
 301         rxq->stats.ibytes += rcvd_byte;
 302 #endif
 303
 304         rxq->cq_ci += mcqe_n;
 305         return mcqe_n;
 306 }
 307
 308 /**
 309  * Calculate packet type and offload flag for mbuf and store it.
 310  *
 311  * @param rxq
 312  *   Pointer to RX queue structure.
 313  * @param cqes[4]
 314  *   Array of four 16bytes completions extracted from the original completion
 315  *   descriptor.
 316  * @param op_err
 317  *   Opcode vector having responder error status. Each field is 4B.
 318  * @param pkts
 319  *   Pointer to array of packets to be filled.
 320  */
 321 static inline void
 322 rxq_cq_to_ptype_oflags_v(struct mlx5_rxq_data *rxq,
 323                 vector unsigned char cqes[4], vector unsigned char op_err,
 324                 struct rte_mbuf **pkts)
 325 {
 326         vector unsigned char pinfo0, pinfo1;
 327         vector unsigned char pinfo, ptype;
 328         vector unsigned char ol_flags = (vector unsigned char)
 329                 (vector unsigned int){
 330                         rxq->rss_hash * PKT_RX_RSS_HASH |
 331                                 rxq->hw_timestamp * rxq->timestamp_rx_flag,
 332                         rxq->rss_hash * PKT_RX_RSS_HASH |
 333                                 rxq->hw_timestamp * rxq->timestamp_rx_flag,
 334                         rxq->rss_hash * PKT_RX_RSS_HASH |
 335                                 rxq->hw_timestamp * rxq->timestamp_rx_flag,
 336                         rxq->rss_hash * PKT_RX_RSS_HASH |
 337                                 rxq->hw_timestamp * rxq->timestamp_rx_flag};
 338         vector unsigned char cv_flags;
 339         const vector unsigned char zero = (vector unsigned char){0};
 340         const vector unsigned char ptype_mask =
 341                 (vector unsigned char)(vector unsigned int){
 342                 0x0000fd06, 0x0000fd06, 0x0000fd06, 0x0000fd06};
 343         const vector unsigned char ptype_ol_mask =
 344                 (vector unsigned char)(vector unsigned int){
 345                 0x00000106, 0x00000106, 0x00000106, 0x00000106};
 346         const vector unsigned char pinfo_mask =
 347                 (vector unsigned char)(vector unsigned int){
 348                 0x00000003, 0x00000003, 0x00000003, 0x00000003};
 349         const vector unsigned char cv_flag_sel = (vector unsigned char){
 350                 0, (uint8_t)(PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED),
 351                 (uint8_t)(PKT_RX_IP_CKSUM_GOOD >> 1), 0,
 352                 (uint8_t)(PKT_RX_L4_CKSUM_GOOD >> 1), 0,
 353                 (uint8_t)((PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD) >> 1),
 354                 0, 0, 0, 0, 0, 0, 0, 0, 0};
 355         const vector unsigned char cv_mask =
 356                 (vector unsigned char)(vector unsigned int){
 357                 PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
 358                 PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED,
 359                 PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
 360                 PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED,
 361                 PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
 362                 PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED,
 363                 PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
 364                 PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED};
 365         const vector unsigned char mbuf_init =
 366                 (vector unsigned char)vec_vsx_ld
 367                         (0, (vector unsigned char *)&rxq->mbuf_initializer);
 368         const vector unsigned short rearm_sel_mask =
 369                 (vector unsigned short){0, 0, 0, 0, 0xffff, 0xffff, 0, 0};
 370         vector unsigned char rearm0, rearm1, rearm2, rearm3;
 371         uint8_t pt_idx0, pt_idx1, pt_idx2, pt_idx3;
 372
 373         /* Extract pkt_info field. */
 374         pinfo0 = (vector unsigned char)
 375                 vec_mergeh((vector unsigned int)cqes[0],
 376                 (vector unsigned int)cqes[1]);
 377         pinfo1 = (vector unsigned char)
 378                 vec_mergeh((vector unsigned int)cqes[2],
 379                 (vector unsigned int)cqes[3]);
 380         pinfo = (vector unsigned char)
 381                 vec_mergeh((vector unsigned long)pinfo0,
 382                 (vector unsigned long)pinfo1);
 383
 384         /* Extract hdr_type_etc field. */
 385         pinfo0 = (vector unsigned char)
 386                 vec_mergel((vector unsigned int)cqes[0],
 387                 (vector unsigned int)cqes[1]);
 388         pinfo1 = (vector unsigned char)
 389                 vec_mergel((vector unsigned int)cqes[2],
 390                 (vector unsigned int)cqes[3]);
 391         ptype = (vector unsigned char)
 392                 vec_mergeh((vector unsigned long)pinfo0,
 393                 (vector unsigned long)pinfo1);
 394
 395         if (rxq->mark) {
 396                 const vector unsigned char pinfo_ft_mask =
 397                         (vector unsigned char)(vector unsigned int){
 398                         0xffffff00, 0xffffff00, 0xffffff00, 0xffffff00};
 399                 const vector unsigned char fdir_flags =
 400                         (vector unsigned char)(vector unsigned int){
 401                         PKT_RX_FDIR, PKT_RX_FDIR,
 402                         PKT_RX_FDIR, PKT_RX_FDIR};
 403                 vector unsigned char fdir_id_flags =
 404                         (vector unsigned char)(vector unsigned int){
 405                         PKT_RX_FDIR_ID, PKT_RX_FDIR_ID,
 406                         PKT_RX_FDIR_ID, PKT_RX_FDIR_ID};
 407                 vector unsigned char flow_tag, invalid_mask;
 408
 409                 flow_tag = (vector unsigned char)
 410                         vec_and((vector unsigned long)pinfo,
 411                         (vector unsigned long)pinfo_ft_mask);
 412
 413                 /* Check if flow tag is non-zero then set PKT_RX_FDIR. */
 414                 invalid_mask = (vector unsigned char)
 415                         vec_cmpeq((vector unsigned int)flow_tag,
 416                         (vector unsigned int)zero);
 417                 ol_flags = (vector unsigned char)
 418                         vec_or((vector unsigned long)ol_flags,
 419                         (vector unsigned long)
 420                         vec_andc((vector unsigned long)fdir_flags,
 421                         (vector unsigned long)invalid_mask));
 422
 423                 /* Mask out invalid entries. */
 424                 fdir_id_flags = (vector unsigned char)
 425                         vec_andc((vector unsigned long)fdir_id_flags,
 426                         (vector unsigned long)invalid_mask);
 427
 428                 /* Check if flow tag MLX5_FLOW_MARK_DEFAULT. */
 429                 ol_flags = (vector unsigned char)
 430                         vec_or((vector unsigned long)ol_flags,
 431                         (vector unsigned long)
 432                         vec_andc((vector unsigned long)fdir_id_flags,
 433                         (vector unsigned long)
 434                         vec_cmpeq((vector unsigned int)flow_tag,
 435                         (vector unsigned int)pinfo_ft_mask)));
 436         }
 437         /*
 438          * Merge the two fields to generate the following:
 439          * bit[1]     = l3_ok
 440          * bit[2]     = l4_ok
 441          * bit[8]     = cv
 442          * bit[11:10] = l3_hdr_type
 443          * bit[14:12] = l4_hdr_type
 444          * bit[15]    = ip_frag
 445          * bit[16]    = tunneled
 446          * bit[17]    = outer_l3_type
 447          */
 448         ptype = (vector unsigned char)
 449                 vec_and((vector unsigned long)ptype,
 450                 (vector unsigned long)ptype_mask);
 451         pinfo = (vector unsigned char)
 452                 vec_and((vector unsigned long)pinfo,
 453                 (vector unsigned long)pinfo_mask);
 454         pinfo = (vector unsigned char)
 455                 vec_sl((vector unsigned int)pinfo,
 456                 (vector unsigned int){16, 16, 16, 16});
 457
 458         /* Make pinfo has merged fields for ol_flags calculation. */
 459         pinfo = (vector unsigned char)
 460                 vec_or((vector unsigned long)ptype,
 461                 (vector unsigned long)pinfo);
 462         ptype = (vector unsigned char)
 463                 vec_sr((vector unsigned int)pinfo,
 464                 (vector unsigned int){10, 10, 10, 10});
 465         ptype = (vector unsigned char)
 466                 vec_packs((vector unsigned int)ptype,
 467                 (vector unsigned int)zero);
 468
 469         /* Errored packets will have RTE_PTYPE_ALL_MASK. */
 470         op_err = (vector unsigned char)
 471                 vec_sr((vector unsigned short)op_err,
 472                 (vector unsigned short){8, 8, 8, 8, 8, 8, 8, 8});
 473         ptype = (vector unsigned char)
 474                 vec_or((vector unsigned long)ptype,
 475                 (vector unsigned long)op_err);
 476
 477         pt_idx0 = (uint8_t)((vector unsigned char)ptype)[0];
 478         pt_idx1 = (uint8_t)((vector unsigned char)ptype)[2];
 479         pt_idx2 = (uint8_t)((vector unsigned char)ptype)[4];
 480         pt_idx3 = (uint8_t)((vector unsigned char)ptype)[6];
 481
 482         pkts[0]->packet_type = mlx5_ptype_table[pt_idx0] |
 483                 !!(pt_idx0 & (1 << 6)) * rxq->tunnel;
 484         pkts[1]->packet_type = mlx5_ptype_table[pt_idx1] |
 485                 !!(pt_idx1 & (1 << 6)) * rxq->tunnel;
 486         pkts[2]->packet_type = mlx5_ptype_table[pt_idx2] |
 487                 !!(pt_idx2 & (1 << 6)) * rxq->tunnel;
 488         pkts[3]->packet_type = mlx5_ptype_table[pt_idx3] |
 489                 !!(pt_idx3 & (1 << 6)) * rxq->tunnel;
 490
 491         /* Fill flags for checksum and VLAN. */
 492         pinfo = (vector unsigned char)
 493                 vec_and((vector unsigned long)pinfo,
 494                 (vector unsigned long)ptype_ol_mask);
 495         pinfo = vec_perm(cv_flag_sel, zero, pinfo);
 496
 497         /* Locate checksum flags at byte[2:1] and merge with VLAN flags. */
 498         cv_flags = (vector unsigned char)
 499                 vec_sl((vector unsigned int)pinfo,
 500                 (vector unsigned int){9, 9, 9, 9});
 501         cv_flags = (vector unsigned char)
 502                 vec_or((vector unsigned long)pinfo,
 503                 (vector unsigned long)cv_flags);
 504
 505         /* Move back flags to start from byte[0]. */
 506         cv_flags = (vector unsigned char)
 507                 vec_sr((vector unsigned int)cv_flags,
 508                 (vector unsigned int){8, 8, 8, 8});
 509
 510         /* Mask out garbage bits. */
 511         cv_flags = (vector unsigned char)
 512                 vec_and((vector unsigned long)cv_flags,
 513                 (vector unsigned long)cv_mask);
 514
 515         /* Merge to ol_flags. */
 516         ol_flags = (vector unsigned char)
 517                 vec_or((vector unsigned long)ol_flags,
 518                 (vector unsigned long)cv_flags);
 519
 520         /* Merge mbuf_init and ol_flags. */
 521         rearm0 = (vector unsigned char)
 522                 vec_sel((vector unsigned short)mbuf_init,
 523                 (vector unsigned short)
 524                 vec_slo((vector unsigned short)ol_flags,
 525                 (vector unsigned char){64}), rearm_sel_mask);
 526         rearm1 = (vector unsigned char)
 527                 vec_sel((vector unsigned short)mbuf_init,
 528                 (vector unsigned short)
 529                 vec_slo((vector unsigned short)ol_flags,
 530                 (vector unsigned char){32}), rearm_sel_mask);
 531         rearm2 = (vector unsigned char)
 532                 vec_sel((vector unsigned short)mbuf_init,
 533                 (vector unsigned short)ol_flags, rearm_sel_mask);
 534         rearm3 = (vector unsigned char)
 535                 vec_sel((vector unsigned short)mbuf_init,
 536                 (vector unsigned short)
 537                 vec_sro((vector unsigned short)ol_flags,
 538                 (vector unsigned char){32}), rearm_sel_mask);
 539
 540         /* Write 8B rearm_data and 8B ol_flags. */
 541         vec_vsx_st(rearm0, 0,
 542                 (vector unsigned char *)&pkts[0]->rearm_data);
 543         vec_vsx_st(rearm1, 0,
 544                 (vector unsigned char *)&pkts[1]->rearm_data);
 545         vec_vsx_st(rearm2, 0,
 546                 (vector unsigned char *)&pkts[2]->rearm_data);
 547         vec_vsx_st(rearm3, 0,
 548                 (vector unsigned char *)&pkts[3]->rearm_data);
 549 }
 550
 551 /**
 552  * Process a non-compressed completion and fill in mbufs in RX SW ring
 553  * with data extracted from the title completion descriptor.
 554  *
 555  * @param rxq
 556  *   Pointer to RX queue structure.
 557  * @param cq
 558  *   Pointer to completion array having a non-compressed completion at first.
 559  * @param elts
 560  *   Pointer to SW ring to be filled. The first mbuf has to be pre-built from
 561  *   the title completion descriptor to be copied to the rest of mbufs.
 562  * @param[out] pkts
 563  *   Array to store received packets.
 564  * @param pkts_n
 565  *   Maximum number of packets in array.
 566  * @param[out] err
 567  *   Pointer to a flag. Set non-zero value if pkts array has at least one error
 568  *   packet to handle.
 569  * @param[out] comp
 570  *   Pointer to a index. Set it to the first compressed completion if any.
 571  *
 572  * @return
 573  *   Number of CQEs successfully processed.
 574  */
 575 static inline uint16_t
 576 rxq_cq_process_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq,
 577                  struct rte_mbuf **elts, struct rte_mbuf **pkts,
 578                  uint16_t pkts_n, uint64_t *err, uint64_t *comp)
 579 {
 580         const uint16_t q_n = 1 << rxq->cqe_n;
 581         const uint16_t q_mask = q_n - 1;
 582         unsigned int pos;
 583         uint64_t n = 0;
 584         uint64_t comp_idx = MLX5_VPMD_DESCS_PER_LOOP;
 585         uint16_t nocmp_n = 0;
 586         unsigned int ownership = !!(rxq->cq_ci & (q_mask + 1));
 587         const vector unsigned char zero = (vector unsigned char){0};
 588         const vector unsigned char ones = vec_splat_u8(-1);
 589         const vector unsigned char owner_check =
 590                 (vector unsigned char)(vector unsigned long){
 591                 0x0100000001000000LL, 0x0100000001000000LL};
 592         const vector unsigned char opcode_check =
 593                 (vector unsigned char)(vector unsigned long){
 594                 0xf0000000f0000000LL, 0xf0000000f0000000LL};
 595         const vector unsigned char format_check =
 596                 (vector unsigned char)(vector unsigned long){
 597                 0x0c0000000c000000LL, 0x0c0000000c000000LL};
 598         const vector unsigned char resp_err_check =
 599                 (vector unsigned char)(vector unsigned long){
 600                 0xe0000000e0000000LL, 0xe0000000e0000000LL};
 601 #ifdef MLX5_PMD_SOFT_COUNTERS
 602         uint32_t rcvd_byte = 0;
 603         /* Mask to shuffle byte_cnt to add up stats. Do bswap16 for all. */
 604         const vector unsigned char len_shuf_mask = (vector unsigned char){
 605                  1,  0,  5,  4,
 606                  9,  8, 13, 12,
 607                 -1, -1, -1, -1,
 608                 -1, -1, -1, -1};
 609 #endif
 610         /* Mask to shuffle from extracted CQE to mbuf. */
 611         const vector unsigned char shuf_mask = (vector unsigned char){
 612                  5,  4,           /* bswap16, pkt_len */
 613                 -1, -1,           /* zero out 2nd half of pkt_len */
 614                  5,  4,           /* bswap16, data_len */
 615                 11, 10,           /* bswap16, vlan+tci */
 616                 15, 14, 13, 12,   /* bswap32, rss */
 617                  1,  2,  3, -1};  /* fdir.hi */
 618         /* Mask to blend from the last Qword to the first DQword. */
 619         /* Mask to blend from the last Qword to the first DQword. */
 620         const vector unsigned char blend_mask = (vector unsigned char){
 621                 -1,  0,  0,  0,
 622                  0,  0,  0,  0,
 623                 -1, -1, -1, -1,
 624                 -1, -1, -1, -1};
 625         const vector unsigned char crc_adj =
 626                 (vector unsigned char)(vector unsigned short){
 627                 rxq->crc_present * RTE_ETHER_CRC_LEN, 0,
 628                 rxq->crc_present * RTE_ETHER_CRC_LEN, 0, 0, 0, 0, 0};
 629         const vector unsigned char flow_mark_adj =
 630                 (vector unsigned char)(vector unsigned int){
 631                 0, 0, 0, rxq->mark * (-1)};
 632         const vector unsigned short cqe_sel_mask1 =
 633                 (vector unsigned short){0, 0, 0, 0, 0xffff, 0xffff, 0, 0};
 634         const vector unsigned short cqe_sel_mask2 =
 635                 (vector unsigned short){0, 0, 0xffff, 0, 0, 0, 0, 0};
 636
 637         /*
 638          * A. load first Qword (8bytes) in one loop.
 639          * B. copy 4 mbuf pointers from elts ring to returing pkts.
 640          * C. load remaining CQE data and extract necessary fields.
 641          *    Final 16bytes cqes[] extracted from original 64bytes CQE has the
 642          *    following structure:
 643          *        struct {
 644          *          uint8_t  pkt_info;
 645          *          uint8_t  flow_tag[3];
 646          *          uint16_t byte_cnt;
 647          *          uint8_t  rsvd4;
 648          *          uint8_t  op_own;
 649          *          uint16_t hdr_type_etc;
 650          *          uint16_t vlan_info;
 651          *          uint32_t rx_has_res;
 652          *        } c;
 653          * D. fill in mbuf.
 654          * E. get valid CQEs.
 655          * F. find compressed CQE.
 656          */
 657         for (pos = 0;
 658              pos < pkts_n;
 659              pos += MLX5_VPMD_DESCS_PER_LOOP) {
 660                 vector unsigned char cqes[MLX5_VPMD_DESCS_PER_LOOP];
 661                 vector unsigned char cqe_tmp1, cqe_tmp2;
 662                 vector unsigned char pkt_mb0, pkt_mb1, pkt_mb2, pkt_mb3;
 663                 vector unsigned char op_own, op_own_tmp1, op_own_tmp2;
 664                 vector unsigned char opcode, owner_mask, invalid_mask;
 665                 vector unsigned char comp_mask;
 666                 vector unsigned char mask;
 667 #ifdef MLX5_PMD_SOFT_COUNTERS
 668                 const vector unsigned char lower_half = {
 669                         0, 1, 4, 5, 8, 9, 12, 13,
 670                         16, 17, 20, 21, 24, 25, 28, 29};
 671                 const vector unsigned char upper_half = {
 672                         2, 3, 6, 7, 10, 11, 14, 15,
 673                         18, 19, 22, 23, 26, 27, 30, 31};
 674                 const vector unsigned long shmax = {64, 64};
 675                 vector unsigned char byte_cnt;
 676                 vector unsigned short left, right;
 677                 vector unsigned long lshift;
 678                 vector __attribute__((altivec(bool__)))
 679                         unsigned long shmask;
 680 #endif
 681                 vector unsigned char mbp1, mbp2;
 682                 vector unsigned char p =
 683                         (vector unsigned char)(vector unsigned short){
 684                                 0, 1, 2, 3, 0, 0, 0, 0};
 685                 unsigned int p1, p2, p3;
 686
 687                 /* Prefetch next 4 CQEs. */
 688                 if (pkts_n - pos >= 2 * MLX5_VPMD_DESCS_PER_LOOP) {
 689                         rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP]);
 690                         rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP + 1]);
 691                         rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP + 2]);
 692                         rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP + 3]);
 693                 }
 694
 695                 /* A.0 do not cross the end of CQ. */
 696                 mask = (vector unsigned char)(vector unsigned long){
 697                         (pkts_n - pos) * sizeof(uint16_t) * 8, 0};
 698
 699                 {
 700                         vector unsigned long lshift;
 701                         vector __attribute__((altivec(bool__)))
 702                                 unsigned long shmask;
 703                         const vector unsigned long shmax = {64, 64};
 704
 705                         lshift = vec_splat((vector unsigned long)mask, 0);
 706                         shmask = vec_cmpgt(shmax, lshift);
 707                         mask = (vector unsigned char)
 708                                 vec_sl((vector unsigned long)ones, lshift);
 709                         mask = (vector unsigned char)
 710                                 vec_sel((vector unsigned long)shmask,
 711                                 (vector unsigned long)mask, shmask);
 712                 }
 713
 714                 p = (vector unsigned char)
 715                         vec_andc((vector unsigned long)p,
 716                         (vector unsigned long)mask);
 717
 718                 /* A.1 load cqes. */
 719                 p3 = (unsigned int)((vector unsigned short)p)[3];
 720                 cqes[3] = (vector unsigned char)(vector unsigned long){
 721                         *(__rte_aligned(8) unsigned long *)
 722                         &cq[pos + p3].sop_drop_qpn, 0LL};
 723                 rte_compiler_barrier();
 724
 725                 p2 = (unsigned int)((vector unsigned short)p)[2];
 726                 cqes[2] = (vector unsigned char)(vector unsigned long){
 727                         *(__rte_aligned(8) unsigned long *)
 728                         &cq[pos + p2].sop_drop_qpn, 0LL};
 729                 rte_compiler_barrier();
 730
 731                 /* B.1 load mbuf pointers. */
 732                 mbp1 = (vector unsigned char)vec_vsx_ld(0,
 733                         (signed int const *)&elts[pos]);
 734                 mbp2 = (vector unsigned char)vec_vsx_ld(0,
 735                         (signed int const *)&elts[pos + 2]);
 736
 737                 /* A.1 load a block having op_own. */
 738                 p1 = (unsigned int)((vector unsigned short)p)[1];
 739                 cqes[1] = (vector unsigned char)(vector unsigned long){
 740                         *(__rte_aligned(8) unsigned long *)
 741                         &cq[pos + p1].sop_drop_qpn, 0LL};
 742                 rte_compiler_barrier();
 743
 744                 cqes[0] = (vector unsigned char)(vector unsigned long){
 745                         *(__rte_aligned(8) unsigned long *)
 746                         &cq[pos].sop_drop_qpn, 0LL};
 747                 rte_compiler_barrier();
 748
 749                 /* B.2 copy mbuf pointers. */
 750                 *(vector unsigned char *)&pkts[pos] = mbp1;
 751                 *(vector unsigned char *)&pkts[pos + 2] = mbp2;
 752                 rte_io_rmb();
 753
 754                 /* C.1 load remaining CQE data and extract necessary fields. */
 755                 cqe_tmp2 = *(vector unsigned char *)
 756                         &cq[pos + p3].pkt_info;
 757                 cqe_tmp1 = *(vector unsigned char *)
 758                         &cq[pos + p2].pkt_info;
 759                 cqes[3] = vec_sel(cqes[3], cqe_tmp2, blend_mask);
 760                 cqes[2] = vec_sel(cqes[2], cqe_tmp1, blend_mask);
 761                 cqe_tmp2 = (vector unsigned char)vec_vsx_ld(0,
 762                         (signed int const *)&cq[pos + p3].csum);
 763                 cqe_tmp1 = (vector unsigned char)vec_vsx_ld(0,
 764                         (signed int const *)&cq[pos + p2].csum);
 765                 cqes[3] = (vector unsigned char)
 766                         vec_sel((vector unsigned short)cqes[3],
 767                         (vector unsigned short)cqe_tmp2, cqe_sel_mask1);
 768                 cqes[2] = (vector unsigned char)
 769                         vec_sel((vector unsigned short)cqes[2],
 770                         (vector unsigned short)cqe_tmp1, cqe_sel_mask1);
 771                 cqe_tmp2 = (vector unsigned char)(vector unsigned long){
 772                         *(__rte_aligned(8) unsigned long *)
 773                         &cq[pos + p3].rsvd3[9], 0LL};
 774                 cqe_tmp1 = (vector unsigned char)(vector unsigned long){
 775                         *(__rte_aligned(8) unsigned long *)
 776                         &cq[pos + p2].rsvd3[9], 0LL};
 777                 cqes[3] = (vector unsigned char)
 778                         vec_sel((vector unsigned short)cqes[3],
 779                         (vector unsigned short)cqe_tmp2,
 780                         (vector unsigned short)cqe_sel_mask2);
 781                 cqes[2] = (vector unsigned char)
 782                         vec_sel((vector unsigned short)cqes[2],
 783                         (vector unsigned short)cqe_tmp1,
 784                         (vector unsigned short)cqe_sel_mask2);
 785
 786                 /* C.2 generate final structure for mbuf with swapping bytes. */
 787                 pkt_mb3 = vec_perm(cqes[3], zero, shuf_mask);
 788                 pkt_mb2 = vec_perm(cqes[2], zero, shuf_mask);
 789
 790                 /* C.3 adjust CRC length. */
 791                 pkt_mb3 = (vector unsigned char)
 792                         ((vector unsigned short)pkt_mb3 -
 793                         (vector unsigned short)crc_adj);
 794                 pkt_mb2 = (vector unsigned char)
 795                         ((vector unsigned short)pkt_mb2 -
 796                         (vector unsigned short)crc_adj);
 797
 798                 /* C.4 adjust flow mark. */
 799                 pkt_mb3 = (vector unsigned char)
 800                         ((vector unsigned int)pkt_mb3 +
 801                         (vector unsigned int)flow_mark_adj);
 802                 pkt_mb2 = (vector unsigned char)
 803                         ((vector unsigned int)pkt_mb2 +
 804                         (vector unsigned int)flow_mark_adj);
 805
 806                 /* D.1 fill in mbuf - rx_descriptor_fields1. */
 807                 *(vector unsigned char *)
 808                         &pkts[pos + 3]->pkt_len = pkt_mb3;
 809                 *(vector unsigned char *)
 810                         &pkts[pos + 2]->pkt_len = pkt_mb2;
 811
 812                 /* E.1 extract op_own field. */
 813                 op_own_tmp2 = (vector unsigned char)
 814                         vec_mergeh((vector unsigned int)cqes[2],
 815                         (vector unsigned int)cqes[3]);
 816
 817                 /* C.1 load remaining CQE data and extract necessary fields. */
 818                 cqe_tmp2 = *(vector unsigned char *)
 819                         &cq[pos + p1].pkt_info;
 820                 cqe_tmp1 = *(vector unsigned char *)
 821                         &cq[pos].pkt_info;
 822                 cqes[1] = vec_sel(cqes[1], cqe_tmp2, blend_mask);
 823                 cqes[0] = vec_sel(cqes[0], cqe_tmp2, blend_mask);
 824                 cqe_tmp2 = (vector unsigned char)vec_vsx_ld(0,
 825                         (signed int const *)&cq[pos + p1].csum);
 826                 cqe_tmp1 = (vector unsigned char)vec_vsx_ld(0,
 827                         (signed int const *)&cq[pos].csum);
 828                 cqes[1] = (vector unsigned char)
 829                         vec_sel((vector unsigned short)cqes[1],
 830                         (vector unsigned short)cqe_tmp2, cqe_sel_mask1);
 831                 cqes[0] = (vector unsigned char)
 832                         vec_sel((vector unsigned short)cqes[0],
 833                         (vector unsigned short)cqe_tmp1, cqe_sel_mask1);
 834                 cqe_tmp2 = (vector unsigned char)(vector unsigned long){
 835                         *(__rte_aligned(8) unsigned long *)
 836                         &cq[pos + p1].rsvd3[9], 0LL};
 837                 cqe_tmp1 = (vector unsigned char)(vector unsigned long){
 838                         *(__rte_aligned(8) unsigned long *)
 839                         &cq[pos].rsvd3[9], 0LL};
 840                 cqes[1] = (vector unsigned char)
 841                         vec_sel((vector unsigned short)cqes[1],
 842                         (vector unsigned short)cqe_tmp2, cqe_sel_mask2);
 843                 cqes[0] = (vector unsigned char)
 844                         vec_sel((vector unsigned short)cqes[0],
 845                         (vector unsigned short)cqe_tmp1, cqe_sel_mask2);
 846
 847                 /* C.2 generate final structure for mbuf with swapping bytes. */
 848                 pkt_mb1 = vec_perm(cqes[1], zero, shuf_mask);
 849                 pkt_mb0 = vec_perm(cqes[0], zero, shuf_mask);
 850
 851                 /* C.3 adjust CRC length. */
 852                 pkt_mb1 = (vector unsigned char)
 853                         ((vector unsigned short)pkt_mb1 -
 854                         (vector unsigned short)crc_adj);
 855                 pkt_mb0 = (vector unsigned char)
 856                         ((vector unsigned short)pkt_mb0 -
 857                         (vector unsigned short)crc_adj);
 858
 859                 /* C.4 adjust flow mark. */
 860                 pkt_mb1 = (vector unsigned char)
 861                         ((vector unsigned int)pkt_mb1 +
 862                         (vector unsigned int)flow_mark_adj);
 863                 pkt_mb0 = (vector unsigned char)
 864                         ((vector unsigned int)pkt_mb0 +
 865                         (vector unsigned int)flow_mark_adj);
 866
 867                 /* E.1 extract op_own byte. */
 868                 op_own_tmp1 = (vector unsigned char)
 869                         vec_mergeh((vector unsigned int)cqes[0],
 870                         (vector unsigned int)cqes[1]);
 871                 op_own = (vector unsigned char)
 872                         vec_mergel((vector unsigned long)op_own_tmp1,
 873                         (vector unsigned long)op_own_tmp2);
 874
 875                 /* D.1 fill in mbuf - rx_descriptor_fields1. */
 876                 *(vector unsigned char *)
 877                         &pkts[pos + 1]->pkt_len = pkt_mb1;
 878                 *(vector unsigned char *)
 879                         &pkts[pos]->pkt_len = pkt_mb0;
 880
 881                 /* E.2 flip owner bit to mark CQEs from last round. */
 882                 owner_mask = (vector unsigned char)
 883                         vec_and((vector unsigned long)op_own,
 884                         (vector unsigned long)owner_check);
 885                 if (ownership)
 886                         owner_mask = (vector unsigned char)
 887                                 vec_xor((vector unsigned long)owner_mask,
 888                                 (vector unsigned long)owner_check);
 889                 owner_mask = (vector unsigned char)
 890                         vec_cmpeq((vector unsigned int)owner_mask,
 891                         (vector unsigned int)owner_check);
 892                 owner_mask = (vector unsigned char)
 893                         vec_packs((vector unsigned int)owner_mask,
 894                         (vector unsigned int)zero);
 895
 896                 /* E.3 get mask for invalidated CQEs. */
 897                 opcode = (vector unsigned char)
 898                         vec_and((vector unsigned long)op_own,
 899                         (vector unsigned long)opcode_check);
 900                 invalid_mask = (vector unsigned char)
 901                         vec_cmpeq((vector unsigned int)opcode_check,
 902                         (vector unsigned int)opcode);
 903                 invalid_mask = (vector unsigned char)
 904                         vec_packs((vector unsigned int)invalid_mask,
 905                         (vector unsigned int)zero);
 906
 907                 /* E.4 mask out beyond boundary. */
 908                 invalid_mask = (vector unsigned char)
 909                         vec_or((vector unsigned long)invalid_mask,
 910                         (vector unsigned long)mask);
 911
 912                 /* E.5 merge invalid_mask with invalid owner. */
 913                 invalid_mask = (vector unsigned char)
 914                         vec_or((vector unsigned long)invalid_mask,
 915                         (vector unsigned long)owner_mask);
 916
 917                 /* F.1 find compressed CQE format. */
 918                 comp_mask = (vector unsigned char)
 919                         vec_and((vector unsigned long)op_own,
 920                         (vector unsigned long)format_check);
 921                 comp_mask = (vector unsigned char)
 922                         vec_cmpeq((vector unsigned int)comp_mask,
 923                         (vector unsigned int)format_check);
 924                 comp_mask = (vector unsigned char)
 925                         vec_packs((vector unsigned int)comp_mask,
 926                         (vector unsigned int)zero);
 927
 928                 /* F.2 mask out invalid entries. */
 929                 comp_mask = (vector unsigned char)
 930                         vec_andc((vector unsigned long)comp_mask,
 931                         (vector unsigned long)invalid_mask);
 932                 comp_idx = ((vector unsigned long)comp_mask)[0];
 933
 934                 /* F.3 get the first compressed CQE. */
 935                 comp_idx = comp_idx ? __builtin_ctzll(comp_idx) /
 936                         (sizeof(uint16_t) * 8) : MLX5_VPMD_DESCS_PER_LOOP;
 937
 938                 /* E.6 mask out entries after the compressed CQE. */
 939                 mask = (vector unsigned char)(vector unsigned long){
 940                         (comp_idx * sizeof(uint16_t) * 8), 0};
 941                 lshift = vec_splat((vector unsigned long)mask, 0);
 942                 shmask = vec_cmpgt(shmax, lshift);
 943                 mask = (vector unsigned char)
 944                         vec_sl((vector unsigned long)ones, lshift);
 945                 mask = (vector unsigned char)
 946                         vec_sel((vector unsigned long)shmask,
 947                         (vector unsigned long)mask, shmask);
 948                 invalid_mask = (vector unsigned char)
 949                         vec_or((vector unsigned long)invalid_mask,
 950                         (vector unsigned long)mask);
 951
 952                 /* E.7 count non-compressed valid CQEs. */
 953                 n = ((vector unsigned long)invalid_mask)[0];
 954                 n = n ? __builtin_ctzll(n) / (sizeof(uint16_t) * 8) :
 955                         MLX5_VPMD_DESCS_PER_LOOP;
 956                 nocmp_n += n;
 957
 958                 /* D.2 get the final invalid mask. */
 959                 mask = (vector unsigned char)(vector unsigned long){
 960                         (n * sizeof(uint16_t) * 8), 0};
 961                 lshift = vec_splat((vector unsigned long)mask, 0);
 962                 shmask = vec_cmpgt(shmax, lshift);
 963                 mask = (vector unsigned char)
 964                         vec_sl((vector unsigned long)ones, lshift);
 965                 mask = (vector unsigned char)
 966                         vec_sel((vector unsigned long)shmask,
 967                         (vector unsigned long)mask, shmask);
 968                 invalid_mask = (vector unsigned char)
 969                         vec_or((vector unsigned long)invalid_mask,
 970                         (vector unsigned long)mask);
 971
 972                 /* D.3 check error in opcode. */
 973                 opcode = (vector unsigned char)
 974                         vec_cmpeq((vector unsigned int)resp_err_check,
 975                         (vector unsigned int)opcode);
 976                 opcode = (vector unsigned char)
 977                         vec_packs((vector unsigned int)opcode,
 978                         (vector unsigned int)zero);
 979                 opcode = (vector unsigned char)
 980                         vec_andc((vector unsigned long)opcode,
 981                         (vector unsigned long)invalid_mask);
 982
 983                 /* D.4 mark if any error is set */
 984                 *err |= ((vector unsigned long)opcode)[0];
 985
 986                 /* D.5 fill in mbuf - rearm_data and packet_type. */
 987                 rxq_cq_to_ptype_oflags_v(rxq, cqes, opcode, &pkts[pos]);
 988                 if (rxq->hw_timestamp) {
 989                         int offset = rxq->timestamp_offset;
 990                         if (rxq->rt_timestamp) {
 991                                 struct mlx5_dev_ctx_shared *sh = rxq->sh;
 992                                 uint64_t ts;
 993
 994                                 ts = rte_be_to_cpu_64(cq[pos].timestamp);
 995                                 mlx5_timestamp_set(pkts[pos], offset,
 996                                         mlx5_txpp_convert_rx_ts(sh, ts));
 997                                 ts = rte_be_to_cpu_64(cq[pos + p1].timestamp);
 998                                 mlx5_timestamp_set(pkts[pos + 1], offset,
 999                                         mlx5_txpp_convert_rx_ts(sh, ts));
1000                                 ts = rte_be_to_cpu_64(cq[pos + p2].timestamp);
1001                                 mlx5_timestamp_set(pkts[pos + 2], offset,
1002                                         mlx5_txpp_convert_rx_ts(sh, ts));
1003                                 ts = rte_be_to_cpu_64(cq[pos + p3].timestamp);
1004                                 mlx5_timestamp_set(pkts[pos + 3], offset,
1005                                         mlx5_txpp_convert_rx_ts(sh, ts));
1006                         } else {
1007                                 mlx5_timestamp_set(pkts[pos], offset,
1008                                         rte_be_to_cpu_64(cq[pos].timestamp));
1009                                 mlx5_timestamp_set(pkts[pos + 1], offset,
1010                                         rte_be_to_cpu_64(cq[pos + p1].timestamp));
1011                                 mlx5_timestamp_set(pkts[pos + 2], offset,
1012                                         rte_be_to_cpu_64(cq[pos + p2].timestamp));
1013                                 mlx5_timestamp_set(pkts[pos + 3], offset,
1014                                         rte_be_to_cpu_64(cq[pos + p3].timestamp));
1015                         }
1016                 }
1017                 if (rxq->dynf_meta) {
1018                         uint64_t flag = rxq->flow_meta_mask;
1019                         int32_t offs = rxq->flow_meta_offset;
1020                         uint32_t metadata;
1021
1022                         /* This code is subject for futher optimization. */
1023                         metadata = cq[pos].flow_table_metadata;
1024                         *RTE_MBUF_DYNFIELD(pkts[pos], offs, uint32_t *) =
1025                                                                 metadata;
1026                         pkts[pos]->ol_flags |= metadata ? flag : 0ULL;
1027                         metadata = cq[pos + 1].flow_table_metadata;
1028                         *RTE_MBUF_DYNFIELD(pkts[pos + 1], offs, uint32_t *) =
1029                                                                 metadata;
1030                         pkts[pos + 1]->ol_flags |= metadata ? flag : 0ULL;
1031                         metadata = cq[pos + 2].flow_table_metadata;
1032                         *RTE_MBUF_DYNFIELD(pkts[pos + 2], offs, uint32_t *) =
1033                                                                 metadata;
1034                         pkts[pos + 2]->ol_flags |= metadata ? flag : 0ULL;
1035                         metadata = cq[pos + 3].flow_table_metadata;
1036                         *RTE_MBUF_DYNFIELD(pkts[pos + 3], offs, uint32_t *) =
1037                                                                 metadata;
1038                         pkts[pos + 3]->ol_flags |= metadata ? flag : 0ULL;
1039                 }
1040 #ifdef MLX5_PMD_SOFT_COUNTERS
1041                 /* Add up received bytes count. */
1042                 byte_cnt = vec_perm(op_own, zero, len_shuf_mask);
1043                 byte_cnt = (vector unsigned char)
1044                         vec_andc((vector unsigned long)byte_cnt,
1045                         (vector unsigned long)invalid_mask);
1046                 left = vec_perm((vector unsigned short)byte_cnt,
1047                         (vector unsigned short)zero, lower_half);
1048                 right = vec_perm((vector unsigned short)byte_cnt,
1049                         (vector unsigned short)zero, upper_half);
1050                 byte_cnt = (vector unsigned char)vec_add(left, right);
1051                 left = vec_perm((vector unsigned short)byte_cnt,
1052                         (vector unsigned short)zero, lower_half);
1053                 right = vec_perm((vector unsigned short)byte_cnt,
1054                         (vector unsigned short)zero, upper_half);
1055                 byte_cnt = (vector unsigned char)vec_add(left, right);
1056                 rcvd_byte += ((vector unsigned long)byte_cnt)[0];
1057 #endif
1058
1059                 /*
1060                  * Break the loop unless more valid CQE is expected, or if
1061                  * there's a compressed CQE.
1062                  */
1063                 if (n != MLX5_VPMD_DESCS_PER_LOOP)
1064                         break;
1065         }
1066 #ifdef MLX5_PMD_SOFT_COUNTERS
1067         rxq->stats.ipackets += nocmp_n;
1068         rxq->stats.ibytes += rcvd_byte;
1069 #endif
1070         if (comp_idx == n)
1071                 *comp = comp_idx;
1072         return nocmp_n;
1073 }
1074
1075 #endif /* RTE_PMD_MLX5_RXTX_VEC_ALTIVEC_H_ */