drivers/net/mlx5/mlx5_rx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2021 6WIND S.A.
   3  * Copyright 2021 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <stdint.h>
   7 #include <string.h>
   8 #include <stdlib.h>
   9
  10 #include <rte_mbuf.h>
  11 #include <rte_mempool.h>
  12 #include <rte_prefetch.h>
  13 #include <rte_common.h>
  14 #include <rte_branch_prediction.h>
  15 #include <rte_ether.h>
  16 #include <rte_cycles.h>
  17 #include <rte_flow.h>
  18
  19 #include <mlx5_prm.h>
  20 #include <mlx5_common.h>
  21
  22 #include "mlx5_autoconf.h"
  23 #include "mlx5_defs.h"
  24 #include "mlx5.h"
  25 #include "mlx5_mr.h"
  26 #include "mlx5_utils.h"
  27 #include "mlx5_rxtx.h"
  28 #include "mlx5_rx.h"
  29
  30
  31 static __rte_always_inline uint32_t
  32 rxq_cq_to_pkt_type(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe,
  33                    volatile struct mlx5_mini_cqe8 *mcqe);
  34
  35 static __rte_always_inline int
  36 mlx5_rx_poll_len(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe,
  37                  uint16_t cqe_cnt, volatile struct mlx5_mini_cqe8 **mcqe);
  38
  39 static __rte_always_inline uint32_t
  40 rxq_cq_to_ol_flags(volatile struct mlx5_cqe *cqe);
  41
  42 static __rte_always_inline void
  43 rxq_cq_to_mbuf(struct mlx5_rxq_data *rxq, struct rte_mbuf *pkt,
  44                volatile struct mlx5_cqe *cqe,
  45                volatile struct mlx5_mini_cqe8 *mcqe);
  46
  47 static inline void
  48 mlx5_lro_update_tcp_hdr(struct rte_tcp_hdr *__rte_restrict tcp,
  49                         volatile struct mlx5_cqe *__rte_restrict cqe,
  50                         uint32_t phcsum, uint8_t l4_type);
  51
  52 static inline void
  53 mlx5_lro_update_hdr(uint8_t *__rte_restrict padd,
  54                     volatile struct mlx5_cqe *__rte_restrict cqe,
  55                     volatile struct mlx5_mini_cqe8 *mcqe,
  56                     struct mlx5_rxq_data *rxq, uint32_t len);
  57
  58
  59 /**
  60  * Internal function to compute the number of used descriptors in an RX queue.
  61  *
  62  * @param rxq
  63  *   The Rx queue.
  64  *
  65  * @return
  66  *   The number of used Rx descriptor.
  67  */
  68 static uint32_t
  69 rx_queue_count(struct mlx5_rxq_data *rxq)
  70 {
  71         struct rxq_zip *zip = &rxq->zip;
  72         volatile struct mlx5_cqe *cqe;
  73         const unsigned int cqe_n = (1 << rxq->cqe_n);
  74         const unsigned int sges_n = (1 << rxq->sges_n);
  75         const unsigned int elts_n = (1 << rxq->elts_n);
  76         const unsigned int strd_n = (1 << rxq->strd_num_n);
  77         const unsigned int cqe_cnt = cqe_n - 1;
  78         unsigned int cq_ci, used;
  79
  80         /* if we are processing a compressed cqe */
  81         if (zip->ai) {
  82                 used = zip->cqe_cnt - zip->ai;
  83                 cq_ci = zip->cq_ci;
  84         } else {
  85                 used = 0;
  86                 cq_ci = rxq->cq_ci;
  87         }
  88         cqe = &(*rxq->cqes)[cq_ci & cqe_cnt];
  89         while (check_cqe(cqe, cqe_n, cq_ci) != MLX5_CQE_STATUS_HW_OWN) {
  90                 int8_t op_own;
  91                 unsigned int n;
  92
  93                 op_own = cqe->op_own;
  94                 if (MLX5_CQE_FORMAT(op_own) == MLX5_COMPRESSED)
  95                         n = rte_be_to_cpu_32(cqe->byte_cnt);
  96                 else
  97                         n = 1;
  98                 cq_ci += n;
  99                 used += n;
 100                 cqe = &(*rxq->cqes)[cq_ci & cqe_cnt];
 101         }
 102         used = RTE_MIN(used * sges_n, elts_n * strd_n);
 103         return used;
 104 }
 105
 106 /**
 107  * DPDK callback to check the status of a Rx descriptor.
 108  *
 109  * @param rx_queue
 110  *   The Rx queue.
 111  * @param[in] offset
 112  *   The index of the descriptor in the ring.
 113  *
 114  * @return
 115  *   The status of the Rx descriptor.
 116  */
 117 int
 118 mlx5_rx_descriptor_status(void *rx_queue, uint16_t offset)
 119 {
 120         struct mlx5_rxq_data *rxq = rx_queue;
 121         struct mlx5_rxq_ctrl *rxq_ctrl =
 122                         container_of(rxq, struct mlx5_rxq_ctrl, rxq);
 123         struct rte_eth_dev *dev = ETH_DEV(rxq_ctrl->priv);
 124
 125         if (dev->rx_pkt_burst == NULL ||
 126             dev->rx_pkt_burst == removed_rx_burst) {
 127                 rte_errno = ENOTSUP;
 128                 return -rte_errno;
 129         }
 130         if (offset >= (1 << rxq->cqe_n)) {
 131                 rte_errno = EINVAL;
 132                 return -rte_errno;
 133         }
 134         if (offset < rx_queue_count(rxq))
 135                 return RTE_ETH_RX_DESC_DONE;
 136         return RTE_ETH_RX_DESC_AVAIL;
 137 }
 138
 139 /**
 140  * DPDK callback to get the RX queue information.
 141  *
 142  * @param dev
 143  *   Pointer to the device structure.
 144  *
 145  * @param rx_queue_id
 146  *   Rx queue identificator.
 147  *
 148  * @param qinfo
 149  *   Pointer to the RX queue information structure.
 150  *
 151  * @return
 152  *   None.
 153  */
 154
 155 void
 156 mlx5_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
 157                   struct rte_eth_rxq_info *qinfo)
 158 {
 159         struct mlx5_priv *priv = dev->data->dev_private;
 160         struct mlx5_rxq_data *rxq = (*priv->rxqs)[rx_queue_id];
 161         struct mlx5_rxq_ctrl *rxq_ctrl =
 162                 container_of(rxq, struct mlx5_rxq_ctrl, rxq);
 163
 164         if (!rxq)
 165                 return;
 166         qinfo->mp = mlx5_rxq_mprq_enabled(rxq) ?
 167                                         rxq->mprq_mp : rxq->mp;
 168         qinfo->conf.rx_thresh.pthresh = 0;
 169         qinfo->conf.rx_thresh.hthresh = 0;
 170         qinfo->conf.rx_thresh.wthresh = 0;
 171         qinfo->conf.rx_free_thresh = rxq->rq_repl_thresh;
 172         qinfo->conf.rx_drop_en = 1;
 173         qinfo->conf.rx_deferred_start = rxq_ctrl ? 0 : 1;
 174         qinfo->conf.offloads = dev->data->dev_conf.rxmode.offloads;
 175         qinfo->scattered_rx = dev->data->scattered_rx;
 176         qinfo->nb_desc = mlx5_rxq_mprq_enabled(rxq) ?
 177                 (1 << rxq->elts_n) * (1 << rxq->strd_num_n) :
 178                 (1 << rxq->elts_n);
 179 }
 180
 181 /**
 182  * DPDK callback to get the RX packet burst mode information.
 183  *
 184  * @param dev
 185  *   Pointer to the device structure.
 186  *
 187  * @param rx_queue_id
 188  *   Rx queue identificatior.
 189  *
 190  * @param mode
 191  *   Pointer to the burts mode information.
 192  *
 193  * @return
 194  *   0 as success, -EINVAL as failure.
 195  */
 196 int
 197 mlx5_rx_burst_mode_get(struct rte_eth_dev *dev,
 198                        uint16_t rx_queue_id __rte_unused,
 199                        struct rte_eth_burst_mode *mode)
 200 {
 201         eth_rx_burst_t pkt_burst = dev->rx_pkt_burst;
 202         struct mlx5_priv *priv = dev->data->dev_private;
 203         struct mlx5_rxq_data *rxq;
 204
 205         rxq = (*priv->rxqs)[rx_queue_id];
 206         if (!rxq) {
 207                 rte_errno = EINVAL;
 208                 return -rte_errno;
 209         }
 210         if (pkt_burst == mlx5_rx_burst) {
 211                 snprintf(mode->info, sizeof(mode->info), "%s", "Scalar");
 212         } else if (pkt_burst == mlx5_rx_burst_mprq) {
 213                 snprintf(mode->info, sizeof(mode->info), "%s", "Multi-Packet RQ");
 214         } else if (pkt_burst == mlx5_rx_burst_vec) {
 215 #if defined RTE_ARCH_X86_64
 216                 snprintf(mode->info, sizeof(mode->info), "%s", "Vector SSE");
 217 #elif defined RTE_ARCH_ARM64
 218                 snprintf(mode->info, sizeof(mode->info), "%s", "Vector Neon");
 219 #elif defined RTE_ARCH_PPC_64
 220                 snprintf(mode->info, sizeof(mode->info), "%s", "Vector AltiVec");
 221 #else
 222                 return -EINVAL;
 223 #endif
 224         } else if (pkt_burst == mlx5_rx_burst_mprq_vec) {
 225 #if defined RTE_ARCH_X86_64
 226                 snprintf(mode->info, sizeof(mode->info), "%s", "MPRQ Vector SSE");
 227 #elif defined RTE_ARCH_ARM64
 228                 snprintf(mode->info, sizeof(mode->info), "%s", "MPRQ Vector Neon");
 229 #elif defined RTE_ARCH_PPC_64
 230                 snprintf(mode->info, sizeof(mode->info), "%s", "MPRQ Vector AltiVec");
 231 #else
 232                 return -EINVAL;
 233 #endif
 234         } else {
 235                 return -EINVAL;
 236         }
 237         return 0;
 238 }
 239
 240 /**
 241  * DPDK callback to get the number of used descriptors in a RX queue.
 242  *
 243  * @param dev
 244  *   Pointer to the device structure.
 245  *
 246  * @param rx_queue_id
 247  *   The Rx queue.
 248  *
 249  * @return
 250  *   The number of used rx descriptor.
 251  *   -EINVAL if the queue is invalid
 252  */
 253 uint32_t
 254 mlx5_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 255 {
 256         struct mlx5_priv *priv = dev->data->dev_private;
 257         struct mlx5_rxq_data *rxq;
 258
 259         if (dev->rx_pkt_burst == NULL ||
 260             dev->rx_pkt_burst == removed_rx_burst) {
 261                 rte_errno = ENOTSUP;
 262                 return -rte_errno;
 263         }
 264         rxq = (*priv->rxqs)[rx_queue_id];
 265         if (!rxq) {
 266                 rte_errno = EINVAL;
 267                 return -rte_errno;
 268         }
 269         return rx_queue_count(rxq);
 270 }
 271
 272 /**
 273  * Translate RX completion flags to packet type.
 274  *
 275  * @param[in] rxq
 276  *   Pointer to RX queue structure.
 277  * @param[in] cqe
 278  *   Pointer to CQE.
 279  *
 280  * @note: fix mlx5_dev_supported_ptypes_get() if any change here.
 281  *
 282  * @return
 283  *   Packet type for struct rte_mbuf.
 284  */
 285 static inline uint32_t
 286 rxq_cq_to_pkt_type(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe,
 287                                    volatile struct mlx5_mini_cqe8 *mcqe)
 288 {
 289         uint8_t idx;
 290         uint8_t ptype;
 291         uint8_t pinfo = (cqe->pkt_info & 0x3) << 6;
 292
 293         /* Get l3/l4 header from mini-CQE in case L3/L4 format*/
 294         if (mcqe == NULL ||
 295             rxq->mcqe_format != MLX5_CQE_RESP_FORMAT_L34H_STRIDX)
 296                 ptype = (cqe->hdr_type_etc & 0xfc00) >> 10;
 297         else
 298                 ptype = mcqe->hdr_type >> 2;
 299         /*
 300          * The index to the array should have:
 301          * bit[1:0] = l3_hdr_type
 302          * bit[4:2] = l4_hdr_type
 303          * bit[5] = ip_frag
 304          * bit[6] = tunneled
 305          * bit[7] = outer_l3_type
 306          */
 307         idx = pinfo | ptype;
 308         return mlx5_ptype_table[idx] | rxq->tunnel * !!(idx & (1 << 6));
 309 }
 310
 311 /**
 312  * Initialize Rx WQ and indexes.
 313  *
 314  * @param[in] rxq
 315  *   Pointer to RX queue structure.
 316  */
 317 void
 318 mlx5_rxq_initialize(struct mlx5_rxq_data *rxq)
 319 {
 320         const unsigned int wqe_n = 1 << rxq->elts_n;
 321         unsigned int i;
 322
 323         for (i = 0; (i != wqe_n); ++i) {
 324                 volatile struct mlx5_wqe_data_seg *scat;
 325                 uintptr_t addr;
 326                 uint32_t byte_count;
 327
 328                 if (mlx5_rxq_mprq_enabled(rxq)) {
 329                         struct mlx5_mprq_buf *buf = (*rxq->mprq_bufs)[i];
 330
 331                         scat = &((volatile struct mlx5_wqe_mprq *)
 332                                 rxq->wqes)[i].dseg;
 333                         addr = (uintptr_t)mlx5_mprq_buf_addr(buf,
 334                                                          1 << rxq->strd_num_n);
 335                         byte_count = (1 << rxq->strd_sz_n) *
 336                                         (1 << rxq->strd_num_n);
 337                 } else {
 338                         struct rte_mbuf *buf = (*rxq->elts)[i];
 339
 340                         scat = &((volatile struct mlx5_wqe_data_seg *)
 341                                         rxq->wqes)[i];
 342                         addr = rte_pktmbuf_mtod(buf, uintptr_t);
 343                         byte_count = DATA_LEN(buf);
 344                 }
 345                 /* scat->addr must be able to store a pointer. */
 346                 MLX5_ASSERT(sizeof(scat->addr) >= sizeof(uintptr_t));
 347                 *scat = (struct mlx5_wqe_data_seg){
 348                         .addr = rte_cpu_to_be_64(addr),
 349                         .byte_count = rte_cpu_to_be_32(byte_count),
 350                         .lkey = mlx5_rx_addr2mr(rxq, addr),
 351                 };
 352         }
 353         rxq->consumed_strd = 0;
 354         rxq->decompressed = 0;
 355         rxq->rq_pi = 0;
 356         rxq->zip = (struct rxq_zip){
 357                 .ai = 0,
 358         };
 359         rxq->elts_ci = mlx5_rxq_mprq_enabled(rxq) ?
 360                 (wqe_n >> rxq->sges_n) * (1 << rxq->strd_num_n) : 0;
 361         /* Update doorbell counter. */
 362         rxq->rq_ci = wqe_n >> rxq->sges_n;
 363         rte_io_wmb();
 364         *rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci);
 365 }
 366
 367 /**
 368  * Handle a Rx error.
 369  * The function inserts the RQ state to reset when the first error CQE is
 370  * shown, then drains the CQ by the caller function loop. When the CQ is empty,
 371  * it moves the RQ state to ready and initializes the RQ.
 372  * Next CQE identification and error counting are in the caller responsibility.
 373  *
 374  * @param[in] rxq
 375  *   Pointer to RX queue structure.
 376  * @param[in] vec
 377  *   1 when called from vectorized Rx burst, need to prepare mbufs for the RQ.
 378  *   0 when called from non-vectorized Rx burst.
 379  *
 380  * @return
 381  *   -1 in case of recovery error, otherwise the CQE status.
 382  */
 383 int
 384 mlx5_rx_err_handle(struct mlx5_rxq_data *rxq, uint8_t vec)
 385 {
 386         const uint16_t cqe_n = 1 << rxq->cqe_n;
 387         const uint16_t cqe_mask = cqe_n - 1;
 388         const uint16_t wqe_n = 1 << rxq->elts_n;
 389         const uint16_t strd_n = 1 << rxq->strd_num_n;
 390         struct mlx5_rxq_ctrl *rxq_ctrl =
 391                         container_of(rxq, struct mlx5_rxq_ctrl, rxq);
 392         union {
 393                 volatile struct mlx5_cqe *cqe;
 394                 volatile struct mlx5_err_cqe *err_cqe;
 395         } u = {
 396                 .cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_mask],
 397         };
 398         struct mlx5_mp_arg_queue_state_modify sm;
 399         int ret;
 400
 401         switch (rxq->err_state) {
 402         case MLX5_RXQ_ERR_STATE_NO_ERROR:
 403                 rxq->err_state = MLX5_RXQ_ERR_STATE_NEED_RESET;
 404                 /* Fall-through */
 405         case MLX5_RXQ_ERR_STATE_NEED_RESET:
 406                 sm.is_wq = 1;
 407                 sm.queue_id = rxq->idx;
 408                 sm.state = IBV_WQS_RESET;
 409                 if (mlx5_queue_state_modify(ETH_DEV(rxq_ctrl->priv), &sm))
 410                         return -1;
 411                 if (rxq_ctrl->dump_file_n <
 412                     rxq_ctrl->priv->config.max_dump_files_num) {
 413                         MKSTR(err_str, "Unexpected CQE error syndrome "
 414                               "0x%02x CQN = %u RQN = %u wqe_counter = %u"
 415                               " rq_ci = %u cq_ci = %u", u.err_cqe->syndrome,
 416                               rxq->cqn, rxq_ctrl->wqn,
 417                               rte_be_to_cpu_16(u.err_cqe->wqe_counter),
 418                               rxq->rq_ci << rxq->sges_n, rxq->cq_ci);
 419                         MKSTR(name, "dpdk_mlx5_port_%u_rxq_%u_%u",
 420                               rxq->port_id, rxq->idx, (uint32_t)rte_rdtsc());
 421                         mlx5_dump_debug_information(name, NULL, err_str, 0);
 422                         mlx5_dump_debug_information(name, "MLX5 Error CQ:",
 423                                                     (const void *)((uintptr_t)
 424                                                                     rxq->cqes),
 425                                                     sizeof(*u.cqe) * cqe_n);
 426                         mlx5_dump_debug_information(name, "MLX5 Error RQ:",
 427                                                     (const void *)((uintptr_t)
 428                                                                     rxq->wqes),
 429                                                     16 * wqe_n);
 430                         rxq_ctrl->dump_file_n++;
 431                 }
 432                 rxq->err_state = MLX5_RXQ_ERR_STATE_NEED_READY;
 433                 /* Fall-through */
 434         case MLX5_RXQ_ERR_STATE_NEED_READY:
 435                 ret = check_cqe(u.cqe, cqe_n, rxq->cq_ci);
 436                 if (ret == MLX5_CQE_STATUS_HW_OWN) {
 437                         rte_io_wmb();
 438                         *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci);
 439                         rte_io_wmb();
 440                         /*
 441                          * The RQ consumer index must be zeroed while moving
 442                          * from RESET state to RDY state.
 443                          */
 444                         *rxq->rq_db = rte_cpu_to_be_32(0);
 445                         rte_io_wmb();
 446                         sm.is_wq = 1;
 447                         sm.queue_id = rxq->idx;
 448                         sm.state = IBV_WQS_RDY;
 449                         if (mlx5_queue_state_modify(ETH_DEV(rxq_ctrl->priv),
 450                                                     &sm))
 451                                 return -1;
 452                         if (vec) {
 453                                 const uint32_t elts_n =
 454                                         mlx5_rxq_mprq_enabled(rxq) ?
 455                                         wqe_n * strd_n : wqe_n;
 456                                 const uint32_t e_mask = elts_n - 1;
 457                                 uint32_t elts_ci =
 458                                         mlx5_rxq_mprq_enabled(rxq) ?
 459                                         rxq->elts_ci : rxq->rq_ci;
 460                                 uint32_t elt_idx;
 461                                 struct rte_mbuf **elt;
 462                                 int i;
 463                                 unsigned int n = elts_n - (elts_ci -
 464                                                           rxq->rq_pi);
 465
 466                                 for (i = 0; i < (int)n; ++i) {
 467                                         elt_idx = (elts_ci + i) & e_mask;
 468                                         elt = &(*rxq->elts)[elt_idx];
 469                                         *elt = rte_mbuf_raw_alloc(rxq->mp);
 470                                         if (!*elt) {
 471                                                 for (i--; i >= 0; --i) {
 472                                                         elt_idx = (elts_ci +
 473                                                                    i) & elts_n;
 474                                                         elt = &(*rxq->elts)
 475                                                                 [elt_idx];
 476                                                         rte_pktmbuf_free_seg
 477                                                                 (*elt);
 478                                                 }
 479                                                 return -1;
 480                                         }
 481                                 }
 482                                 for (i = 0; i < (int)elts_n; ++i) {
 483                                         elt = &(*rxq->elts)[i];
 484                                         DATA_LEN(*elt) =
 485                                                 (uint16_t)((*elt)->buf_len -
 486                                                 rte_pktmbuf_headroom(*elt));
 487                                 }
 488                                 /* Padding with a fake mbuf for vec Rx. */
 489                                 for (i = 0; i < MLX5_VPMD_DESCS_PER_LOOP; ++i)
 490                                         (*rxq->elts)[elts_n + i] =
 491                                                                 &rxq->fake_mbuf;
 492                         }
 493                         mlx5_rxq_initialize(rxq);
 494                         rxq->err_state = MLX5_RXQ_ERR_STATE_NO_ERROR;
 495                 }
 496                 return ret;
 497         default:
 498                 return -1;
 499         }
 500 }
 501
 502 /**
 503  * Get size of the next packet for a given CQE. For compressed CQEs, the
 504  * consumer index is updated only once all packets of the current one have
 505  * been processed.
 506  *
 507  * @param rxq
 508  *   Pointer to RX queue.
 509  * @param cqe
 510  *   CQE to process.
 511  * @param[out] mcqe
 512  *   Store pointer to mini-CQE if compressed. Otherwise, the pointer is not
 513  *   written.
 514  *
 515  * @return
 516  *   0 in case of empty CQE, otherwise the packet size in bytes.
 517  */
 518 static inline int
 519 mlx5_rx_poll_len(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe,
 520                  uint16_t cqe_cnt, volatile struct mlx5_mini_cqe8 **mcqe)
 521 {
 522         struct rxq_zip *zip = &rxq->zip;
 523         uint16_t cqe_n = cqe_cnt + 1;
 524         int len;
 525         uint16_t idx, end;
 526
 527         do {
 528                 len = 0;
 529                 /* Process compressed data in the CQE and mini arrays. */
 530                 if (zip->ai) {
 531                         volatile struct mlx5_mini_cqe8 (*mc)[8] =
 532                                 (volatile struct mlx5_mini_cqe8 (*)[8])
 533                                 (uintptr_t)(&(*rxq->cqes)[zip->ca &
 534                                                           cqe_cnt].pkt_info);
 535                         len = rte_be_to_cpu_32((*mc)[zip->ai & 7].byte_cnt &
 536                                                rxq->byte_mask);
 537                         *mcqe = &(*mc)[zip->ai & 7];
 538                         if ((++zip->ai & 7) == 0) {
 539                                 /* Invalidate consumed CQEs */
 540                                 idx = zip->ca;
 541                                 end = zip->na;
 542                                 while (idx != end) {
 543                                         (*rxq->cqes)[idx & cqe_cnt].op_own =
 544                                                 MLX5_CQE_INVALIDATE;
 545                                         ++idx;
 546                                 }
 547                                 /*
 548                                  * Increment consumer index to skip the number
 549                                  * of CQEs consumed. Hardware leaves holes in
 550                                  * the CQ ring for software use.
 551                                  */
 552                                 zip->ca = zip->na;
 553                                 zip->na += 8;
 554                         }
 555                         if (unlikely(rxq->zip.ai == rxq->zip.cqe_cnt)) {
 556                                 /* Invalidate the rest */
 557                                 idx = zip->ca;
 558                                 end = zip->cq_ci;
 559
 560                                 while (idx != end) {
 561                                         (*rxq->cqes)[idx & cqe_cnt].op_own =
 562                                                 MLX5_CQE_INVALIDATE;
 563                                         ++idx;
 564                                 }
 565                                 rxq->cq_ci = zip->cq_ci;
 566                                 zip->ai = 0;
 567                         }
 568                 /*
 569                  * No compressed data, get next CQE and verify if it is
 570                  * compressed.
 571                  */
 572                 } else {
 573                         int ret;
 574                         int8_t op_own;
 575                         uint32_t cq_ci;
 576
 577                         ret = check_cqe(cqe, cqe_n, rxq->cq_ci);
 578                         if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
 579                                 if (unlikely(ret == MLX5_CQE_STATUS_ERR ||
 580                                              rxq->err_state)) {
 581                                         ret = mlx5_rx_err_handle(rxq, 0);
 582                                         if (ret == MLX5_CQE_STATUS_HW_OWN ||
 583                                             ret == -1)
 584                                                 return 0;
 585                                 } else {
 586                                         return 0;
 587                                 }
 588                         }
 589                         /*
 590                          * Introduce the local variable to have queue cq_ci
 591                          * index in queue structure always consistent with
 592                          * actual CQE boundary (not pointing to the middle
 593                          * of compressed CQE session).
 594                          */
 595                         cq_ci = rxq->cq_ci + 1;
 596                         op_own = cqe->op_own;
 597                         if (MLX5_CQE_FORMAT(op_own) == MLX5_COMPRESSED) {
 598                                 volatile struct mlx5_mini_cqe8 (*mc)[8] =
 599                                         (volatile struct mlx5_mini_cqe8 (*)[8])
 600                                         (uintptr_t)(&(*rxq->cqes)
 601                                                 [cq_ci & cqe_cnt].pkt_info);
 602
 603                                 /* Fix endianness. */
 604                                 zip->cqe_cnt = rte_be_to_cpu_32(cqe->byte_cnt);
 605                                 /*
 606                                  * Current mini array position is the one
 607                                  * returned by check_cqe64().
 608                                  *
 609                                  * If completion comprises several mini arrays,
 610                                  * as a special case the second one is located
 611                                  * 7 CQEs after the initial CQE instead of 8
 612                                  * for subsequent ones.
 613                                  */
 614                                 zip->ca = cq_ci;
 615                                 zip->na = zip->ca + 7;
 616                                 /* Compute the next non compressed CQE. */
 617                                 zip->cq_ci = rxq->cq_ci + zip->cqe_cnt;
 618                                 /* Get packet size to return. */
 619                                 len = rte_be_to_cpu_32((*mc)[0].byte_cnt &
 620                                                        rxq->byte_mask);
 621                                 *mcqe = &(*mc)[0];
 622                                 zip->ai = 1;
 623                                 /* Prefetch all to be invalidated */
 624                                 idx = zip->ca;
 625                                 end = zip->cq_ci;
 626                                 while (idx != end) {
 627                                         rte_prefetch0(&(*rxq->cqes)[(idx) &
 628                                                                     cqe_cnt]);
 629                                         ++idx;
 630                                 }
 631                         } else {
 632                                 rxq->cq_ci = cq_ci;
 633                                 len = rte_be_to_cpu_32(cqe->byte_cnt);
 634                         }
 635                 }
 636                 if (unlikely(rxq->err_state)) {
 637                         cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_cnt];
 638                         ++rxq->stats.idropped;
 639                 } else {
 640                         return len;
 641                 }
 642         } while (1);
 643 }
 644
 645 /**
 646  * Translate RX completion flags to offload flags.
 647  *
 648  * @param[in] cqe
 649  *   Pointer to CQE.
 650  *
 651  * @return
 652  *   Offload flags (ol_flags) for struct rte_mbuf.
 653  */
 654 static inline uint32_t
 655 rxq_cq_to_ol_flags(volatile struct mlx5_cqe *cqe)
 656 {
 657         uint32_t ol_flags = 0;
 658         uint16_t flags = rte_be_to_cpu_16(cqe->hdr_type_etc);
 659
 660         ol_flags =
 661                 TRANSPOSE(flags,
 662                           MLX5_CQE_RX_L3_HDR_VALID,
 663                           PKT_RX_IP_CKSUM_GOOD) |
 664                 TRANSPOSE(flags,
 665                           MLX5_CQE_RX_L4_HDR_VALID,
 666                           PKT_RX_L4_CKSUM_GOOD);
 667         return ol_flags;
 668 }
 669
 670 /**
 671  * Fill in mbuf fields from RX completion flags.
 672  * Note that pkt->ol_flags should be initialized outside of this function.
 673  *
 674  * @param rxq
 675  *   Pointer to RX queue.
 676  * @param pkt
 677  *   mbuf to fill.
 678  * @param cqe
 679  *   CQE to process.
 680  * @param rss_hash_res
 681  *   Packet RSS Hash result.
 682  */
 683 static inline void
 684 rxq_cq_to_mbuf(struct mlx5_rxq_data *rxq, struct rte_mbuf *pkt,
 685                volatile struct mlx5_cqe *cqe,
 686                volatile struct mlx5_mini_cqe8 *mcqe)
 687 {
 688         /* Update packet information. */
 689         pkt->packet_type = rxq_cq_to_pkt_type(rxq, cqe, mcqe);
 690
 691         if (rxq->rss_hash) {
 692                 uint32_t rss_hash_res = 0;
 693
 694                 /* If compressed, take hash result from mini-CQE. */
 695                 if (mcqe == NULL ||
 696                     rxq->mcqe_format != MLX5_CQE_RESP_FORMAT_HASH)
 697                         rss_hash_res = rte_be_to_cpu_32(cqe->rx_hash_res);
 698                 else
 699                         rss_hash_res = rte_be_to_cpu_32(mcqe->rx_hash_result);
 700                 if (rss_hash_res) {
 701                         pkt->hash.rss = rss_hash_res;
 702                         pkt->ol_flags |= PKT_RX_RSS_HASH;
 703                 }
 704         }
 705         if (rxq->mark) {
 706                 uint32_t mark = 0;
 707
 708                 /* If compressed, take flow tag from mini-CQE. */
 709                 if (mcqe == NULL ||
 710                     rxq->mcqe_format != MLX5_CQE_RESP_FORMAT_FTAG_STRIDX)
 711                         mark = cqe->sop_drop_qpn;
 712                 else
 713                         mark = ((mcqe->byte_cnt_flow & 0xff) << 8) |
 714                                 (mcqe->flow_tag_high << 16);
 715                 if (MLX5_FLOW_MARK_IS_VALID(mark)) {
 716                         pkt->ol_flags |= PKT_RX_FDIR;
 717                         if (mark != RTE_BE32(MLX5_FLOW_MARK_DEFAULT)) {
 718                                 pkt->ol_flags |= PKT_RX_FDIR_ID;
 719                                 pkt->hash.fdir.hi = mlx5_flow_mark_get(mark);
 720                         }
 721                 }
 722         }
 723         if (rxq->dynf_meta) {
 724                 uint32_t meta = cqe->flow_table_metadata &
 725                                 rxq->flow_meta_port_mask;
 726
 727                 if (meta) {
 728                         pkt->ol_flags |= rxq->flow_meta_mask;
 729                         *RTE_MBUF_DYNFIELD(pkt, rxq->flow_meta_offset,
 730                                                 uint32_t *) = meta;
 731                 }
 732         }
 733         if (rxq->csum)
 734                 pkt->ol_flags |= rxq_cq_to_ol_flags(cqe);
 735         if (rxq->vlan_strip) {
 736                 bool vlan_strip;
 737
 738                 if (mcqe == NULL ||
 739                     rxq->mcqe_format != MLX5_CQE_RESP_FORMAT_L34H_STRIDX)
 740                         vlan_strip = cqe->hdr_type_etc &
 741                                      RTE_BE16(MLX5_CQE_VLAN_STRIPPED);
 742                 else
 743                         vlan_strip = mcqe->hdr_type &
 744                                      RTE_BE16(MLX5_CQE_VLAN_STRIPPED);
 745                 if (vlan_strip) {
 746                         pkt->ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED;
 747                         pkt->vlan_tci = rte_be_to_cpu_16(cqe->vlan_info);
 748                 }
 749         }
 750         if (rxq->hw_timestamp) {
 751                 uint64_t ts = rte_be_to_cpu_64(cqe->timestamp);
 752
 753                 if (rxq->rt_timestamp)
 754                         ts = mlx5_txpp_convert_rx_ts(rxq->sh, ts);
 755                 mlx5_timestamp_set(pkt, rxq->timestamp_offset, ts);
 756                 pkt->ol_flags |= rxq->timestamp_rx_flag;
 757         }
 758 }
 759
 760 /**
 761  * DPDK callback for RX.
 762  *
 763  * @param dpdk_rxq
 764  *   Generic pointer to RX queue structure.
 765  * @param[out] pkts
 766  *   Array to store received packets.
 767  * @param pkts_n
 768  *   Maximum number of packets in array.
 769  *
 770  * @return
 771  *   Number of packets successfully received (<= pkts_n).
 772  */
 773 uint16_t
 774 mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 775 {
 776         struct mlx5_rxq_data *rxq = dpdk_rxq;
 777         const unsigned int wqe_cnt = (1 << rxq->elts_n) - 1;
 778         const unsigned int cqe_cnt = (1 << rxq->cqe_n) - 1;
 779         const unsigned int sges_n = rxq->sges_n;
 780         struct rte_mbuf *pkt = NULL;
 781         struct rte_mbuf *seg = NULL;
 782         volatile struct mlx5_cqe *cqe =
 783                 &(*rxq->cqes)[rxq->cq_ci & cqe_cnt];
 784         unsigned int i = 0;
 785         unsigned int rq_ci = rxq->rq_ci << sges_n;
 786         int len = 0; /* keep its value across iterations. */
 787
 788         while (pkts_n) {
 789                 unsigned int idx = rq_ci & wqe_cnt;
 790                 volatile struct mlx5_wqe_data_seg *wqe =
 791                         &((volatile struct mlx5_wqe_data_seg *)rxq->wqes)[idx];
 792                 struct rte_mbuf *rep = (*rxq->elts)[idx];
 793                 volatile struct mlx5_mini_cqe8 *mcqe = NULL;
 794
 795                 if (pkt)
 796                         NEXT(seg) = rep;
 797                 seg = rep;
 798                 rte_prefetch0(seg);
 799                 rte_prefetch0(cqe);
 800                 rte_prefetch0(wqe);
 801                 /* Allocate the buf from the same pool. */
 802                 rep = rte_mbuf_raw_alloc(seg->pool);
 803                 if (unlikely(rep == NULL)) {
 804                         ++rxq->stats.rx_nombuf;
 805                         if (!pkt) {
 806                                 /*
 807                                  * no buffers before we even started,
 808                                  * bail out silently.
 809                                  */
 810                                 break;
 811                         }
 812                         while (pkt != seg) {
 813                                 MLX5_ASSERT(pkt != (*rxq->elts)[idx]);
 814                                 rep = NEXT(pkt);
 815                                 NEXT(pkt) = NULL;
 816                                 NB_SEGS(pkt) = 1;
 817                                 rte_mbuf_raw_free(pkt);
 818                                 pkt = rep;
 819                         }
 820                         rq_ci >>= sges_n;
 821                         ++rq_ci;
 822                         rq_ci <<= sges_n;
 823                         break;
 824                 }
 825                 if (!pkt) {
 826                         cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_cnt];
 827                         len = mlx5_rx_poll_len(rxq, cqe, cqe_cnt, &mcqe);
 828                         if (!len) {
 829                                 rte_mbuf_raw_free(rep);
 830                                 break;
 831                         }
 832                         pkt = seg;
 833                         MLX5_ASSERT(len >= (rxq->crc_present << 2));
 834                         pkt->ol_flags &= EXT_ATTACHED_MBUF;
 835                         rxq_cq_to_mbuf(rxq, pkt, cqe, mcqe);
 836                         if (rxq->crc_present)
 837                                 len -= RTE_ETHER_CRC_LEN;
 838                         PKT_LEN(pkt) = len;
 839                         if (cqe->lro_num_seg > 1) {
 840                                 mlx5_lro_update_hdr
 841                                         (rte_pktmbuf_mtod(pkt, uint8_t *), cqe,
 842                                          mcqe, rxq, len);
 843                                 pkt->ol_flags |= PKT_RX_LRO;
 844                                 pkt->tso_segsz = len / cqe->lro_num_seg;
 845                         }
 846                 }
 847                 DATA_LEN(rep) = DATA_LEN(seg);
 848                 PKT_LEN(rep) = PKT_LEN(seg);
 849                 SET_DATA_OFF(rep, DATA_OFF(seg));
 850                 PORT(rep) = PORT(seg);
 851                 (*rxq->elts)[idx] = rep;
 852                 /*
 853                  * Fill NIC descriptor with the new buffer. The lkey and size
 854                  * of the buffers are already known, only the buffer address
 855                  * changes.
 856                  */
 857                 wqe->addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(rep, uintptr_t));
 858                 /* If there's only one MR, no need to replace LKey in WQE. */
 859                 if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1))
 860                         wqe->lkey = mlx5_rx_mb2mr(rxq, rep);
 861                 if (len > DATA_LEN(seg)) {
 862                         len -= DATA_LEN(seg);
 863                         ++NB_SEGS(pkt);
 864                         ++rq_ci;
 865                         continue;
 866                 }
 867                 DATA_LEN(seg) = len;
 868 #ifdef MLX5_PMD_SOFT_COUNTERS
 869                 /* Increment bytes counter. */
 870                 rxq->stats.ibytes += PKT_LEN(pkt);
 871 #endif
 872                 /* Return packet. */
 873                 *(pkts++) = pkt;
 874                 pkt = NULL;
 875                 --pkts_n;
 876                 ++i;
 877                 /* Align consumer index to the next stride. */
 878                 rq_ci >>= sges_n;
 879                 ++rq_ci;
 880                 rq_ci <<= sges_n;
 881         }
 882         if (unlikely(i == 0 && ((rq_ci >> sges_n) == rxq->rq_ci)))
 883                 return 0;
 884         /* Update the consumer index. */
 885         rxq->rq_ci = rq_ci >> sges_n;
 886         rte_io_wmb();
 887         *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci);
 888         rte_io_wmb();
 889         *rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci);
 890 #ifdef MLX5_PMD_SOFT_COUNTERS
 891         /* Increment packets counter. */
 892         rxq->stats.ipackets += i;
 893 #endif
 894         return i;
 895 }
 896
 897 /**
 898  * Update LRO packet TCP header.
 899  * The HW LRO feature doesn't update the TCP header after coalescing the
 900  * TCP segments but supplies information in CQE to fill it by SW.
 901  *
 902  * @param tcp
 903  *   Pointer to the TCP header.
 904  * @param cqe
 905  *   Pointer to the completion entry.
 906  * @param phcsum
 907  *   The L3 pseudo-header checksum.
 908  */
 909 static inline void
 910 mlx5_lro_update_tcp_hdr(struct rte_tcp_hdr *__rte_restrict tcp,
 911                         volatile struct mlx5_cqe *__rte_restrict cqe,
 912                         uint32_t phcsum, uint8_t l4_type)
 913 {
 914         /*
 915          * The HW calculates only the TCP payload checksum, need to complete
 916          * the TCP header checksum and the L3 pseudo-header checksum.
 917          */
 918         uint32_t csum = phcsum + cqe->csum;
 919
 920         if (l4_type == MLX5_L4_HDR_TYPE_TCP_EMPTY_ACK ||
 921             l4_type == MLX5_L4_HDR_TYPE_TCP_WITH_ACL) {
 922                 tcp->tcp_flags |= RTE_TCP_ACK_FLAG;
 923                 tcp->recv_ack = cqe->lro_ack_seq_num;
 924                 tcp->rx_win = cqe->lro_tcp_win;
 925         }
 926         if (cqe->lro_tcppsh_abort_dupack & MLX5_CQE_LRO_PUSH_MASK)
 927                 tcp->tcp_flags |= RTE_TCP_PSH_FLAG;
 928         tcp->cksum = 0;
 929         csum += rte_raw_cksum(tcp, (tcp->data_off >> 4) * 4);
 930         csum = ((csum & 0xffff0000) >> 16) + (csum & 0xffff);
 931         csum = (~csum) & 0xffff;
 932         if (csum == 0)
 933                 csum = 0xffff;
 934         tcp->cksum = csum;
 935 }
 936
 937 /**
 938  * Update LRO packet headers.
 939  * The HW LRO feature doesn't update the L3/TCP headers after coalescing the
 940  * TCP segments but supply information in CQE to fill it by SW.
 941  *
 942  * @param padd
 943  *   The packet address.
 944  * @param cqe
 945  *   Pointer to the completion entry.
 946  * @param len
 947  *   The packet length.
 948  */
 949 static inline void
 950 mlx5_lro_update_hdr(uint8_t *__rte_restrict padd,
 951                     volatile struct mlx5_cqe *__rte_restrict cqe,
 952                     volatile struct mlx5_mini_cqe8 *mcqe,
 953                     struct mlx5_rxq_data *rxq, uint32_t len)
 954 {
 955         union {
 956                 struct rte_ether_hdr *eth;
 957                 struct rte_vlan_hdr *vlan;
 958                 struct rte_ipv4_hdr *ipv4;
 959                 struct rte_ipv6_hdr *ipv6;
 960                 struct rte_tcp_hdr *tcp;
 961                 uint8_t *hdr;
 962         } h = {
 963                 .hdr = padd,
 964         };
 965         uint16_t proto = h.eth->ether_type;
 966         uint32_t phcsum;
 967         uint8_t l4_type;
 968
 969         h.eth++;
 970         while (proto == RTE_BE16(RTE_ETHER_TYPE_VLAN) ||
 971                proto == RTE_BE16(RTE_ETHER_TYPE_QINQ)) {
 972                 proto = h.vlan->eth_proto;
 973                 h.vlan++;
 974         }
 975         if (proto == RTE_BE16(RTE_ETHER_TYPE_IPV4)) {
 976                 h.ipv4->time_to_live = cqe->lro_min_ttl;
 977                 h.ipv4->total_length = rte_cpu_to_be_16(len - (h.hdr - padd));
 978                 h.ipv4->hdr_checksum = 0;
 979                 h.ipv4->hdr_checksum = rte_ipv4_cksum(h.ipv4);
 980                 phcsum = rte_ipv4_phdr_cksum(h.ipv4, 0);
 981                 h.ipv4++;
 982         } else {
 983                 h.ipv6->hop_limits = cqe->lro_min_ttl;
 984                 h.ipv6->payload_len = rte_cpu_to_be_16(len - (h.hdr - padd) -
 985                                                        sizeof(*h.ipv6));
 986                 phcsum = rte_ipv6_phdr_cksum(h.ipv6, 0);
 987                 h.ipv6++;
 988         }
 989         if (mcqe == NULL ||
 990             rxq->mcqe_format != MLX5_CQE_RESP_FORMAT_L34H_STRIDX)
 991                 l4_type = (rte_be_to_cpu_16(cqe->hdr_type_etc) &
 992                            MLX5_CQE_L4_TYPE_MASK) >> MLX5_CQE_L4_TYPE_SHIFT;
 993         else
 994                 l4_type = (rte_be_to_cpu_16(mcqe->hdr_type) &
 995                            MLX5_CQE_L4_TYPE_MASK) >> MLX5_CQE_L4_TYPE_SHIFT;
 996         mlx5_lro_update_tcp_hdr(h.tcp, cqe, phcsum, l4_type);
 997 }
 998
 999 void
1000 mlx5_mprq_buf_free_cb(void *addr __rte_unused, void *opaque)
1001 {
1002         struct mlx5_mprq_buf *buf = opaque;
1003
1004         if (__atomic_load_n(&buf->refcnt, __ATOMIC_RELAXED) == 1) {
1005                 rte_mempool_put(buf->mp, buf);
1006         } else if (unlikely(__atomic_sub_fetch(&buf->refcnt, 1,
1007                                                __ATOMIC_RELAXED) == 0)) {
1008                 __atomic_store_n(&buf->refcnt, 1, __ATOMIC_RELAXED);
1009                 rte_mempool_put(buf->mp, buf);
1010         }
1011 }
1012
1013 void
1014 mlx5_mprq_buf_free(struct mlx5_mprq_buf *buf)
1015 {
1016         mlx5_mprq_buf_free_cb(NULL, buf);
1017 }
1018
1019 /**
1020  * DPDK callback for RX with Multi-Packet RQ support.
1021  *
1022  * @param dpdk_rxq
1023  *   Generic pointer to RX queue structure.
1024  * @param[out] pkts
1025  *   Array to store received packets.
1026  * @param pkts_n
1027  *   Maximum number of packets in array.
1028  *
1029  * @return
1030  *   Number of packets successfully received (<= pkts_n).
1031  */
1032 uint16_t
1033 mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
1034 {
1035         struct mlx5_rxq_data *rxq = dpdk_rxq;
1036         const uint32_t strd_n = 1 << rxq->strd_num_n;
1037         const uint32_t strd_sz = 1 << rxq->strd_sz_n;
1038         const uint32_t cq_mask = (1 << rxq->cqe_n) - 1;
1039         const uint32_t wq_mask = (1 << rxq->elts_n) - 1;
1040         volatile struct mlx5_cqe *cqe = &(*rxq->cqes)[rxq->cq_ci & cq_mask];
1041         unsigned int i = 0;
1042         uint32_t rq_ci = rxq->rq_ci;
1043         uint16_t consumed_strd = rxq->consumed_strd;
1044         struct mlx5_mprq_buf *buf = (*rxq->mprq_bufs)[rq_ci & wq_mask];
1045
1046         while (i < pkts_n) {
1047                 struct rte_mbuf *pkt;
1048                 int ret;
1049                 uint32_t len;
1050                 uint16_t strd_cnt;
1051                 uint16_t strd_idx;
1052                 uint32_t byte_cnt;
1053                 volatile struct mlx5_mini_cqe8 *mcqe = NULL;
1054                 enum mlx5_rqx_code rxq_code;
1055
1056                 if (consumed_strd == strd_n) {
1057                         /* Replace WQE if the buffer is still in use. */
1058                         mprq_buf_replace(rxq, rq_ci & wq_mask);
1059                         /* Advance to the next WQE. */
1060                         consumed_strd = 0;
1061                         ++rq_ci;
1062                         buf = (*rxq->mprq_bufs)[rq_ci & wq_mask];
1063                 }
1064                 cqe = &(*rxq->cqes)[rxq->cq_ci & cq_mask];
1065                 ret = mlx5_rx_poll_len(rxq, cqe, cq_mask, &mcqe);
1066                 if (!ret)
1067                         break;
1068                 byte_cnt = ret;
1069                 len = (byte_cnt & MLX5_MPRQ_LEN_MASK) >> MLX5_MPRQ_LEN_SHIFT;
1070                 MLX5_ASSERT((int)len >= (rxq->crc_present << 2));
1071                 if (rxq->crc_present)
1072                         len -= RTE_ETHER_CRC_LEN;
1073                 if (mcqe &&
1074                     rxq->mcqe_format == MLX5_CQE_RESP_FORMAT_FTAG_STRIDX)
1075                         strd_cnt = (len / strd_sz) + !!(len % strd_sz);
1076                 else
1077                         strd_cnt = (byte_cnt & MLX5_MPRQ_STRIDE_NUM_MASK) >>
1078                                    MLX5_MPRQ_STRIDE_NUM_SHIFT;
1079                 MLX5_ASSERT(strd_cnt);
1080                 consumed_strd += strd_cnt;
1081                 if (byte_cnt & MLX5_MPRQ_FILLER_MASK)
1082                         continue;
1083                 strd_idx = rte_be_to_cpu_16(mcqe == NULL ?
1084                                         cqe->wqe_counter :
1085                                         mcqe->stride_idx);
1086                 MLX5_ASSERT(strd_idx < strd_n);
1087                 MLX5_ASSERT(!((rte_be_to_cpu_16(cqe->wqe_id) ^ rq_ci) &
1088                             wq_mask));
1089                 pkt = rte_pktmbuf_alloc(rxq->mp);
1090                 if (unlikely(pkt == NULL)) {
1091                         ++rxq->stats.rx_nombuf;
1092                         break;
1093                 }
1094                 len = (byte_cnt & MLX5_MPRQ_LEN_MASK) >> MLX5_MPRQ_LEN_SHIFT;
1095                 MLX5_ASSERT((int)len >= (rxq->crc_present << 2));
1096                 if (rxq->crc_present)
1097                         len -= RTE_ETHER_CRC_LEN;
1098                 rxq_code = mprq_buf_to_pkt(rxq, pkt, len, buf,
1099                                            strd_idx, strd_cnt);
1100                 if (unlikely(rxq_code != MLX5_RXQ_CODE_EXIT)) {
1101                         rte_pktmbuf_free_seg(pkt);
1102                         if (rxq_code == MLX5_RXQ_CODE_DROPPED) {
1103                                 ++rxq->stats.idropped;
1104                                 continue;
1105                         }
1106                         if (rxq_code == MLX5_RXQ_CODE_NOMBUF) {
1107                                 ++rxq->stats.rx_nombuf;
1108                                 break;
1109                         }
1110                 }
1111                 rxq_cq_to_mbuf(rxq, pkt, cqe, mcqe);
1112                 if (cqe->lro_num_seg > 1) {
1113                         mlx5_lro_update_hdr(rte_pktmbuf_mtod(pkt, uint8_t *),
1114                                             cqe, mcqe, rxq, len);
1115                         pkt->ol_flags |= PKT_RX_LRO;
1116                         pkt->tso_segsz = len / cqe->lro_num_seg;
1117                 }
1118                 PKT_LEN(pkt) = len;
1119                 PORT(pkt) = rxq->port_id;
1120 #ifdef MLX5_PMD_SOFT_COUNTERS
1121                 /* Increment bytes counter. */
1122                 rxq->stats.ibytes += PKT_LEN(pkt);
1123 #endif
1124                 /* Return packet. */
1125                 *(pkts++) = pkt;
1126                 ++i;
1127         }
1128         /* Update the consumer indexes. */
1129         rxq->consumed_strd = consumed_strd;
1130         rte_io_wmb();
1131         *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci);
1132         if (rq_ci != rxq->rq_ci) {
1133                 rxq->rq_ci = rq_ci;
1134                 rte_io_wmb();
1135                 *rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci);
1136         }
1137 #ifdef MLX5_PMD_SOFT_COUNTERS
1138         /* Increment packets counter. */
1139         rxq->stats.ipackets += i;
1140 #endif
1141         return i;
1142 }
1143
1144 /**
1145  * Dummy DPDK callback for RX.
1146  *
1147  * This function is used to temporarily replace the real callback during
1148  * unsafe control operations on the queue, or in case of error.
1149  *
1150  * @param dpdk_rxq
1151  *   Generic pointer to RX queue structure.
1152  * @param[out] pkts
1153  *   Array to store received packets.
1154  * @param pkts_n
1155  *   Maximum number of packets in array.
1156  *
1157  * @return
1158  *   Number of packets successfully received (<= pkts_n).
1159  */
1160 uint16_t
1161 removed_rx_burst(void *dpdk_rxq __rte_unused,
1162                  struct rte_mbuf **pkts __rte_unused,
1163                  uint16_t pkts_n __rte_unused)
1164 {
1165         rte_mb();
1166         return 0;
1167 }
1168
1169 /*
1170  * Vectorized Rx routines are not compiled in when required vector instructions
1171  * are not supported on a target architecture.
1172  * The following null stubs are needed for linkage when those are not included
1173  * outside of this file (e.g. mlx5_rxtx_vec_sse.c for x86).
1174  */
1175
1176 __rte_weak uint16_t
1177 mlx5_rx_burst_vec(void *dpdk_rxq __rte_unused,
1178                   struct rte_mbuf **pkts __rte_unused,
1179                   uint16_t pkts_n __rte_unused)
1180 {
1181         return 0;
1182 }
1183
1184 __rte_weak uint16_t
1185 mlx5_rx_burst_mprq_vec(void *dpdk_rxq __rte_unused,
1186                        struct rte_mbuf **pkts __rte_unused,
1187                        uint16_t pkts_n __rte_unused)
1188 {
1189         return 0;
1190 }
1191
1192 __rte_weak int
1193 mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq __rte_unused)
1194 {
1195         return -ENOTSUP;
1196 }
1197
1198 __rte_weak int
1199 mlx5_check_vec_rx_support(struct rte_eth_dev *dev __rte_unused)
1200 {
1201         return -ENOTSUP;
1202 }
1203