drivers/net/hinic/hinic_pmd_rx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Huawei Technologies Co., Ltd
   3  */
   4
   5 #include <rte_ether.h>
   6 #include <rte_mbuf.h>
   7 #ifdef __ARM64_NEON__
   8 #include <arm_neon.h>
   9 #endif
  10
  11 #include "base/hinic_compat.h"
  12 #include "base/hinic_pmd_hwdev.h"
  13 #include "base/hinic_pmd_wq.h"
  14 #include "base/hinic_pmd_niccfg.h"
  15 #include "base/hinic_pmd_nicio.h"
  16 #include "hinic_pmd_ethdev.h"
  17 #include "hinic_pmd_rx.h"
  18
  19 /* rxq wq operations */
  20 #define HINIC_GET_RQ_WQE_MASK(rxq)      \
  21         ((rxq)->wq->mask)
  22
  23 #define HINIC_GET_RQ_LOCAL_CI(rxq)      \
  24         (((rxq)->wq->cons_idx) & HINIC_GET_RQ_WQE_MASK(rxq))
  25
  26 #define HINIC_GET_RQ_LOCAL_PI(rxq)      \
  27         (((rxq)->wq->prod_idx) & HINIC_GET_RQ_WQE_MASK(rxq))
  28
  29 #define HINIC_UPDATE_RQ_LOCAL_CI(rxq, wqebb_cnt)        \
  30         do {                                            \
  31                 (rxq)->wq->cons_idx += (wqebb_cnt);     \
  32                 (rxq)->wq->delta += (wqebb_cnt);        \
  33         } while (0)
  34
  35 #define HINIC_UPDATE_RQ_HW_PI(rxq, pi)  \
  36         (*((rxq)->pi_virt_addr) =       \
  37                 cpu_to_be16((pi) & HINIC_GET_RQ_WQE_MASK(rxq)))
  38
  39 #define HINIC_GET_RQ_FREE_WQEBBS(rxq)   ((rxq)->wq->delta - 1)
  40
  41 /* rxq cqe done and status bit */
  42 #define HINIC_GET_RX_DONE_BE(status)    \
  43         ((status) & 0x80U)
  44
  45 #define HINIC_RX_CSUM_OFFLOAD_EN        0xFFF
  46
  47 #define RQ_CQE_SGE_VLAN_SHIFT                   0
  48 #define RQ_CQE_SGE_LEN_SHIFT                    16
  49
  50 #define RQ_CQE_SGE_VLAN_MASK                    0xFFFFU
  51 #define RQ_CQE_SGE_LEN_MASK                     0xFFFFU
  52
  53 #define RQ_CQE_SGE_GET(val, member)             \
  54         (((val) >> RQ_CQE_SGE_##member##_SHIFT) & RQ_CQE_SGE_##member##_MASK)
  55
  56 #define HINIC_GET_RX_VLAN_TAG(vlan_len) \
  57                 RQ_CQE_SGE_GET(vlan_len, VLAN)
  58
  59 #define HINIC_GET_RX_PKT_LEN(vlan_len)  \
  60                 RQ_CQE_SGE_GET(vlan_len, LEN)
  61
  62 #define RQ_CQE_STATUS_CSUM_ERR_SHIFT            0
  63 #define RQ_CQE_STATUS_NUM_LRO_SHIFT             16
  64 #define RQ_CQE_STATUS_LRO_PUSH_SHIFT            25
  65 #define RQ_CQE_STATUS_LRO_ENTER_SHIFT           26
  66 #define RQ_CQE_STATUS_LRO_INTR_SHIFT            27
  67
  68 #define RQ_CQE_STATUS_BP_EN_SHIFT               30
  69 #define RQ_CQE_STATUS_RXDONE_SHIFT              31
  70 #define RQ_CQE_STATUS_FLUSH_SHIFT               28
  71
  72 #define RQ_CQE_STATUS_CSUM_ERR_MASK             0xFFFFU
  73 #define RQ_CQE_STATUS_NUM_LRO_MASK              0xFFU
  74 #define RQ_CQE_STATUS_LRO_PUSH_MASK             0X1U
  75 #define RQ_CQE_STATUS_LRO_ENTER_MASK            0X1U
  76 #define RQ_CQE_STATUS_LRO_INTR_MASK             0X1U
  77 #define RQ_CQE_STATUS_BP_EN_MASK                0X1U
  78 #define RQ_CQE_STATUS_RXDONE_MASK               0x1U
  79 #define RQ_CQE_STATUS_FLUSH_MASK                0x1U
  80
  81 #define RQ_CQE_STATUS_GET(val, member)          \
  82                 (((val) >> RQ_CQE_STATUS_##member##_SHIFT) & \
  83                                 RQ_CQE_STATUS_##member##_MASK)
  84
  85 #define RQ_CQE_STATUS_CLEAR(val, member)        \
  86                 ((val) & (~(RQ_CQE_STATUS_##member##_MASK << \
  87                                 RQ_CQE_STATUS_##member##_SHIFT)))
  88
  89 #define HINIC_GET_RX_CSUM_ERR(status)   \
  90                 RQ_CQE_STATUS_GET(status, CSUM_ERR)
  91
  92 #define HINIC_GET_RX_DONE(status)       \
  93                 RQ_CQE_STATUS_GET(status, RXDONE)
  94
  95 #define HINIC_GET_RX_FLUSH(status)      \
  96                 RQ_CQE_STATUS_GET(status, FLUSH)
  97
  98 #define HINIC_GET_RX_BP_EN(status)      \
  99                 RQ_CQE_STATUS_GET(status, BP_EN)
 100
 101 #define HINIC_GET_RX_NUM_LRO(status)    \
 102                 RQ_CQE_STATUS_GET(status, NUM_LRO)
 103
 104 /* RQ_CTRL */
 105 #define RQ_CTRL_BUFDESC_SECT_LEN_SHIFT          0
 106 #define RQ_CTRL_COMPLETE_FORMAT_SHIFT           15
 107 #define RQ_CTRL_COMPLETE_LEN_SHIFT              27
 108 #define RQ_CTRL_LEN_SHIFT                       29
 109
 110 #define RQ_CTRL_BUFDESC_SECT_LEN_MASK           0xFFU
 111 #define RQ_CTRL_COMPLETE_FORMAT_MASK            0x1U
 112 #define RQ_CTRL_COMPLETE_LEN_MASK               0x3U
 113 #define RQ_CTRL_LEN_MASK                        0x3U
 114
 115 #define RQ_CTRL_SET(val, member)                \
 116         (((val) & RQ_CTRL_##member##_MASK) << RQ_CTRL_##member##_SHIFT)
 117
 118 #define RQ_CTRL_GET(val, member)                \
 119         (((val) >> RQ_CTRL_##member##_SHIFT) & RQ_CTRL_##member##_MASK)
 120
 121 #define RQ_CTRL_CLEAR(val, member)              \
 122         ((val) & (~(RQ_CTRL_##member##_MASK << RQ_CTRL_##member##_SHIFT)))
 123
 124 #define RQ_CQE_PKT_NUM_SHIFT                    1
 125 #define RQ_CQE_PKT_FIRST_LEN_SHIFT              19
 126 #define RQ_CQE_PKT_LAST_LEN_SHIFT               6
 127 #define RQ_CQE_SUPER_CQE_EN_SHIFT               0
 128
 129 #define RQ_CQE_PKT_FIRST_LEN_MASK               0x1FFFU
 130 #define RQ_CQE_PKT_LAST_LEN_MASK                0x1FFFU
 131 #define RQ_CQE_PKT_NUM_MASK                     0x1FU
 132 #define RQ_CQE_SUPER_CQE_EN_MASK                0x1
 133
 134 #define RQ_CQE_PKT_NUM_GET(val, member)         \
 135         (((val) >> RQ_CQE_PKT_##member##_SHIFT) & RQ_CQE_PKT_##member##_MASK)
 136
 137 #define HINIC_GET_RQ_CQE_PKT_NUM(pkt_info) RQ_CQE_PKT_NUM_GET(pkt_info, NUM)
 138
 139 #define RQ_CQE_SUPER_CQE_EN_GET(val, member)    \
 140         (((val) >> RQ_CQE_##member##_SHIFT) & RQ_CQE_##member##_MASK)
 141
 142 #define HINIC_GET_SUPER_CQE_EN(pkt_info)        \
 143         RQ_CQE_SUPER_CQE_EN_GET(pkt_info, SUPER_CQE_EN)
 144
 145 #define RQ_CQE_OFFOLAD_TYPE_VLAN_EN_SHIFT               21
 146 #define RQ_CQE_OFFOLAD_TYPE_VLAN_EN_MASK                0x1U
 147
 148 #define RQ_CQE_OFFOLAD_TYPE_PKT_TYPE_SHIFT              0
 149 #define RQ_CQE_OFFOLAD_TYPE_PKT_TYPE_MASK               0xFFFU
 150
 151 #define RQ_CQE_OFFOLAD_TYPE_PKT_UMBCAST_SHIFT           19
 152 #define RQ_CQE_OFFOLAD_TYPE_PKT_UMBCAST_MASK            0x3U
 153
 154 #define RQ_CQE_OFFOLAD_TYPE_RSS_TYPE_SHIFT              24
 155 #define RQ_CQE_OFFOLAD_TYPE_RSS_TYPE_MASK               0xFFU
 156
 157 #define RQ_CQE_OFFOLAD_TYPE_GET(val, member)            (((val) >> \
 158                                 RQ_CQE_OFFOLAD_TYPE_##member##_SHIFT) & \
 159                                 RQ_CQE_OFFOLAD_TYPE_##member##_MASK)
 160
 161 #define HINIC_GET_RX_VLAN_OFFLOAD_EN(offload_type)      \
 162                 RQ_CQE_OFFOLAD_TYPE_GET(offload_type, VLAN_EN)
 163
 164 #define HINIC_GET_RSS_TYPES(offload_type)       \
 165                 RQ_CQE_OFFOLAD_TYPE_GET(offload_type, RSS_TYPE)
 166
 167 #define HINIC_GET_RX_PKT_TYPE(offload_type)     \
 168                 RQ_CQE_OFFOLAD_TYPE_GET(offload_type, PKT_TYPE)
 169
 170 #define HINIC_GET_RX_PKT_UMBCAST(offload_type)  \
 171                 RQ_CQE_OFFOLAD_TYPE_GET(offload_type, PKT_UMBCAST)
 172
 173 #define RQ_CQE_STATUS_CSUM_BYPASS_VAL                   0x80U
 174 #define RQ_CQE_STATUS_CSUM_ERR_IP_MASK                  0x39U
 175 #define RQ_CQE_STATUS_CSUM_ERR_L4_MASK                  0x46U
 176 #define RQ_CQE_STATUS_CSUM_ERR_OTHER                    0x100U
 177
 178 #define HINIC_CSUM_ERR_BYPASSED(csum_err)        \
 179         ((csum_err) == RQ_CQE_STATUS_CSUM_BYPASS_VAL)
 180
 181 #define HINIC_CSUM_ERR_IP(csum_err)      \
 182         ((csum_err) & RQ_CQE_STATUS_CSUM_ERR_IP_MASK)
 183
 184 #define HINIC_CSUM_ERR_L4(csum_err)      \
 185         ((csum_err) & RQ_CQE_STATUS_CSUM_ERR_L4_MASK)
 186
 187 #define HINIC_CSUM_ERR_OTHER(csum_err)   \
 188         ((csum_err) == RQ_CQE_STATUS_CSUM_ERR_OTHER)
 189
 190
 191 void hinic_get_func_rx_buf_size(struct hinic_nic_dev *nic_dev)
 192 {
 193         struct hinic_rxq *rxq;
 194         u16 q_id;
 195         u16 buf_size = 0;
 196
 197         for (q_id = 0; q_id < nic_dev->num_rq; q_id++) {
 198                 rxq = nic_dev->rxqs[q_id];
 199
 200                 if (rxq == NULL)
 201                         continue;
 202
 203                 if (q_id == 0)
 204                         buf_size = rxq->buf_len;
 205
 206                 buf_size = buf_size > rxq->buf_len ? rxq->buf_len : buf_size;
 207         }
 208
 209         nic_dev->hwdev->nic_io->rq_buf_size = buf_size;
 210 }
 211
 212 int hinic_create_rq(struct hinic_hwdev *hwdev, u16 q_id, u16 rq_depth)
 213 {
 214         int err;
 215         struct hinic_nic_io *nic_io = hwdev->nic_io;
 216         struct hinic_qp *qp = &nic_io->qps[q_id];
 217         struct hinic_rq *rq = &qp->rq;
 218
 219         /* in case of hardware still generate interrupt, do not use msix 0 */
 220         rq->msix_entry_idx = 1;
 221         rq->q_id = q_id;
 222         rq->rq_depth = rq_depth;
 223         nic_io->rq_depth = rq_depth;
 224
 225         err = hinic_wq_allocate(hwdev, &nic_io->rq_wq[q_id],
 226                                 HINIC_RQ_WQEBB_SHIFT, nic_io->rq_depth);
 227         if (err) {
 228                 PMD_DRV_LOG(ERR, "Failed to allocate WQ for RQ");
 229                 return err;
 230         }
 231         rq->wq = &nic_io->rq_wq[q_id];
 232
 233         rq->pi_virt_addr =
 234                 (volatile u16 *)dma_zalloc_coherent(hwdev, HINIC_PAGE_SIZE,
 235                                                     &rq->pi_dma_addr,
 236                                                     GFP_KERNEL);
 237         if (!rq->pi_virt_addr) {
 238                 PMD_DRV_LOG(ERR, "Failed to allocate rq pi virt addr");
 239                 err = -ENOMEM;
 240                 goto rq_pi_alloc_err;
 241         }
 242
 243         return HINIC_OK;
 244
 245 rq_pi_alloc_err:
 246         hinic_wq_free(hwdev, &nic_io->rq_wq[q_id]);
 247
 248         return err;
 249 }
 250
 251 void hinic_destroy_rq(struct hinic_hwdev *hwdev, u16 q_id)
 252 {
 253         struct hinic_nic_io *nic_io = hwdev->nic_io;
 254         struct hinic_qp *qp = &nic_io->qps[q_id];
 255         struct hinic_rq *rq = &qp->rq;
 256
 257         if (qp->rq.wq == NULL)
 258                 return;
 259
 260         dma_free_coherent_volatile(hwdev, HINIC_PAGE_SIZE,
 261                                    (volatile void *)rq->pi_virt_addr,
 262                                    rq->pi_dma_addr);
 263         hinic_wq_free(nic_io->hwdev, qp->rq.wq);
 264         qp->rq.wq = NULL;
 265 }
 266
 267 static void
 268 hinic_prepare_rq_wqe(void *wqe, __rte_unused u16 pi, dma_addr_t buf_addr,
 269                         dma_addr_t cqe_dma)
 270 {
 271         struct hinic_rq_wqe *rq_wqe = wqe;
 272         struct hinic_rq_ctrl *ctrl = &rq_wqe->ctrl;
 273         struct hinic_rq_cqe_sect *cqe_sect = &rq_wqe->cqe_sect;
 274         struct hinic_rq_bufdesc *buf_desc = &rq_wqe->buf_desc;
 275         u32 rq_ceq_len = sizeof(struct hinic_rq_cqe);
 276
 277         ctrl->ctrl_fmt =
 278                 RQ_CTRL_SET(SIZE_8BYTES(sizeof(*ctrl)),  LEN) |
 279                 RQ_CTRL_SET(SIZE_8BYTES(sizeof(*cqe_sect)), COMPLETE_LEN) |
 280                 RQ_CTRL_SET(SIZE_8BYTES(sizeof(*buf_desc)), BUFDESC_SECT_LEN) |
 281                 RQ_CTRL_SET(RQ_COMPLETE_SGE, COMPLETE_FORMAT);
 282
 283         hinic_set_sge(&cqe_sect->sge, cqe_dma, rq_ceq_len);
 284
 285         buf_desc->addr_high = upper_32_bits(buf_addr);
 286         buf_desc->addr_low = lower_32_bits(buf_addr);
 287 }
 288
 289 void hinic_rxq_get_stats(struct hinic_rxq *rxq, struct hinic_rxq_stats *stats)
 290 {
 291         if (!rxq || !stats)
 292                 return;
 293
 294         memcpy(stats, &rxq->rxq_stats, sizeof(rxq->rxq_stats));
 295 }
 296
 297 void hinic_rxq_stats_reset(struct hinic_rxq *rxq)
 298 {
 299         struct hinic_rxq_stats *rxq_stats;
 300
 301         if (rxq == NULL)
 302                 return;
 303
 304         rxq_stats = &rxq->rxq_stats;
 305         memset(rxq_stats, 0, sizeof(*rxq_stats));
 306 }
 307
 308 static int hinic_rx_alloc_cqe(struct hinic_rxq *rxq)
 309 {
 310         size_t cqe_mem_size;
 311
 312         cqe_mem_size = sizeof(struct hinic_rq_cqe) * rxq->q_depth;
 313         rxq->cqe_start_vaddr =
 314                 dma_zalloc_coherent(rxq->nic_dev->hwdev,
 315                                     cqe_mem_size, &rxq->cqe_start_paddr,
 316                                     GFP_KERNEL);
 317         if (!rxq->cqe_start_vaddr) {
 318                 PMD_DRV_LOG(ERR, "Allocate cqe dma memory failed");
 319                 return -ENOMEM;
 320         }
 321
 322         rxq->rx_cqe = (struct hinic_rq_cqe *)rxq->cqe_start_vaddr;
 323
 324         return HINIC_OK;
 325 }
 326
 327 static void hinic_rx_free_cqe(struct hinic_rxq *rxq)
 328 {
 329         size_t cqe_mem_size;
 330
 331         cqe_mem_size = sizeof(struct hinic_rq_cqe) * rxq->q_depth;
 332         dma_free_coherent(rxq->nic_dev->hwdev, cqe_mem_size,
 333                           rxq->cqe_start_vaddr, rxq->cqe_start_paddr);
 334         rxq->cqe_start_vaddr = NULL;
 335 }
 336
 337 static int hinic_rx_fill_wqe(struct hinic_rxq *rxq)
 338 {
 339         struct hinic_nic_dev *nic_dev = rxq->nic_dev;
 340         struct hinic_rq_wqe *rq_wqe;
 341         dma_addr_t buf_dma_addr, cqe_dma_addr;
 342         u16 pi = 0;
 343         int i;
 344
 345         buf_dma_addr = 0;
 346         cqe_dma_addr = rxq->cqe_start_paddr;
 347         for (i = 0; i < rxq->q_depth; i++) {
 348                 rq_wqe = hinic_get_rq_wqe(nic_dev->hwdev, rxq->q_id, &pi);
 349                 if (!rq_wqe) {
 350                         PMD_DRV_LOG(ERR, "Get rq wqe failed");
 351                         break;
 352                 }
 353
 354                 hinic_prepare_rq_wqe(rq_wqe, pi, buf_dma_addr, cqe_dma_addr);
 355                 cqe_dma_addr +=  sizeof(struct hinic_rq_cqe);
 356
 357                 hinic_cpu_to_be32(rq_wqe, sizeof(struct hinic_rq_wqe));
 358         }
 359
 360         hinic_return_rq_wqe(nic_dev->hwdev, rxq->q_id, i);
 361
 362         return i;
 363 }
 364
 365 /* alloc cqe and prepare rqe */
 366 int hinic_setup_rx_resources(struct hinic_rxq *rxq)
 367 {
 368         u64 rx_info_sz;
 369         int err, pkts;
 370
 371         rx_info_sz = rxq->q_depth * sizeof(*rxq->rx_info);
 372         rxq->rx_info = kzalloc_aligned(rx_info_sz, GFP_KERNEL);
 373         if (!rxq->rx_info)
 374                 return -ENOMEM;
 375
 376         err = hinic_rx_alloc_cqe(rxq);
 377         if (err) {
 378                 PMD_DRV_LOG(ERR, "Allocate rx cqe failed");
 379                 goto rx_cqe_err;
 380         }
 381
 382         pkts = hinic_rx_fill_wqe(rxq);
 383         if (pkts != rxq->q_depth) {
 384                 PMD_DRV_LOG(ERR, "Fill rx wqe failed");
 385                 err = -ENOMEM;
 386                 goto rx_fill_err;
 387         }
 388
 389         return 0;
 390
 391 rx_fill_err:
 392         hinic_rx_free_cqe(rxq);
 393
 394 rx_cqe_err:
 395         kfree(rxq->rx_info);
 396         rxq->rx_info = NULL;
 397
 398         return err;
 399 }
 400
 401 void hinic_free_rx_resources(struct hinic_rxq *rxq)
 402 {
 403         if (rxq->rx_info == NULL)
 404                 return;
 405
 406         hinic_rx_free_cqe(rxq);
 407         kfree(rxq->rx_info);
 408         rxq->rx_info = NULL;
 409 }
 410
 411 void hinic_free_all_rx_resources(struct rte_eth_dev *eth_dev)
 412 {
 413         u16 q_id;
 414         struct hinic_nic_dev *nic_dev =
 415                                 HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
 416
 417         for (q_id = 0; q_id < nic_dev->num_rq; q_id++) {
 418                 eth_dev->data->rx_queues[q_id] = NULL;
 419
 420                 if (nic_dev->rxqs[q_id] == NULL)
 421                         continue;
 422
 423                 hinic_free_all_rx_mbufs(nic_dev->rxqs[q_id]);
 424                 hinic_free_rx_resources(nic_dev->rxqs[q_id]);
 425                 kfree(nic_dev->rxqs[q_id]);
 426                 nic_dev->rxqs[q_id] = NULL;
 427         }
 428 }
 429
 430 void hinic_free_all_rx_mbuf(struct rte_eth_dev *eth_dev)
 431 {
 432         struct hinic_nic_dev *nic_dev =
 433                                 HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
 434         u16 q_id;
 435
 436         for (q_id = 0; q_id < nic_dev->num_rq; q_id++)
 437                 hinic_free_all_rx_mbufs(nic_dev->rxqs[q_id]);
 438 }
 439
 440 static void hinic_recv_jumbo_pkt(struct hinic_rxq *rxq,
 441                                  struct rte_mbuf *head_mbuf,
 442                                  u32 remain_pkt_len)
 443 {
 444         struct hinic_nic_dev *nic_dev = rxq->nic_dev;
 445         struct rte_mbuf *cur_mbuf, *rxm = NULL;
 446         struct hinic_rx_info *rx_info;
 447         u16 sw_ci, rx_buf_len = rxq->buf_len;
 448         u32 pkt_len;
 449
 450         while (remain_pkt_len > 0) {
 451                 sw_ci = hinic_get_rq_local_ci(nic_dev->hwdev, rxq->q_id);
 452                 rx_info = &rxq->rx_info[sw_ci];
 453
 454                 hinic_update_rq_local_ci(nic_dev->hwdev, rxq->q_id, 1);
 455
 456                 pkt_len = remain_pkt_len > rx_buf_len ?
 457                         rx_buf_len : remain_pkt_len;
 458                 remain_pkt_len -= pkt_len;
 459
 460                 cur_mbuf = rx_info->mbuf;
 461                 cur_mbuf->data_len = (u16)pkt_len;
 462                 cur_mbuf->next = NULL;
 463
 464                 head_mbuf->pkt_len += cur_mbuf->data_len;
 465                 head_mbuf->nb_segs++;
 466
 467                 if (!rxm)
 468                         head_mbuf->next = cur_mbuf;
 469                 else
 470                         rxm->next = cur_mbuf;
 471
 472                 rxm = cur_mbuf;
 473         }
 474 }
 475
 476 static void hinic_rss_deinit(struct hinic_nic_dev *nic_dev)
 477 {
 478         u8 prio_tc[HINIC_DCB_UP_MAX] = {0};
 479         (void)hinic_rss_cfg(nic_dev->hwdev, 0,
 480                             nic_dev->rss_tmpl_idx, 0, prio_tc);
 481 }
 482
 483 static int hinic_rss_key_init(struct hinic_nic_dev *nic_dev,
 484                               struct rte_eth_rss_conf *rss_conf)
 485 {
 486         u8 default_rss_key[HINIC_RSS_KEY_SIZE] = {
 487                          0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2,
 488                          0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0,
 489                          0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4,
 490                          0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c,
 491                          0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa};
 492         u8 hashkey[HINIC_RSS_KEY_SIZE] = {0};
 493         u8 tmpl_idx = nic_dev->rss_tmpl_idx;
 494
 495         if (rss_conf->rss_key == NULL)
 496                 memcpy(hashkey, default_rss_key, HINIC_RSS_KEY_SIZE);
 497         else
 498                 memcpy(hashkey, rss_conf->rss_key, rss_conf->rss_key_len);
 499
 500         return hinic_rss_set_template_tbl(nic_dev->hwdev, tmpl_idx, hashkey);
 501 }
 502
 503 static void hinic_fill_rss_type(struct nic_rss_type *rss_type,
 504                                 struct rte_eth_rss_conf *rss_conf)
 505 {
 506         u64 rss_hf = rss_conf->rss_hf;
 507
 508         rss_type->ipv4 = (rss_hf & (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4)) ? 1 : 0;
 509         rss_type->tcp_ipv4 = (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) ? 1 : 0;
 510         rss_type->ipv6 = (rss_hf & (ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6)) ? 1 : 0;
 511         rss_type->ipv6_ext = (rss_hf & ETH_RSS_IPV6_EX) ? 1 : 0;
 512         rss_type->tcp_ipv6 = (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP) ? 1 : 0;
 513         rss_type->tcp_ipv6_ext = (rss_hf & ETH_RSS_IPV6_TCP_EX) ? 1 : 0;
 514         rss_type->udp_ipv4 = (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) ? 1 : 0;
 515         rss_type->udp_ipv6 = (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP) ? 1 : 0;
 516 }
 517
 518 static void hinic_fillout_indir_tbl(struct hinic_nic_dev *nic_dev, u32 *indir)
 519 {
 520         u8 rss_queue_count = nic_dev->num_rss;
 521         int i = 0, j;
 522
 523         if (rss_queue_count == 0) {
 524                 /* delete q_id from indir tbl */
 525                 for (i = 0; i < HINIC_RSS_INDIR_SIZE; i++)
 526                         indir[i] = 0xFF;        /* Invalid value in indir tbl */
 527         } else {
 528                 while (i < HINIC_RSS_INDIR_SIZE)
 529                         for (j = 0; (j < rss_queue_count) &&
 530                              (i < HINIC_RSS_INDIR_SIZE); j++)
 531                                 indir[i++] = nic_dev->rx_queue_list[j];
 532         }
 533 }
 534
 535 static int hinic_rss_init(struct hinic_nic_dev *nic_dev,
 536                           __attribute__((unused)) u8 *rq2iq_map,
 537                           struct rte_eth_rss_conf *rss_conf)
 538 {
 539         u32 indir_tbl[HINIC_RSS_INDIR_SIZE] = {0};
 540         struct nic_rss_type rss_type = {0};
 541         u8 prio_tc[HINIC_DCB_UP_MAX] = {0};
 542         u8 tmpl_idx = 0xFF, num_tc = 0;
 543         int err;
 544
 545         tmpl_idx = nic_dev->rss_tmpl_idx;
 546
 547         err = hinic_rss_key_init(nic_dev, rss_conf);
 548         if (err)
 549                 return err;
 550
 551         if (!nic_dev->rss_indir_flag) {
 552                 hinic_fillout_indir_tbl(nic_dev, indir_tbl);
 553                 err = hinic_rss_set_indir_tbl(nic_dev->hwdev, tmpl_idx,
 554                                               indir_tbl);
 555                 if (err)
 556                         return err;
 557         }
 558
 559         hinic_fill_rss_type(&rss_type, rss_conf);
 560         err = hinic_set_rss_type(nic_dev->hwdev, tmpl_idx, rss_type);
 561         if (err)
 562                 return err;
 563
 564         err = hinic_rss_set_hash_engine(nic_dev->hwdev, tmpl_idx,
 565                                         HINIC_RSS_HASH_ENGINE_TYPE_TOEP);
 566         if (err)
 567                 return err;
 568
 569         return hinic_rss_cfg(nic_dev->hwdev, 1, tmpl_idx, num_tc, prio_tc);
 570 }
 571
 572 static void
 573 hinic_add_rq_to_rx_queue_list(struct hinic_nic_dev *nic_dev, u16 queue_id)
 574 {
 575         u8 rss_queue_count = nic_dev->num_rss;
 576
 577         RTE_ASSERT(rss_queue_count <= (RTE_DIM(nic_dev->rx_queue_list) - 1));
 578
 579         nic_dev->rx_queue_list[rss_queue_count] = queue_id;
 580         nic_dev->num_rss++;
 581 }
 582
 583 /**
 584  * hinic_setup_num_qps - determine num_qps from rss_tmpl_id
 585  * @nic_dev: pointer to the private ethernet device
 586  * Return: 0 on Success, error code otherwise.
 587  **/
 588 static int hinic_setup_num_qps(struct hinic_nic_dev *nic_dev)
 589 {
 590         int err, i;
 591
 592         if (!(nic_dev->flags & ETH_MQ_RX_RSS_FLAG)) {
 593                 nic_dev->flags &= ~ETH_MQ_RX_RSS_FLAG;
 594                 nic_dev->num_rss = 0;
 595                 if (nic_dev->num_rq > 1) {
 596                         /* get rss template id */
 597                         err = hinic_rss_template_alloc(nic_dev->hwdev,
 598                                                        &nic_dev->rss_tmpl_idx);
 599                         if (err) {
 600                                 PMD_DRV_LOG(WARNING, "Alloc rss template failed");
 601                                 return err;
 602                         }
 603                         nic_dev->flags |= ETH_MQ_RX_RSS_FLAG;
 604                         for (i = 0; i < nic_dev->num_rq; i++)
 605                                 hinic_add_rq_to_rx_queue_list(nic_dev, i);
 606                 }
 607         }
 608
 609         return 0;
 610 }
 611
 612 static void hinic_destroy_num_qps(struct hinic_nic_dev *nic_dev)
 613 {
 614         if (nic_dev->flags & ETH_MQ_RX_RSS_FLAG) {
 615                 if (hinic_rss_template_free(nic_dev->hwdev,
 616                                             nic_dev->rss_tmpl_idx))
 617                         PMD_DRV_LOG(WARNING, "Free rss template failed");
 618
 619                 nic_dev->flags &= ~ETH_MQ_RX_RSS_FLAG;
 620         }
 621 }
 622
 623 static int hinic_config_mq_rx_rss(struct hinic_nic_dev *nic_dev, bool on)
 624 {
 625         int ret = 0;
 626
 627         if (on) {
 628                 ret = hinic_setup_num_qps(nic_dev);
 629                 if (ret)
 630                         PMD_DRV_LOG(ERR, "Setup num_qps failed");
 631         } else {
 632                 hinic_destroy_num_qps(nic_dev);
 633         }
 634
 635         return ret;
 636 }
 637
 638 int hinic_config_mq_mode(struct rte_eth_dev *dev, bool on)
 639 {
 640         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 641         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 642         int ret = 0;
 643
 644         switch (dev_conf->rxmode.mq_mode) {
 645         case ETH_MQ_RX_RSS:
 646                 ret = hinic_config_mq_rx_rss(nic_dev, on);
 647                 break;
 648         default:
 649                 break;
 650         }
 651
 652         return ret;
 653 }
 654
 655 int hinic_rx_configure(struct rte_eth_dev *dev)
 656 {
 657         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 658         struct rte_eth_rss_conf rss_conf =
 659                 dev->data->dev_conf.rx_adv_conf.rss_conf;
 660         int err;
 661
 662         if (nic_dev->flags & ETH_MQ_RX_RSS_FLAG) {
 663                 if (rss_conf.rss_hf == 0) {
 664                         rss_conf.rss_hf = HINIC_RSS_OFFLOAD_ALL;
 665                 } else if ((rss_conf.rss_hf & HINIC_RSS_OFFLOAD_ALL) == 0) {
 666                         PMD_DRV_LOG(ERR, "Do not support rss offload all");
 667                         goto rss_config_err;
 668                 }
 669
 670                 err = hinic_rss_init(nic_dev, NULL, &rss_conf);
 671                 if (err) {
 672                         PMD_DRV_LOG(ERR, "Init rss failed");
 673                         goto rss_config_err;
 674                 }
 675         }
 676
 677         /* Enable both L3/L4 rx checksum offload */
 678         if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_CHECKSUM)
 679                 nic_dev->rx_csum_en = HINIC_RX_CSUM_OFFLOAD_EN;
 680
 681         err = hinic_set_rx_csum_offload(nic_dev->hwdev,
 682                                         HINIC_RX_CSUM_OFFLOAD_EN);
 683         if (err)
 684                 goto rx_csum_ofl_err;
 685
 686         return 0;
 687
 688 rx_csum_ofl_err:
 689 rss_config_err:
 690         hinic_destroy_num_qps(nic_dev);
 691
 692         return HINIC_ERROR;
 693 }
 694
 695 void hinic_rx_remove_configure(struct rte_eth_dev *dev)
 696 {
 697         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 698
 699         if (nic_dev->flags & ETH_MQ_RX_RSS_FLAG) {
 700                 hinic_rss_deinit(nic_dev);
 701                 hinic_destroy_num_qps(nic_dev);
 702         }
 703 }
 704
 705 void hinic_free_all_rx_mbufs(struct hinic_rxq *rxq)
 706 {
 707         struct hinic_nic_dev *nic_dev = rxq->nic_dev;
 708         struct hinic_rx_info *rx_info;
 709         int free_wqebbs =
 710                 hinic_get_rq_free_wqebbs(nic_dev->hwdev, rxq->q_id) + 1;
 711         volatile struct hinic_rq_cqe *rx_cqe;
 712         u16 ci;
 713
 714         while (free_wqebbs++ < rxq->q_depth) {
 715                 ci = hinic_get_rq_local_ci(nic_dev->hwdev, rxq->q_id);
 716
 717                 rx_cqe = &rxq->rx_cqe[ci];
 718
 719                 /* clear done bit */
 720                 rx_cqe->status = 0;
 721
 722                 rx_info = &rxq->rx_info[ci];
 723                 rte_pktmbuf_free(rx_info->mbuf);
 724                 rx_info->mbuf = NULL;
 725
 726                 hinic_update_rq_local_ci(nic_dev->hwdev, rxq->q_id, 1);
 727         }
 728 }
 729
 730 static inline void hinic_rq_cqe_be_to_cpu32(void *dst_le32,
 731                                             volatile void *src_be32)
 732 {
 733 #if defined(__X86_64_SSE__)
 734         volatile __m128i *wqe_be = (volatile __m128i *)src_be32;
 735         __m128i *wqe_le = (__m128i *)dst_le32;
 736         __m128i shuf_mask =  _mm_set_epi8(12, 13, 14, 15, 8, 9, 10,
 737                                         11, 4, 5, 6, 7, 0, 1, 2, 3);
 738
 739         /* l2nic just use first 128 bits */
 740         wqe_le[0] = _mm_shuffle_epi8(wqe_be[0], shuf_mask);
 741 #elif defined(__ARM64_NEON__)
 742         volatile uint8x16_t *wqe_be = (volatile uint8x16_t *)src_be32;
 743         uint8x16_t *wqe_le = (uint8x16_t *)dst_le32;
 744         const uint8x16_t shuf_mask = {3, 2, 1, 0, 7, 6, 5, 4, 11, 10,
 745                                         9, 8, 15, 14, 13, 12};
 746
 747         /* l2nic just use first 128 bits */
 748         wqe_le[0] = vqtbl1q_u8(wqe_be[0], shuf_mask);
 749 #else
 750         u32 i;
 751         volatile u32 *wqe_be = (volatile u32 *)src_be32;
 752         u32 *wqe_le = (u32 *)dst_le32;
 753
 754 #define HINIC_L2NIC_RQ_CQE_USED         4 /* 4Bytes unit */
 755
 756         for (i = 0; i < HINIC_L2NIC_RQ_CQE_USED; i++) {
 757                 *wqe_le = rte_be_to_cpu_32(*wqe_be);
 758                 wqe_be++;
 759                 wqe_le++;
 760         }
 761 #endif
 762 }
 763
 764 static inline uint64_t hinic_rx_rss_hash(uint32_t offload_type,
 765                                          uint32_t cqe_hass_val,
 766                                          uint32_t *rss_hash)
 767 {
 768         uint32_t rss_type;
 769
 770         rss_type = HINIC_GET_RSS_TYPES(offload_type);
 771         if (likely(rss_type != 0)) {
 772                 *rss_hash = cqe_hass_val;
 773                 return PKT_RX_RSS_HASH;
 774         }
 775
 776         return 0;
 777 }
 778
 779 static inline uint64_t hinic_rx_csum(uint32_t status, struct hinic_rxq *rxq)
 780 {
 781         uint32_t checksum_err;
 782         uint64_t flags;
 783         struct hinic_nic_dev *nic_dev = rxq->nic_dev;
 784
 785         if (unlikely(!(nic_dev->rx_csum_en & HINIC_RX_CSUM_OFFLOAD_EN)))
 786                 return PKT_RX_IP_CKSUM_UNKNOWN;
 787
 788         /* most case checksum is ok */
 789         checksum_err = HINIC_GET_RX_CSUM_ERR(status);
 790         if (likely(checksum_err == 0))
 791                 return (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD);
 792
 793         /* If BYPASS bit set, all other status indications should be ignored */
 794         if (unlikely(HINIC_CSUM_ERR_BYPASSED(checksum_err)))
 795                 return PKT_RX_IP_CKSUM_UNKNOWN;
 796
 797         flags = 0;
 798
 799         /* IP checksum error */
 800         if (HINIC_CSUM_ERR_IP(checksum_err))
 801                 flags |= PKT_RX_IP_CKSUM_BAD;
 802         else
 803                 flags |= PKT_RX_IP_CKSUM_GOOD;
 804
 805         /* L4 checksum error */
 806         if (HINIC_CSUM_ERR_L4(checksum_err))
 807                 flags |= PKT_RX_L4_CKSUM_BAD;
 808         else
 809                 flags |= PKT_RX_L4_CKSUM_GOOD;
 810
 811         if (unlikely(HINIC_CSUM_ERR_OTHER(checksum_err)))
 812                 flags = PKT_RX_L4_CKSUM_NONE;
 813
 814         rxq->rxq_stats.errors++;
 815
 816         return flags;
 817 }
 818
 819 static inline uint64_t hinic_rx_vlan(uint32_t offload_type, uint32_t vlan_len,
 820                                      uint16_t *vlan_tci)
 821 {
 822         uint16_t vlan_tag;
 823
 824         vlan_tag = HINIC_GET_RX_VLAN_TAG(vlan_len);
 825         if (!HINIC_GET_RX_VLAN_OFFLOAD_EN(offload_type) || 0 == vlan_tag) {
 826                 *vlan_tci = 0;
 827                 return 0;
 828         }
 829
 830         *vlan_tci = vlan_tag;
 831
 832         return PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED;
 833 }
 834
 835 static inline u32 hinic_rx_alloc_mbuf_bulk(struct hinic_rxq *rxq,
 836                                            struct rte_mbuf **mbufs,
 837                                            u32 exp_mbuf_cnt)
 838 {
 839         int rc;
 840         u32 avail_cnt;
 841
 842         rc = rte_pktmbuf_alloc_bulk(rxq->mb_pool, mbufs, exp_mbuf_cnt);
 843         if (likely(rc == HINIC_OK)) {
 844                 avail_cnt = exp_mbuf_cnt;
 845         } else {
 846                 avail_cnt = 0;
 847                 rxq->rxq_stats.rx_nombuf += exp_mbuf_cnt;
 848         }
 849
 850         return avail_cnt;
 851 }
 852
 853 static struct rte_mbuf *hinic_rx_alloc_mbuf(struct hinic_rxq *rxq,
 854                                         dma_addr_t *dma_addr)
 855 {
 856         struct rte_mbuf *mbuf;
 857
 858         mbuf = rte_mbuf_raw_alloc(rxq->mb_pool);
 859         if (unlikely(!mbuf))
 860                 return NULL;
 861
 862         *dma_addr = rte_mbuf_data_iova_default(mbuf);
 863
 864         return mbuf;
 865 }
 866
 867 static inline void hinic_rearm_rxq_mbuf(struct hinic_rxq *rxq)
 868 {
 869         u16 pi;
 870         u32 i, free_wqebbs, rearm_wqebbs, exp_wqebbs;
 871         dma_addr_t dma_addr;
 872         struct hinic_rq_wqe *rq_wqe;
 873         struct rte_mbuf **rearm_mbufs;
 874
 875         /* check free wqebb fo rearm */
 876         free_wqebbs = HINIC_GET_RQ_FREE_WQEBBS(rxq);
 877         if (unlikely(free_wqebbs < rxq->rx_free_thresh))
 878                 return;
 879
 880         /* get rearm mbuf array */
 881         pi = HINIC_GET_RQ_LOCAL_PI(rxq);
 882         rearm_mbufs = (struct rte_mbuf **)(&rxq->rx_info[pi]);
 883
 884         /* check rxq free wqebbs turn around */
 885         exp_wqebbs = rxq->q_depth - pi;
 886         if (free_wqebbs < exp_wqebbs)
 887                 exp_wqebbs = free_wqebbs;
 888
 889         /* alloc mbuf in bulk */
 890         rearm_wqebbs = hinic_rx_alloc_mbuf_bulk(rxq, rearm_mbufs, exp_wqebbs);
 891         if (unlikely(rearm_wqebbs == 0))
 892                 return;
 893
 894         /* rearm rx mbuf */
 895         rq_wqe = WQ_WQE_ADDR(rxq->wq, (u32)pi);
 896         for (i = 0; i < rearm_wqebbs; i++) {
 897                 dma_addr = rte_mbuf_data_iova_default(rearm_mbufs[i]);
 898                 rq_wqe->buf_desc.addr_high =
 899                                         cpu_to_be32(upper_32_bits(dma_addr));
 900                 rq_wqe->buf_desc.addr_low =
 901                                         cpu_to_be32(lower_32_bits(dma_addr));
 902                 rq_wqe++;
 903         }
 904         rxq->wq->prod_idx += rearm_wqebbs;
 905         rxq->wq->delta -= rearm_wqebbs;
 906
 907         /* update rq hw_pi */
 908         rte_wmb();
 909         HINIC_UPDATE_RQ_HW_PI(rxq, pi + rearm_wqebbs);
 910 }
 911
 912 void hinic_rx_alloc_pkts(struct hinic_rxq *rxq)
 913 {
 914         struct hinic_nic_dev *nic_dev = rxq->nic_dev;
 915         struct hinic_rq_wqe *rq_wqe;
 916         struct hinic_rx_info *rx_info;
 917         struct rte_mbuf *mb;
 918         dma_addr_t dma_addr;
 919         u16 pi = 0;
 920         int i, free_wqebbs;
 921
 922         free_wqebbs = HINIC_GET_RQ_FREE_WQEBBS(rxq);
 923         for (i = 0; i < free_wqebbs; i++) {
 924                 mb = hinic_rx_alloc_mbuf(rxq, &dma_addr);
 925                 if (unlikely(!mb)) {
 926                         rxq->rxq_stats.rx_nombuf++;
 927                         break;
 928                 }
 929
 930                 rq_wqe = hinic_get_rq_wqe(nic_dev->hwdev, rxq->q_id, &pi);
 931                 if (unlikely(!rq_wqe)) {
 932                         rte_pktmbuf_free(mb);
 933                         break;
 934                 }
 935
 936                 /* fill buffer address only */
 937                 rq_wqe->buf_desc.addr_high =
 938                                 cpu_to_be32(upper_32_bits(dma_addr));
 939                 rq_wqe->buf_desc.addr_low =
 940                                 cpu_to_be32(lower_32_bits(dma_addr));
 941
 942                 rx_info = &rxq->rx_info[pi];
 943                 rx_info->mbuf = mb;
 944         }
 945
 946         if (likely(i > 0)) {
 947                 rte_wmb();
 948                 HINIC_UPDATE_RQ_HW_PI(rxq, pi + 1);
 949         }
 950 }
 951
 952 u16 hinic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, u16 nb_pkts)
 953 {
 954         struct rte_mbuf *rxm;
 955         struct hinic_rxq *rxq = rx_queue;
 956         struct hinic_rx_info *rx_info;
 957         volatile struct hinic_rq_cqe *rx_cqe;
 958         u16 rx_buf_len, pkts = 0;
 959         u16 sw_ci, ci_mask, wqebb_cnt = 0;
 960         u32 pkt_len, status, vlan_len;
 961         u64 rx_bytes = 0;
 962         struct hinic_rq_cqe cqe;
 963         u32 offload_type, rss_hash;
 964
 965         rx_buf_len = rxq->buf_len;
 966
 967         /* 1. get polling start ci */
 968         ci_mask = HINIC_GET_RQ_WQE_MASK(rxq);
 969         sw_ci = HINIC_GET_RQ_LOCAL_CI(rxq);
 970
 971         while (pkts < nb_pkts) {
 972                  /* 2. current ci is done */
 973                 rx_cqe = &rxq->rx_cqe[sw_ci];
 974                 status = __atomic_load_n(&rx_cqe->status, __ATOMIC_ACQUIRE);
 975                 if (!HINIC_GET_RX_DONE_BE(status))
 976                         break;
 977
 978                 /* convert cqe and get packet length */
 979                 hinic_rq_cqe_be_to_cpu32(&cqe, (volatile void *)rx_cqe);
 980                 vlan_len = cqe.vlan_len;
 981
 982                 rx_info = &rxq->rx_info[sw_ci];
 983                 rxm = rx_info->mbuf;
 984
 985                 /* 3. next ci point and prefetch */
 986                 sw_ci++;
 987                 sw_ci &= ci_mask;
 988
 989                 /* prefetch next mbuf first 64B */
 990                 rte_prefetch0(rxq->rx_info[sw_ci].mbuf);
 991
 992                 /* 4. jumbo frame process */
 993                 pkt_len = HINIC_GET_RX_PKT_LEN(vlan_len);
 994                 if (likely(pkt_len <= rx_buf_len)) {
 995                         rxm->data_len = pkt_len;
 996                         rxm->pkt_len = pkt_len;
 997                         wqebb_cnt++;
 998                 } else {
 999                         rxm->data_len = rx_buf_len;
1000                         rxm->pkt_len = rx_buf_len;
1001
1002                         /* if receive jumbo, updating ci will be done by
1003                          * hinic_recv_jumbo_pkt function.
1004                          */
1005                         HINIC_UPDATE_RQ_LOCAL_CI(rxq, wqebb_cnt + 1);
1006                         wqebb_cnt = 0;
1007                         hinic_recv_jumbo_pkt(rxq, rxm, pkt_len - rx_buf_len);
1008                         sw_ci = HINIC_GET_RQ_LOCAL_CI(rxq);
1009                 }
1010
1011                 /* 5. vlan/checksum/rss/pkt_type/gro offload */
1012                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1013                 rxm->port = rxq->port_id;
1014                 offload_type = cqe.offload_type;
1015
1016                 /* vlan offload */
1017                 rxm->ol_flags |= hinic_rx_vlan(offload_type, vlan_len,
1018                                                &rxm->vlan_tci);
1019
1020                 /* checksum offload */
1021                 rxm->ol_flags |= hinic_rx_csum(cqe.status, rxq);
1022
1023                 /* rss hash offload */
1024                 rss_hash = cqe.rss_hash;
1025                 rxm->ol_flags |= hinic_rx_rss_hash(offload_type, rss_hash,
1026                                                    &rxm->hash.rss);
1027
1028                 /* 6. clear done bit */
1029                 rx_cqe->status = 0;
1030
1031                 rx_bytes += pkt_len;
1032                 rx_pkts[pkts++] = rxm;
1033         }
1034
1035         if (pkts) {
1036                 /* 7. update ci */
1037                 HINIC_UPDATE_RQ_LOCAL_CI(rxq, wqebb_cnt);
1038
1039                 /* do packet stats */
1040                 rxq->rxq_stats.packets += pkts;
1041                 rxq->rxq_stats.bytes += rx_bytes;
1042         }
1043         rxq->rxq_stats.burst_pkts = pkts;
1044
1045         /* 8. rearm mbuf to rxq */
1046         hinic_rearm_rxq_mbuf(rxq);
1047
1048         return pkts;
1049 }