drivers/net/hinic/hinic_pmd_rx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Huawei Technologies Co., Ltd
   3  */
   4
   5 #include <rte_ether.h>
   6 #include <rte_mbuf.h>
   7 #ifdef __ARM64_NEON__
   8 #include <arm_neon.h>
   9 #endif
  10
  11 #include "base/hinic_compat.h"
  12 #include "base/hinic_pmd_hwdev.h"
  13 #include "base/hinic_pmd_wq.h"
  14 #include "base/hinic_pmd_niccfg.h"
  15 #include "base/hinic_pmd_nicio.h"
  16 #include "hinic_pmd_ethdev.h"
  17 #include "hinic_pmd_rx.h"
  18
  19 /* rxq wq operations */
  20 #define HINIC_GET_RQ_WQE_MASK(rxq)      \
  21         ((rxq)->wq->mask)
  22
  23 #define HINIC_GET_RQ_LOCAL_CI(rxq)      \
  24         (((rxq)->wq->cons_idx) & HINIC_GET_RQ_WQE_MASK(rxq))
  25
  26 #define HINIC_GET_RQ_LOCAL_PI(rxq)      \
  27         (((rxq)->wq->prod_idx) & HINIC_GET_RQ_WQE_MASK(rxq))
  28
  29 #define HINIC_UPDATE_RQ_LOCAL_CI(rxq, wqebb_cnt)        \
  30         do {                                            \
  31                 (rxq)->wq->cons_idx += (wqebb_cnt);     \
  32                 (rxq)->wq->delta += (wqebb_cnt);        \
  33         } while (0)
  34
  35 #define HINIC_UPDATE_RQ_HW_PI(rxq, pi)  \
  36         (*((rxq)->pi_virt_addr) =       \
  37                 cpu_to_be16((pi) & HINIC_GET_RQ_WQE_MASK(rxq)))
  38
  39 #define HINIC_GET_RQ_FREE_WQEBBS(rxq)   ((rxq)->wq->delta - 1)
  40
  41 /* rxq cqe done and status bit */
  42 #define HINIC_GET_RX_DONE_BE(status)    \
  43         ((status) & 0x80U)
  44
  45 #define HINIC_RX_CSUM_OFFLOAD_EN        0xFFF
  46
  47 #define RQ_CQE_SGE_VLAN_SHIFT                   0
  48 #define RQ_CQE_SGE_LEN_SHIFT                    16
  49
  50 #define RQ_CQE_SGE_VLAN_MASK                    0xFFFFU
  51 #define RQ_CQE_SGE_LEN_MASK                     0xFFFFU
  52
  53 #define RQ_CQE_SGE_GET(val, member)             \
  54         (((val) >> RQ_CQE_SGE_##member##_SHIFT) & RQ_CQE_SGE_##member##_MASK)
  55
  56 #define HINIC_GET_RX_VLAN_TAG(vlan_len) \
  57                 RQ_CQE_SGE_GET(vlan_len, VLAN)
  58
  59 #define HINIC_GET_RX_PKT_LEN(vlan_len)  \
  60                 RQ_CQE_SGE_GET(vlan_len, LEN)
  61
  62 #define RQ_CQE_STATUS_CSUM_ERR_SHIFT            0
  63 #define RQ_CQE_STATUS_NUM_LRO_SHIFT             16
  64 #define RQ_CQE_STATUS_LRO_PUSH_SHIFT            25
  65 #define RQ_CQE_STATUS_LRO_ENTER_SHIFT           26
  66 #define RQ_CQE_STATUS_LRO_INTR_SHIFT            27
  67
  68 #define RQ_CQE_STATUS_BP_EN_SHIFT               30
  69 #define RQ_CQE_STATUS_RXDONE_SHIFT              31
  70 #define RQ_CQE_STATUS_FLUSH_SHIFT               28
  71
  72 #define RQ_CQE_STATUS_CSUM_ERR_MASK             0xFFFFU
  73 #define RQ_CQE_STATUS_NUM_LRO_MASK              0xFFU
  74 #define RQ_CQE_STATUS_LRO_PUSH_MASK             0X1U
  75 #define RQ_CQE_STATUS_LRO_ENTER_MASK            0X1U
  76 #define RQ_CQE_STATUS_LRO_INTR_MASK             0X1U
  77 #define RQ_CQE_STATUS_BP_EN_MASK                0X1U
  78 #define RQ_CQE_STATUS_RXDONE_MASK               0x1U
  79 #define RQ_CQE_STATUS_FLUSH_MASK                0x1U
  80
  81 #define RQ_CQE_STATUS_GET(val, member)          \
  82                 (((val) >> RQ_CQE_STATUS_##member##_SHIFT) & \
  83                                 RQ_CQE_STATUS_##member##_MASK)
  84
  85 #define RQ_CQE_STATUS_CLEAR(val, member)        \
  86                 ((val) & (~(RQ_CQE_STATUS_##member##_MASK << \
  87                                 RQ_CQE_STATUS_##member##_SHIFT)))
  88
  89 #define HINIC_GET_RX_CSUM_ERR(status)   \
  90                 RQ_CQE_STATUS_GET(status, CSUM_ERR)
  91
  92 #define HINIC_GET_RX_DONE(status)       \
  93                 RQ_CQE_STATUS_GET(status, RXDONE)
  94
  95 #define HINIC_GET_RX_FLUSH(status)      \
  96                 RQ_CQE_STATUS_GET(status, FLUSH)
  97
  98 #define HINIC_GET_RX_BP_EN(status)      \
  99                 RQ_CQE_STATUS_GET(status, BP_EN)
 100
 101 #define HINIC_GET_RX_NUM_LRO(status)    \
 102                 RQ_CQE_STATUS_GET(status, NUM_LRO)
 103
 104 /* RQ_CTRL */
 105 #define RQ_CTRL_BUFDESC_SECT_LEN_SHIFT          0
 106 #define RQ_CTRL_COMPLETE_FORMAT_SHIFT           15
 107 #define RQ_CTRL_COMPLETE_LEN_SHIFT              27
 108 #define RQ_CTRL_LEN_SHIFT                       29
 109
 110 #define RQ_CTRL_BUFDESC_SECT_LEN_MASK           0xFFU
 111 #define RQ_CTRL_COMPLETE_FORMAT_MASK            0x1U
 112 #define RQ_CTRL_COMPLETE_LEN_MASK               0x3U
 113 #define RQ_CTRL_LEN_MASK                        0x3U
 114
 115 #define RQ_CTRL_SET(val, member)                \
 116         (((val) & RQ_CTRL_##member##_MASK) << RQ_CTRL_##member##_SHIFT)
 117
 118 #define RQ_CTRL_GET(val, member)                \
 119         (((val) >> RQ_CTRL_##member##_SHIFT) & RQ_CTRL_##member##_MASK)
 120
 121 #define RQ_CTRL_CLEAR(val, member)              \
 122         ((val) & (~(RQ_CTRL_##member##_MASK << RQ_CTRL_##member##_SHIFT)))
 123
 124 #define RQ_CQE_PKT_NUM_SHIFT                    1
 125 #define RQ_CQE_PKT_FIRST_LEN_SHIFT              19
 126 #define RQ_CQE_PKT_LAST_LEN_SHIFT               6
 127 #define RQ_CQE_SUPER_CQE_EN_SHIFT               0
 128
 129 #define RQ_CQE_PKT_FIRST_LEN_MASK               0x1FFFU
 130 #define RQ_CQE_PKT_LAST_LEN_MASK                0x1FFFU
 131 #define RQ_CQE_PKT_NUM_MASK                     0x1FU
 132 #define RQ_CQE_SUPER_CQE_EN_MASK                0x1
 133
 134 #define RQ_CQE_PKT_NUM_GET(val, member)         \
 135         (((val) >> RQ_CQE_PKT_##member##_SHIFT) & RQ_CQE_PKT_##member##_MASK)
 136
 137 #define HINIC_GET_RQ_CQE_PKT_NUM(pkt_info) RQ_CQE_PKT_NUM_GET(pkt_info, NUM)
 138
 139 #define RQ_CQE_SUPER_CQE_EN_GET(val, member)    \
 140         (((val) >> RQ_CQE_##member##_SHIFT) & RQ_CQE_##member##_MASK)
 141
 142 #define HINIC_GET_SUPER_CQE_EN(pkt_info)        \
 143         RQ_CQE_SUPER_CQE_EN_GET(pkt_info, SUPER_CQE_EN)
 144
 145 #define RQ_CQE_OFFOLAD_TYPE_VLAN_EN_SHIFT               21
 146 #define RQ_CQE_OFFOLAD_TYPE_VLAN_EN_MASK                0x1U
 147
 148 #define RQ_CQE_OFFOLAD_TYPE_PKT_TYPE_SHIFT              0
 149 #define RQ_CQE_OFFOLAD_TYPE_PKT_TYPE_MASK               0xFFFU
 150
 151 #define RQ_CQE_OFFOLAD_TYPE_PKT_UMBCAST_SHIFT           19
 152 #define RQ_CQE_OFFOLAD_TYPE_PKT_UMBCAST_MASK            0x3U
 153
 154 #define RQ_CQE_OFFOLAD_TYPE_RSS_TYPE_SHIFT              24
 155 #define RQ_CQE_OFFOLAD_TYPE_RSS_TYPE_MASK               0xFFU
 156
 157 #define RQ_CQE_OFFOLAD_TYPE_GET(val, member)            (((val) >> \
 158                                 RQ_CQE_OFFOLAD_TYPE_##member##_SHIFT) & \
 159                                 RQ_CQE_OFFOLAD_TYPE_##member##_MASK)
 160
 161 #define HINIC_GET_RX_VLAN_OFFLOAD_EN(offload_type)      \
 162                 RQ_CQE_OFFOLAD_TYPE_GET(offload_type, VLAN_EN)
 163
 164 #define HINIC_GET_RSS_TYPES(offload_type)       \
 165                 RQ_CQE_OFFOLAD_TYPE_GET(offload_type, RSS_TYPE)
 166
 167 #define HINIC_GET_RX_PKT_TYPE(offload_type)     \
 168                 RQ_CQE_OFFOLAD_TYPE_GET(offload_type, PKT_TYPE)
 169
 170 #define HINIC_GET_RX_PKT_UMBCAST(offload_type)  \
 171                 RQ_CQE_OFFOLAD_TYPE_GET(offload_type, PKT_UMBCAST)
 172
 173 #define RQ_CQE_STATUS_CSUM_BYPASS_VAL                   0x80U
 174 #define RQ_CQE_STATUS_CSUM_ERR_IP_MASK                  0x39U
 175 #define RQ_CQE_STATUS_CSUM_ERR_L4_MASK                  0x46U
 176 #define RQ_CQE_STATUS_CSUM_ERR_OTHER                    0x100U
 177
 178 #define HINIC_CSUM_ERR_BYPASSED(csum_err)        \
 179         ((csum_err) == RQ_CQE_STATUS_CSUM_BYPASS_VAL)
 180
 181 #define HINIC_CSUM_ERR_IP(csum_err)      \
 182         ((csum_err) & RQ_CQE_STATUS_CSUM_ERR_IP_MASK)
 183
 184 #define HINIC_CSUM_ERR_L4(csum_err)      \
 185         ((csum_err) & RQ_CQE_STATUS_CSUM_ERR_L4_MASK)
 186
 187 #define HINIC_CSUM_ERR_OTHER(csum_err)   \
 188         ((csum_err) == RQ_CQE_STATUS_CSUM_ERR_OTHER)
 189
 190
 191 void hinic_get_func_rx_buf_size(struct hinic_nic_dev *nic_dev)
 192 {
 193         struct hinic_rxq *rxq;
 194         u16 q_id;
 195         u16 buf_size = 0;
 196
 197         for (q_id = 0; q_id < nic_dev->num_rq; q_id++) {
 198                 rxq = nic_dev->rxqs[q_id];
 199
 200                 if (rxq == NULL)
 201                         continue;
 202
 203                 if (q_id == 0)
 204                         buf_size = rxq->buf_len;
 205
 206                 buf_size = buf_size > rxq->buf_len ? rxq->buf_len : buf_size;
 207         }
 208
 209         nic_dev->hwdev->nic_io->rq_buf_size = buf_size;
 210 }
 211
 212 int hinic_create_rq(struct hinic_hwdev *hwdev, u16 q_id, u16 rq_depth)
 213 {
 214         int err;
 215         struct hinic_nic_io *nic_io = hwdev->nic_io;
 216         struct hinic_qp *qp = &nic_io->qps[q_id];
 217         struct hinic_rq *rq = &qp->rq;
 218
 219         /* in case of hardware still generate interrupt, do not use msix 0 */
 220         rq->msix_entry_idx = 1;
 221         rq->q_id = q_id;
 222         rq->rq_depth = rq_depth;
 223         nic_io->rq_depth = rq_depth;
 224
 225         err = hinic_wq_allocate(hwdev, &nic_io->rq_wq[q_id],
 226                                 HINIC_RQ_WQEBB_SHIFT, nic_io->rq_depth);
 227         if (err) {
 228                 PMD_DRV_LOG(ERR, "Failed to allocate WQ for RQ");
 229                 return err;
 230         }
 231         rq->wq = &nic_io->rq_wq[q_id];
 232
 233         rq->pi_virt_addr =
 234                 (volatile u16 *)dma_zalloc_coherent(hwdev, HINIC_PAGE_SIZE,
 235                                                     &rq->pi_dma_addr,
 236                                                     GFP_KERNEL);
 237         if (!rq->pi_virt_addr) {
 238                 PMD_DRV_LOG(ERR, "Failed to allocate rq pi virt addr");
 239                 err = -ENOMEM;
 240                 goto rq_pi_alloc_err;
 241         }
 242
 243         return HINIC_OK;
 244
 245 rq_pi_alloc_err:
 246         hinic_wq_free(hwdev, &nic_io->rq_wq[q_id]);
 247
 248         return err;
 249 }
 250
 251 void hinic_destroy_rq(struct hinic_hwdev *hwdev, u16 q_id)
 252 {
 253         struct hinic_nic_io *nic_io = hwdev->nic_io;
 254         struct hinic_qp *qp = &nic_io->qps[q_id];
 255         struct hinic_rq *rq = &qp->rq;
 256
 257         if (qp->rq.wq == NULL)
 258                 return;
 259
 260         dma_free_coherent_volatile(hwdev, HINIC_PAGE_SIZE,
 261                                    (volatile void *)rq->pi_virt_addr,
 262                                    rq->pi_dma_addr);
 263         hinic_wq_free(nic_io->hwdev, qp->rq.wq);
 264         qp->rq.wq = NULL;
 265 }
 266
 267 static void
 268 hinic_prepare_rq_wqe(void *wqe, __rte_unused u16 pi, dma_addr_t buf_addr,
 269                         dma_addr_t cqe_dma)
 270 {
 271         struct hinic_rq_wqe *rq_wqe = wqe;
 272         struct hinic_rq_ctrl *ctrl = &rq_wqe->ctrl;
 273         struct hinic_rq_cqe_sect *cqe_sect = &rq_wqe->cqe_sect;
 274         struct hinic_rq_bufdesc *buf_desc = &rq_wqe->buf_desc;
 275         u32 rq_ceq_len = sizeof(struct hinic_rq_cqe);
 276
 277         ctrl->ctrl_fmt =
 278                 RQ_CTRL_SET(SIZE_8BYTES(sizeof(*ctrl)),  LEN) |
 279                 RQ_CTRL_SET(SIZE_8BYTES(sizeof(*cqe_sect)), COMPLETE_LEN) |
 280                 RQ_CTRL_SET(SIZE_8BYTES(sizeof(*buf_desc)), BUFDESC_SECT_LEN) |
 281                 RQ_CTRL_SET(RQ_COMPLETE_SGE, COMPLETE_FORMAT);
 282
 283         hinic_set_sge(&cqe_sect->sge, cqe_dma, rq_ceq_len);
 284
 285         buf_desc->addr_high = upper_32_bits(buf_addr);
 286         buf_desc->addr_low = lower_32_bits(buf_addr);
 287 }
 288
 289 void hinic_rxq_get_stats(struct hinic_rxq *rxq, struct hinic_rxq_stats *stats)
 290 {
 291         if (!rxq || !stats)
 292                 return;
 293
 294         memcpy(stats, &rxq->rxq_stats, sizeof(rxq->rxq_stats));
 295 }
 296
 297 void hinic_rxq_stats_reset(struct hinic_rxq *rxq)
 298 {
 299         struct hinic_rxq_stats *rxq_stats;
 300
 301         if (rxq == NULL)
 302                 return;
 303
 304         rxq_stats = &rxq->rxq_stats;
 305         memset(rxq_stats, 0, sizeof(*rxq_stats));
 306 }
 307
 308 static int hinic_rx_alloc_cqe(struct hinic_rxq *rxq)
 309 {
 310         size_t cqe_mem_size;
 311
 312         cqe_mem_size = sizeof(struct hinic_rq_cqe) * rxq->q_depth;
 313         rxq->cqe_start_vaddr =
 314                 dma_zalloc_coherent(rxq->nic_dev->hwdev,
 315                                     cqe_mem_size, &rxq->cqe_start_paddr,
 316                                     GFP_KERNEL);
 317         if (!rxq->cqe_start_vaddr) {
 318                 PMD_DRV_LOG(ERR, "Allocate cqe dma memory failed");
 319                 return -ENOMEM;
 320         }
 321
 322         rxq->rx_cqe = (struct hinic_rq_cqe *)rxq->cqe_start_vaddr;
 323
 324         return HINIC_OK;
 325 }
 326
 327 static void hinic_rx_free_cqe(struct hinic_rxq *rxq)
 328 {
 329         size_t cqe_mem_size;
 330
 331         cqe_mem_size = sizeof(struct hinic_rq_cqe) * rxq->q_depth;
 332         dma_free_coherent(rxq->nic_dev->hwdev, cqe_mem_size,
 333                           rxq->cqe_start_vaddr, rxq->cqe_start_paddr);
 334         rxq->cqe_start_vaddr = NULL;
 335 }
 336
 337 static int hinic_rx_fill_wqe(struct hinic_rxq *rxq)
 338 {
 339         struct hinic_nic_dev *nic_dev = rxq->nic_dev;
 340         struct hinic_rq_wqe *rq_wqe;
 341         dma_addr_t buf_dma_addr, cqe_dma_addr;
 342         u16 pi = 0;
 343         int i;
 344
 345         buf_dma_addr = 0;
 346         cqe_dma_addr = rxq->cqe_start_paddr;
 347         for (i = 0; i < rxq->q_depth; i++) {
 348                 rq_wqe = hinic_get_rq_wqe(nic_dev->hwdev, rxq->q_id, &pi);
 349                 if (!rq_wqe) {
 350                         PMD_DRV_LOG(ERR, "Get rq wqe failed");
 351                         break;
 352                 }
 353
 354                 hinic_prepare_rq_wqe(rq_wqe, pi, buf_dma_addr, cqe_dma_addr);
 355                 cqe_dma_addr +=  sizeof(struct hinic_rq_cqe);
 356
 357                 hinic_cpu_to_be32(rq_wqe, sizeof(struct hinic_rq_wqe));
 358         }
 359
 360         hinic_return_rq_wqe(nic_dev->hwdev, rxq->q_id, i);
 361
 362         return i;
 363 }
 364
 365 /* alloc cqe and prepare rqe */
 366 int hinic_setup_rx_resources(struct hinic_rxq *rxq)
 367 {
 368         u64 rx_info_sz;
 369         int err, pkts;
 370
 371         rx_info_sz = rxq->q_depth * sizeof(*rxq->rx_info);
 372         rxq->rx_info = kzalloc_aligned(rx_info_sz, GFP_KERNEL);
 373         if (!rxq->rx_info)
 374                 return -ENOMEM;
 375
 376         err = hinic_rx_alloc_cqe(rxq);
 377         if (err) {
 378                 PMD_DRV_LOG(ERR, "Allocate rx cqe failed");
 379                 goto rx_cqe_err;
 380         }
 381
 382         pkts = hinic_rx_fill_wqe(rxq);
 383         if (pkts != rxq->q_depth) {
 384                 PMD_DRV_LOG(ERR, "Fill rx wqe failed");
 385                 err = -ENOMEM;
 386                 goto rx_fill_err;
 387         }
 388
 389         return 0;
 390
 391 rx_fill_err:
 392         hinic_rx_free_cqe(rxq);
 393
 394 rx_cqe_err:
 395         kfree(rxq->rx_info);
 396         rxq->rx_info = NULL;
 397
 398         return err;
 399 }
 400
 401 void hinic_free_rx_resources(struct hinic_rxq *rxq)
 402 {
 403         if (rxq->rx_info == NULL)
 404                 return;
 405
 406         hinic_rx_free_cqe(rxq);
 407         kfree(rxq->rx_info);
 408         rxq->rx_info = NULL;
 409 }
 410
 411 void hinic_free_all_rx_resources(struct rte_eth_dev *eth_dev)
 412 {
 413         u16 q_id;
 414         struct hinic_nic_dev *nic_dev =
 415                                 HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
 416
 417         for (q_id = 0; q_id < nic_dev->num_rq; q_id++) {
 418                 eth_dev->data->rx_queues[q_id] = NULL;
 419
 420                 if (nic_dev->rxqs[q_id] == NULL)
 421                         continue;
 422
 423                 hinic_free_all_rx_mbufs(nic_dev->rxqs[q_id]);
 424                 hinic_free_rx_resources(nic_dev->rxqs[q_id]);
 425                 kfree(nic_dev->rxqs[q_id]);
 426                 nic_dev->rxqs[q_id] = NULL;
 427         }
 428 }
 429
 430 void hinic_free_all_rx_mbuf(struct rte_eth_dev *eth_dev)
 431 {
 432         struct hinic_nic_dev *nic_dev =
 433                                 HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
 434         u16 q_id;
 435
 436         for (q_id = 0; q_id < nic_dev->num_rq; q_id++)
 437                 hinic_free_all_rx_mbufs(nic_dev->rxqs[q_id]);
 438 }
 439
 440 static void hinic_recv_jumbo_pkt(struct hinic_rxq *rxq,
 441                                  struct rte_mbuf *head_mbuf,
 442                                  u32 remain_pkt_len)
 443 {
 444         struct hinic_nic_dev *nic_dev = rxq->nic_dev;
 445         struct rte_mbuf *cur_mbuf, *rxm = NULL;
 446         struct hinic_rx_info *rx_info;
 447         u16 sw_ci, rx_buf_len = rxq->buf_len;
 448         u32 pkt_len;
 449
 450         while (remain_pkt_len > 0) {
 451                 sw_ci = hinic_get_rq_local_ci(nic_dev->hwdev, rxq->q_id);
 452                 rx_info = &rxq->rx_info[sw_ci];
 453
 454                 hinic_update_rq_local_ci(nic_dev->hwdev, rxq->q_id, 1);
 455
 456                 pkt_len = remain_pkt_len > rx_buf_len ?
 457                         rx_buf_len : remain_pkt_len;
 458                 remain_pkt_len -= pkt_len;
 459
 460                 cur_mbuf = rx_info->mbuf;
 461                 cur_mbuf->data_len = (u16)pkt_len;
 462                 cur_mbuf->next = NULL;
 463
 464                 head_mbuf->pkt_len += cur_mbuf->data_len;
 465                 head_mbuf->nb_segs++;
 466
 467                 if (!rxm)
 468                         head_mbuf->next = cur_mbuf;
 469                 else
 470                         rxm->next = cur_mbuf;
 471
 472                 rxm = cur_mbuf;
 473         }
 474 }
 475
 476 static void hinic_rss_deinit(struct hinic_nic_dev *nic_dev)
 477 {
 478         u8 prio_tc[HINIC_DCB_UP_MAX] = {0};
 479         (void)hinic_rss_cfg(nic_dev->hwdev, 0,
 480                             nic_dev->rss_tmpl_idx, 0, prio_tc);
 481 }
 482
 483 static int hinic_rss_key_init(struct hinic_nic_dev *nic_dev,
 484                               struct rte_eth_rss_conf *rss_conf)
 485 {
 486         u8 default_rss_key[HINIC_RSS_KEY_SIZE] = {
 487                          0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2,
 488                          0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0,
 489                          0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4,
 490                          0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c,
 491                          0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa};
 492         u8 hashkey[HINIC_RSS_KEY_SIZE] = {0};
 493         u8 tmpl_idx = nic_dev->rss_tmpl_idx;
 494
 495         if (rss_conf->rss_key == NULL)
 496                 memcpy(hashkey, default_rss_key, HINIC_RSS_KEY_SIZE);
 497         else
 498                 memcpy(hashkey, rss_conf->rss_key, rss_conf->rss_key_len);
 499
 500         return hinic_rss_set_template_tbl(nic_dev->hwdev, tmpl_idx, hashkey);
 501 }
 502
 503 static void hinic_fill_rss_type(struct nic_rss_type *rss_type,
 504                                 struct rte_eth_rss_conf *rss_conf)
 505 {
 506         u64 rss_hf = rss_conf->rss_hf;
 507
 508         rss_type->ipv4 = (rss_hf & (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4)) ? 1 : 0;
 509         rss_type->tcp_ipv4 = (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) ? 1 : 0;
 510         rss_type->ipv6 = (rss_hf & (ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6)) ? 1 : 0;
 511         rss_type->ipv6_ext = (rss_hf & ETH_RSS_IPV6_EX) ? 1 : 0;
 512         rss_type->tcp_ipv6 = (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP) ? 1 : 0;
 513         rss_type->tcp_ipv6_ext = (rss_hf & ETH_RSS_IPV6_TCP_EX) ? 1 : 0;
 514         rss_type->udp_ipv4 = (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) ? 1 : 0;
 515         rss_type->udp_ipv6 = (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP) ? 1 : 0;
 516 }
 517
 518 static void hinic_fillout_indir_tbl(struct hinic_nic_dev *nic_dev, u32 *indir)
 519 {
 520         u8 rss_queue_count = nic_dev->num_rss;
 521         int i = 0, j;
 522
 523         if (rss_queue_count == 0) {
 524                 /* delete q_id from indir tbl */
 525                 for (i = 0; i < HINIC_RSS_INDIR_SIZE; i++)
 526                         indir[i] = 0xFF;        /* Invalid value in indir tbl */
 527         } else {
 528                 while (i < HINIC_RSS_INDIR_SIZE)
 529                         for (j = 0; (j < rss_queue_count) &&
 530                              (i < HINIC_RSS_INDIR_SIZE); j++)
 531                                 indir[i++] = nic_dev->rx_queue_list[j];
 532         }
 533 }
 534
 535 static int hinic_rss_init(struct hinic_nic_dev *nic_dev,
 536                           __attribute__((unused)) u8 *rq2iq_map,
 537                           struct rte_eth_rss_conf *rss_conf)
 538 {
 539         u32 indir_tbl[HINIC_RSS_INDIR_SIZE] = {0};
 540         struct nic_rss_type rss_type = {0};
 541         u8 prio_tc[HINIC_DCB_UP_MAX] = {0};
 542         u8 tmpl_idx = 0xFF, num_tc = 0;
 543         int err;
 544
 545         tmpl_idx = nic_dev->rss_tmpl_idx;
 546
 547         err = hinic_rss_key_init(nic_dev, rss_conf);
 548         if (err)
 549                 return err;
 550
 551         if (!nic_dev->rss_indir_flag) {
 552                 hinic_fillout_indir_tbl(nic_dev, indir_tbl);
 553                 err = hinic_rss_set_indir_tbl(nic_dev->hwdev, tmpl_idx,
 554                                               indir_tbl);
 555                 if (err)
 556                         return err;
 557         }
 558
 559         hinic_fill_rss_type(&rss_type, rss_conf);
 560         err = hinic_set_rss_type(nic_dev->hwdev, tmpl_idx, rss_type);
 561         if (err)
 562                 return err;
 563
 564         err = hinic_rss_set_hash_engine(nic_dev->hwdev, tmpl_idx,
 565                                         HINIC_RSS_HASH_ENGINE_TYPE_TOEP);
 566         if (err)
 567                 return err;
 568
 569         return hinic_rss_cfg(nic_dev->hwdev, 1, tmpl_idx, num_tc, prio_tc);
 570 }
 571
 572 static void
 573 hinic_add_rq_to_rx_queue_list(struct hinic_nic_dev *nic_dev, u16 queue_id)
 574 {
 575         u8 rss_queue_count = nic_dev->num_rss;
 576
 577         RTE_ASSERT(rss_queue_count <= (RTE_DIM(nic_dev->rx_queue_list) - 1));
 578
 579         nic_dev->rx_queue_list[rss_queue_count] = queue_id;
 580         nic_dev->num_rss++;
 581 }
 582
 583 /**
 584  * hinic_setup_num_qps - determine num_qps from rss_tmpl_id
 585  * @nic_dev: pointer to the private ethernet device
 586  * Return: 0 on Success, error code otherwise.
 587  **/
 588 static int hinic_setup_num_qps(struct hinic_nic_dev *nic_dev)
 589 {
 590         int err, i;
 591
 592         if (!(nic_dev->flags & ETH_MQ_RX_RSS_FLAG)) {
 593                 nic_dev->flags &= ~ETH_MQ_RX_RSS_FLAG;
 594                 nic_dev->num_rss = 0;
 595                 if (nic_dev->num_rq > 1) {
 596                         /* get rss template id */
 597                         err = hinic_rss_template_alloc(nic_dev->hwdev,
 598                                                        &nic_dev->rss_tmpl_idx);
 599                         if (err) {
 600                                 PMD_DRV_LOG(WARNING, "Alloc rss template failed");
 601                                 return err;
 602                         }
 603                         nic_dev->flags |= ETH_MQ_RX_RSS_FLAG;
 604                         for (i = 0; i < nic_dev->num_rq; i++)
 605                                 hinic_add_rq_to_rx_queue_list(nic_dev, i);
 606                 }
 607         }
 608
 609         return 0;
 610 }
 611
 612 static void hinic_destroy_num_qps(struct hinic_nic_dev *nic_dev)
 613 {
 614         if (nic_dev->flags & ETH_MQ_RX_RSS_FLAG) {
 615                 if (hinic_rss_template_free(nic_dev->hwdev,
 616                                             nic_dev->rss_tmpl_idx))
 617                         PMD_DRV_LOG(WARNING, "Free rss template failed");
 618
 619                 nic_dev->flags &= ~ETH_MQ_RX_RSS_FLAG;
 620         }
 621 }
 622
 623 static int hinic_config_mq_rx_rss(struct hinic_nic_dev *nic_dev, bool on)
 624 {
 625         int ret = 0;
 626
 627         if (on) {
 628                 ret = hinic_setup_num_qps(nic_dev);
 629                 if (ret)
 630                         PMD_DRV_LOG(ERR, "Setup num_qps failed");
 631         } else {
 632                 hinic_destroy_num_qps(nic_dev);
 633         }
 634
 635         return ret;
 636 }
 637
 638 int hinic_config_mq_mode(struct rte_eth_dev *dev, bool on)
 639 {
 640         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 641         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 642         int ret = 0;
 643
 644         switch (dev_conf->rxmode.mq_mode) {
 645         case ETH_MQ_RX_RSS:
 646                 ret = hinic_config_mq_rx_rss(nic_dev, on);
 647                 break;
 648         default:
 649                 break;
 650         }
 651
 652         return ret;
 653 }
 654
 655 int hinic_rx_configure(struct rte_eth_dev *dev)
 656 {
 657         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 658         struct rte_eth_rss_conf rss_conf =
 659                 dev->data->dev_conf.rx_adv_conf.rss_conf;
 660         int err;
 661
 662         if (nic_dev->flags & ETH_MQ_RX_RSS_FLAG) {
 663                 if (rss_conf.rss_hf == 0) {
 664                         rss_conf.rss_hf = HINIC_RSS_OFFLOAD_ALL;
 665                 } else if ((rss_conf.rss_hf & HINIC_RSS_OFFLOAD_ALL) == 0) {
 666                         PMD_DRV_LOG(ERR, "Do not support rss offload all");
 667                         goto rss_config_err;
 668                 }
 669
 670                 err = hinic_rss_init(nic_dev, NULL, &rss_conf);
 671                 if (err) {
 672                         PMD_DRV_LOG(ERR, "Init rss failed");
 673                         goto rss_config_err;
 674                 }
 675         }
 676
 677         /* Enable both L3/L4 rx checksum offload */
 678         if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_CHECKSUM)
 679                 nic_dev->rx_csum_en = HINIC_RX_CSUM_OFFLOAD_EN;
 680
 681         err = hinic_set_rx_csum_offload(nic_dev->hwdev,
 682                                         HINIC_RX_CSUM_OFFLOAD_EN);
 683         if (err)
 684                 goto rx_csum_ofl_err;
 685
 686         return 0;
 687
 688 rx_csum_ofl_err:
 689 rss_config_err:
 690         hinic_destroy_num_qps(nic_dev);
 691
 692         return HINIC_ERROR;
 693 }
 694
 695 void hinic_rx_remove_configure(struct rte_eth_dev *dev)
 696 {
 697         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 698
 699         if (nic_dev->flags & ETH_MQ_RX_RSS_FLAG) {
 700                 hinic_rss_deinit(nic_dev);
 701                 hinic_destroy_num_qps(nic_dev);
 702         }
 703 }
 704
 705 void hinic_free_all_rx_mbufs(struct hinic_rxq *rxq)
 706 {
 707         struct hinic_nic_dev *nic_dev = rxq->nic_dev;
 708         struct hinic_rx_info *rx_info;
 709         int free_wqebbs =
 710                 hinic_get_rq_free_wqebbs(nic_dev->hwdev, rxq->q_id) + 1;
 711         volatile struct hinic_rq_cqe *rx_cqe;
 712         u16 ci;
 713
 714         while (free_wqebbs++ < rxq->q_depth) {
 715                 ci = hinic_get_rq_local_ci(nic_dev->hwdev, rxq->q_id);
 716
 717                 rx_cqe = &rxq->rx_cqe[ci];
 718
 719                 /* clear done bit */
 720                 rx_cqe->status = 0;
 721
 722                 rx_info = &rxq->rx_info[ci];
 723                 rte_pktmbuf_free(rx_info->mbuf);
 724                 rx_info->mbuf = NULL;
 725
 726                 hinic_update_rq_local_ci(nic_dev->hwdev, rxq->q_id, 1);
 727         }
 728 }
 729
 730 static inline void hinic_rq_cqe_be_to_cpu32(void *dst_le32,
 731                                             volatile void *src_be32)
 732 {
 733 #if defined(__X86_64_SSE__)
 734         volatile __m128i *wqe_be = (volatile __m128i *)src_be32;
 735         __m128i *wqe_le = (__m128i *)dst_le32;
 736         __m128i shuf_mask =  _mm_set_epi8(12, 13, 14, 15, 8, 9, 10,
 737                                         11, 4, 5, 6, 7, 0, 1, 2, 3);
 738
 739         /* l2nic just use first 128 bits */
 740         wqe_le[0] = _mm_shuffle_epi8(wqe_be[0], shuf_mask);
 741 #elif defined(__ARM64_NEON__)
 742         volatile uint8x16_t *wqe_be = (volatile uint8x16_t *)src_be32;
 743         uint8x16_t *wqe_le = (uint8x16_t *)dst_le32;
 744         const uint8x16_t shuf_mask = {3, 2, 1, 0, 7, 6, 5, 4, 11, 10,
 745                                         9, 8, 15, 14, 13, 12};
 746
 747         /* l2nic just use first 128 bits */
 748         wqe_le[0] = vqtbl1q_u8(wqe_be[0], shuf_mask);
 749 #else
 750         u32 i;
 751         volatile u32 *wqe_be = (volatile u32 *)src_be32;
 752         u32 *wqe_le = (u32 *)dst_le32;
 753
 754 #define HINIC_L2NIC_RQ_CQE_USED         4 /* 4Bytes unit */
 755
 756         for (i = 0; i < HINIC_L2NIC_RQ_CQE_USED; i++) {
 757                 *wqe_le = rte_be_to_cpu_32(*wqe_be);
 758                 wqe_be++;
 759                 wqe_le++;
 760         }
 761 #endif
 762 }
 763
 764 static inline uint64_t hinic_rx_rss_hash(uint32_t offload_type,
 765                                          uint32_t cqe_hass_val,
 766                                          uint32_t *rss_hash)
 767 {
 768         uint32_t rss_type;
 769
 770         rss_type = HINIC_GET_RSS_TYPES(offload_type);
 771         if (likely(rss_type != 0)) {
 772                 *rss_hash = cqe_hass_val;
 773                 return PKT_RX_RSS_HASH;
 774         }
 775
 776         return 0;
 777 }
 778
 779 static inline uint64_t hinic_rx_csum(uint32_t status, struct hinic_rxq *rxq)
 780 {
 781         uint32_t checksum_err;
 782         uint64_t flags;
 783         struct hinic_nic_dev *nic_dev = rxq->nic_dev;
 784
 785         if (unlikely(!(nic_dev->rx_csum_en & HINIC_RX_CSUM_OFFLOAD_EN)))
 786                 return PKT_RX_IP_CKSUM_UNKNOWN;
 787
 788         /* most case checksum is ok */
 789         checksum_err = HINIC_GET_RX_CSUM_ERR(status);
 790         if (likely(checksum_err == 0))
 791                 return (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD);
 792
 793         /* If BYPASS bit set, all other status indications should be ignored */
 794         if (unlikely(HINIC_CSUM_ERR_BYPASSED(checksum_err)))
 795                 return PKT_RX_IP_CKSUM_UNKNOWN;
 796
 797         flags = 0;
 798
 799         /* IP checksum error */
 800         if (HINIC_CSUM_ERR_IP(checksum_err))
 801                 flags |= PKT_RX_IP_CKSUM_BAD;
 802         else
 803                 flags |= PKT_RX_IP_CKSUM_GOOD;
 804
 805         /* L4 checksum error */
 806         if (HINIC_CSUM_ERR_L4(checksum_err))
 807                 flags |= PKT_RX_L4_CKSUM_BAD;
 808         else
 809                 flags |= PKT_RX_L4_CKSUM_GOOD;
 810
 811         if (unlikely(HINIC_CSUM_ERR_OTHER(checksum_err)))
 812                 flags = PKT_RX_L4_CKSUM_NONE;
 813
 814         rxq->rxq_stats.errors++;
 815
 816         return flags;
 817 }
 818
 819 static inline uint64_t hinic_rx_vlan(uint32_t offload_type, uint32_t vlan_len,
 820                                      uint16_t *vlan_tci)
 821 {
 822         uint16_t vlan_tag;
 823
 824         vlan_tag = HINIC_GET_RX_VLAN_TAG(vlan_len);
 825         if (!HINIC_GET_RX_VLAN_OFFLOAD_EN(offload_type) || 0 == vlan_tag) {
 826                 *vlan_tci = 0;
 827                 return 0;
 828         }
 829
 830         *vlan_tci = vlan_tag;
 831
 832         return PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED;
 833 }
 834
 835 static inline u32 hinic_rx_alloc_mbuf_bulk(struct hinic_rxq *rxq,
 836                                            struct rte_mbuf **mbufs,
 837                                            u32 exp_mbuf_cnt)
 838 {
 839         int rc;
 840         u32 avail_cnt;
 841
 842         rc = rte_pktmbuf_alloc_bulk(rxq->mb_pool, mbufs, exp_mbuf_cnt);
 843         if (likely(rc == HINIC_OK)) {
 844                 avail_cnt = exp_mbuf_cnt;
 845         } else {
 846                 avail_cnt = 0;
 847                 rxq->rxq_stats.rx_nombuf += exp_mbuf_cnt;
 848         }
 849
 850         return avail_cnt;
 851 }
 852
 853 static struct rte_mbuf *hinic_rx_alloc_mbuf(struct hinic_rxq *rxq,
 854                                         dma_addr_t *dma_addr)
 855 {
 856         struct rte_mbuf *mbuf = NULL;
 857         int rc;
 858
 859         rc = rte_pktmbuf_alloc_bulk(rxq->mb_pool, &mbuf, 1);
 860         if (unlikely(rc != HINIC_OK))
 861                 return NULL;
 862
 863         *dma_addr = rte_mbuf_data_iova_default(mbuf);
 864
 865         return mbuf;
 866 }
 867
 868 static inline void hinic_rearm_rxq_mbuf(struct hinic_rxq *rxq)
 869 {
 870         u16 pi;
 871         u32 i, free_wqebbs, rearm_wqebbs, exp_wqebbs;
 872         dma_addr_t dma_addr;
 873         struct hinic_rq_wqe *rq_wqe;
 874         struct rte_mbuf **rearm_mbufs;
 875
 876         /* check free wqebb fo rearm */
 877         free_wqebbs = HINIC_GET_RQ_FREE_WQEBBS(rxq);
 878         if (unlikely(free_wqebbs < rxq->rx_free_thresh))
 879                 return;
 880
 881         /* get rearm mbuf array */
 882         pi = HINIC_GET_RQ_LOCAL_PI(rxq);
 883         rearm_mbufs = (struct rte_mbuf **)(&rxq->rx_info[pi]);
 884
 885         /* check rxq free wqebbs turn around */
 886         exp_wqebbs = rxq->q_depth - pi;
 887         if (free_wqebbs < exp_wqebbs)
 888                 exp_wqebbs = free_wqebbs;
 889
 890         /* alloc mbuf in bulk */
 891         rearm_wqebbs = hinic_rx_alloc_mbuf_bulk(rxq, rearm_mbufs, exp_wqebbs);
 892         if (unlikely(rearm_wqebbs == 0))
 893                 return;
 894
 895         /* rearm rx mbuf */
 896         rq_wqe = WQ_WQE_ADDR(rxq->wq, (u32)pi);
 897         for (i = 0; i < rearm_wqebbs; i++) {
 898                 dma_addr = rte_mbuf_data_iova_default(rearm_mbufs[i]);
 899                 rq_wqe->buf_desc.addr_high =
 900                                         cpu_to_be32(upper_32_bits(dma_addr));
 901                 rq_wqe->buf_desc.addr_low =
 902                                         cpu_to_be32(lower_32_bits(dma_addr));
 903                 rq_wqe++;
 904         }
 905         rxq->wq->prod_idx += rearm_wqebbs;
 906         rxq->wq->delta -= rearm_wqebbs;
 907
 908         /* update rq hw_pi */
 909         rte_wmb();
 910         HINIC_UPDATE_RQ_HW_PI(rxq, pi + rearm_wqebbs);
 911 }
 912
 913 void hinic_rx_alloc_pkts(struct hinic_rxq *rxq)
 914 {
 915         struct hinic_nic_dev *nic_dev = rxq->nic_dev;
 916         struct hinic_rq_wqe *rq_wqe;
 917         struct hinic_rx_info *rx_info;
 918         struct rte_mbuf *mb;
 919         dma_addr_t dma_addr;
 920         u16 pi = 0;
 921         int i, free_wqebbs;
 922
 923         free_wqebbs = HINIC_GET_RQ_FREE_WQEBBS(rxq);
 924         for (i = 0; i < free_wqebbs; i++) {
 925                 mb = hinic_rx_alloc_mbuf(rxq, &dma_addr);
 926                 if (unlikely(!mb)) {
 927                         rxq->rxq_stats.rx_nombuf++;
 928                         break;
 929                 }
 930
 931                 rq_wqe = hinic_get_rq_wqe(nic_dev->hwdev, rxq->q_id, &pi);
 932                 if (unlikely(!rq_wqe)) {
 933                         rte_pktmbuf_free(mb);
 934                         break;
 935                 }
 936
 937                 /* fill buffer address only */
 938                 rq_wqe->buf_desc.addr_high =
 939                                 cpu_to_be32(upper_32_bits(dma_addr));
 940                 rq_wqe->buf_desc.addr_low =
 941                                 cpu_to_be32(lower_32_bits(dma_addr));
 942
 943                 rx_info = &rxq->rx_info[pi];
 944                 rx_info->mbuf = mb;
 945         }
 946
 947         if (likely(i > 0)) {
 948                 rte_wmb();
 949                 HINIC_UPDATE_RQ_HW_PI(rxq, pi + 1);
 950         }
 951 }
 952
 953 u16 hinic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, u16 nb_pkts)
 954 {
 955         struct rte_mbuf *rxm;
 956         struct hinic_rxq *rxq = rx_queue;
 957         struct hinic_rx_info *rx_info;
 958         volatile struct hinic_rq_cqe *rx_cqe;
 959         u16 rx_buf_len, pkts = 0;
 960         u16 sw_ci, ci_mask, wqebb_cnt = 0;
 961         u32 pkt_len, status, vlan_len;
 962         u64 rx_bytes = 0;
 963         struct hinic_rq_cqe cqe;
 964         u32 offload_type, rss_hash;
 965
 966         rx_buf_len = rxq->buf_len;
 967
 968         /* 1. get polling start ci */
 969         ci_mask = HINIC_GET_RQ_WQE_MASK(rxq);
 970         sw_ci = HINIC_GET_RQ_LOCAL_CI(rxq);
 971
 972         while (pkts < nb_pkts) {
 973                  /* 2. current ci is done */
 974                 rx_cqe = &rxq->rx_cqe[sw_ci];
 975                 status = __atomic_load_n(&rx_cqe->status, __ATOMIC_ACQUIRE);
 976                 if (!HINIC_GET_RX_DONE_BE(status))
 977                         break;
 978
 979                 /* convert cqe and get packet length */
 980                 hinic_rq_cqe_be_to_cpu32(&cqe, (volatile void *)rx_cqe);
 981                 vlan_len = cqe.vlan_len;
 982
 983                 rx_info = &rxq->rx_info[sw_ci];
 984                 rxm = rx_info->mbuf;
 985
 986                 /* 3. next ci point and prefetch */
 987                 sw_ci++;
 988                 sw_ci &= ci_mask;
 989
 990                 /* prefetch next mbuf first 64B */
 991                 rte_prefetch0(rxq->rx_info[sw_ci].mbuf);
 992
 993                 /* 4. jumbo frame process */
 994                 pkt_len = HINIC_GET_RX_PKT_LEN(vlan_len);
 995                 if (likely(pkt_len <= rx_buf_len)) {
 996                         rxm->data_len = pkt_len;
 997                         rxm->pkt_len = pkt_len;
 998                         wqebb_cnt++;
 999                 } else {
1000                         rxm->data_len = rx_buf_len;
1001                         rxm->pkt_len = rx_buf_len;
1002
1003                         /* if receive jumbo, updating ci will be done by
1004                          * hinic_recv_jumbo_pkt function.
1005                          */
1006                         HINIC_UPDATE_RQ_LOCAL_CI(rxq, wqebb_cnt + 1);
1007                         wqebb_cnt = 0;
1008                         hinic_recv_jumbo_pkt(rxq, rxm, pkt_len - rx_buf_len);
1009                         sw_ci = HINIC_GET_RQ_LOCAL_CI(rxq);
1010                 }
1011
1012                 /* 5. vlan/checksum/rss/pkt_type/gro offload */
1013                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1014                 rxm->port = rxq->port_id;
1015                 offload_type = cqe.offload_type;
1016
1017                 /* vlan offload */
1018                 rxm->ol_flags |= hinic_rx_vlan(offload_type, vlan_len,
1019                                                &rxm->vlan_tci);
1020
1021                 /* checksum offload */
1022                 rxm->ol_flags |= hinic_rx_csum(cqe.status, rxq);
1023
1024                 /* rss hash offload */
1025                 rss_hash = cqe.rss_hash;
1026                 rxm->ol_flags |= hinic_rx_rss_hash(offload_type, rss_hash,
1027                                                    &rxm->hash.rss);
1028
1029                 /* 6. clear done bit */
1030                 rx_cqe->status = 0;
1031
1032                 rx_bytes += pkt_len;
1033                 rx_pkts[pkts++] = rxm;
1034         }
1035
1036         if (pkts) {
1037                 /* 7. update ci */
1038                 HINIC_UPDATE_RQ_LOCAL_CI(rxq, wqebb_cnt);
1039
1040                 /* do packet stats */
1041                 rxq->rxq_stats.packets += pkts;
1042                 rxq->rxq_stats.bytes += rx_bytes;
1043         }
1044         rxq->rxq_stats.burst_pkts = pkts;
1045
1046         /* 8. rearm mbuf to rxq */
1047         hinic_rearm_rxq_mbuf(rxq);
1048
1049         return pkts;
1050 }