drivers/net/hinic/hinic_pmd_rx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Huawei Technologies Co., Ltd
   3  */
   4
   5 #include <rte_ether.h>
   6 #include <rte_mbuf.h>
   7 #ifdef __ARM64_NEON__
   8 #include <arm_neon.h>
   9 #endif
  10
  11 #include "base/hinic_compat.h"
  12 #include "base/hinic_pmd_hwdev.h"
  13 #include "base/hinic_pmd_wq.h"
  14 #include "base/hinic_pmd_niccfg.h"
  15 #include "base/hinic_pmd_nicio.h"
  16 #include "hinic_pmd_ethdev.h"
  17 #include "hinic_pmd_rx.h"
  18
  19 /* rxq wq operations */
  20 #define HINIC_GET_RQ_WQE_MASK(rxq)      \
  21         ((rxq)->wq->mask)
  22
  23 #define HINIC_GET_RQ_LOCAL_CI(rxq)      \
  24         (((rxq)->wq->cons_idx) & HINIC_GET_RQ_WQE_MASK(rxq))
  25
  26 #define HINIC_GET_RQ_LOCAL_PI(rxq)      \
  27         (((rxq)->wq->prod_idx) & HINIC_GET_RQ_WQE_MASK(rxq))
  28
  29 #define HINIC_UPDATE_RQ_LOCAL_CI(rxq, wqebb_cnt)        \
  30         do {                                            \
  31                 (rxq)->wq->cons_idx += (wqebb_cnt);     \
  32                 (rxq)->wq->delta += (wqebb_cnt);        \
  33         } while (0)
  34
  35 #define HINIC_UPDATE_RQ_HW_PI(rxq, pi)  \
  36         (*((rxq)->pi_virt_addr) =       \
  37                 cpu_to_be16((pi) & HINIC_GET_RQ_WQE_MASK(rxq)))
  38
  39 #define HINIC_GET_RQ_FREE_WQEBBS(rxq)   ((rxq)->wq->delta - 1)
  40
  41 /* rxq cqe done and status bit */
  42 #define HINIC_GET_RX_DONE_BE(status)    \
  43         ((status) & 0x80U)
  44
  45 #define HINIC_RX_CSUM_OFFLOAD_EN        0xFFF
  46
  47 #define RQ_CQE_SGE_VLAN_SHIFT                   0
  48 #define RQ_CQE_SGE_LEN_SHIFT                    16
  49
  50 #define RQ_CQE_SGE_VLAN_MASK                    0xFFFFU
  51 #define RQ_CQE_SGE_LEN_MASK                     0xFFFFU
  52
  53 #define RQ_CQE_SGE_GET(val, member)             \
  54         (((val) >> RQ_CQE_SGE_##member##_SHIFT) & RQ_CQE_SGE_##member##_MASK)
  55
  56 #define HINIC_GET_RX_VLAN_TAG(vlan_len) \
  57                 RQ_CQE_SGE_GET(vlan_len, VLAN)
  58
  59 #define HINIC_GET_RX_PKT_LEN(vlan_len)  \
  60                 RQ_CQE_SGE_GET(vlan_len, LEN)
  61
  62 #define RQ_CQE_STATUS_CSUM_ERR_SHIFT            0
  63 #define RQ_CQE_STATUS_NUM_LRO_SHIFT             16
  64 #define RQ_CQE_STATUS_LRO_PUSH_SHIFT            25
  65 #define RQ_CQE_STATUS_LRO_ENTER_SHIFT           26
  66 #define RQ_CQE_STATUS_LRO_INTR_SHIFT            27
  67
  68 #define RQ_CQE_STATUS_BP_EN_SHIFT               30
  69 #define RQ_CQE_STATUS_RXDONE_SHIFT              31
  70 #define RQ_CQE_STATUS_FLUSH_SHIFT               28
  71
  72 #define RQ_CQE_STATUS_CSUM_ERR_MASK             0xFFFFU
  73 #define RQ_CQE_STATUS_NUM_LRO_MASK              0xFFU
  74 #define RQ_CQE_STATUS_LRO_PUSH_MASK             0X1U
  75 #define RQ_CQE_STATUS_LRO_ENTER_MASK            0X1U
  76 #define RQ_CQE_STATUS_LRO_INTR_MASK             0X1U
  77 #define RQ_CQE_STATUS_BP_EN_MASK                0X1U
  78 #define RQ_CQE_STATUS_RXDONE_MASK               0x1U
  79 #define RQ_CQE_STATUS_FLUSH_MASK                0x1U
  80
  81 #define RQ_CQE_STATUS_GET(val, member)          \
  82                 (((val) >> RQ_CQE_STATUS_##member##_SHIFT) & \
  83                                 RQ_CQE_STATUS_##member##_MASK)
  84
  85 #define RQ_CQE_STATUS_CLEAR(val, member)        \
  86                 ((val) & (~(RQ_CQE_STATUS_##member##_MASK << \
  87                                 RQ_CQE_STATUS_##member##_SHIFT)))
  88
  89 #define HINIC_GET_RX_CSUM_ERR(status)   \
  90                 RQ_CQE_STATUS_GET(status, CSUM_ERR)
  91
  92 #define HINIC_GET_RX_DONE(status)       \
  93                 RQ_CQE_STATUS_GET(status, RXDONE)
  94
  95 #define HINIC_GET_RX_FLUSH(status)      \
  96                 RQ_CQE_STATUS_GET(status, FLUSH)
  97
  98 #define HINIC_GET_RX_BP_EN(status)      \
  99                 RQ_CQE_STATUS_GET(status, BP_EN)
 100
 101 #define HINIC_GET_RX_NUM_LRO(status)    \
 102                 RQ_CQE_STATUS_GET(status, NUM_LRO)
 103
 104 /* RQ_CTRL */
 105 #define RQ_CTRL_BUFDESC_SECT_LEN_SHIFT          0
 106 #define RQ_CTRL_COMPLETE_FORMAT_SHIFT           15
 107 #define RQ_CTRL_COMPLETE_LEN_SHIFT              27
 108 #define RQ_CTRL_LEN_SHIFT                       29
 109
 110 #define RQ_CTRL_BUFDESC_SECT_LEN_MASK           0xFFU
 111 #define RQ_CTRL_COMPLETE_FORMAT_MASK            0x1U
 112 #define RQ_CTRL_COMPLETE_LEN_MASK               0x3U
 113 #define RQ_CTRL_LEN_MASK                        0x3U
 114
 115 #define RQ_CTRL_SET(val, member)                \
 116         (((val) & RQ_CTRL_##member##_MASK) << RQ_CTRL_##member##_SHIFT)
 117
 118 #define RQ_CTRL_GET(val, member)                \
 119         (((val) >> RQ_CTRL_##member##_SHIFT) & RQ_CTRL_##member##_MASK)
 120
 121 #define RQ_CTRL_CLEAR(val, member)              \
 122         ((val) & (~(RQ_CTRL_##member##_MASK << RQ_CTRL_##member##_SHIFT)))
 123
 124 #define RQ_CQE_PKT_NUM_SHIFT                    1
 125 #define RQ_CQE_PKT_FIRST_LEN_SHIFT              19
 126 #define RQ_CQE_PKT_LAST_LEN_SHIFT               6
 127 #define RQ_CQE_SUPER_CQE_EN_SHIFT               0
 128
 129 #define RQ_CQE_PKT_FIRST_LEN_MASK               0x1FFFU
 130 #define RQ_CQE_PKT_LAST_LEN_MASK                0x1FFFU
 131 #define RQ_CQE_PKT_NUM_MASK                     0x1FU
 132 #define RQ_CQE_SUPER_CQE_EN_MASK                0x1
 133
 134 #define RQ_CQE_PKT_NUM_GET(val, member)         \
 135         (((val) >> RQ_CQE_PKT_##member##_SHIFT) & RQ_CQE_PKT_##member##_MASK)
 136
 137 #define HINIC_GET_RQ_CQE_PKT_NUM(pkt_info) RQ_CQE_PKT_NUM_GET(pkt_info, NUM)
 138
 139 #define RQ_CQE_SUPER_CQE_EN_GET(val, member)    \
 140         (((val) >> RQ_CQE_##member##_SHIFT) & RQ_CQE_##member##_MASK)
 141
 142 #define HINIC_GET_SUPER_CQE_EN(pkt_info)        \
 143         RQ_CQE_SUPER_CQE_EN_GET(pkt_info, SUPER_CQE_EN)
 144
 145 #define RQ_CQE_OFFOLAD_TYPE_VLAN_EN_SHIFT               21
 146 #define RQ_CQE_OFFOLAD_TYPE_VLAN_EN_MASK                0x1U
 147
 148 #define RQ_CQE_OFFOLAD_TYPE_PKT_TYPE_SHIFT              0
 149 #define RQ_CQE_OFFOLAD_TYPE_PKT_TYPE_MASK               0xFFFU
 150
 151 #define RQ_CQE_OFFOLAD_TYPE_PKT_UMBCAST_SHIFT           19
 152 #define RQ_CQE_OFFOLAD_TYPE_PKT_UMBCAST_MASK            0x3U
 153
 154 #define RQ_CQE_OFFOLAD_TYPE_RSS_TYPE_SHIFT              24
 155 #define RQ_CQE_OFFOLAD_TYPE_RSS_TYPE_MASK               0xFFU
 156
 157 #define RQ_CQE_OFFOLAD_TYPE_GET(val, member)            (((val) >> \
 158                                 RQ_CQE_OFFOLAD_TYPE_##member##_SHIFT) & \
 159                                 RQ_CQE_OFFOLAD_TYPE_##member##_MASK)
 160
 161 #define HINIC_GET_RX_VLAN_OFFLOAD_EN(offload_type)      \
 162                 RQ_CQE_OFFOLAD_TYPE_GET(offload_type, VLAN_EN)
 163
 164 #define HINIC_GET_RSS_TYPES(offload_type)       \
 165                 RQ_CQE_OFFOLAD_TYPE_GET(offload_type, RSS_TYPE)
 166
 167 #define HINIC_GET_RX_PKT_TYPE(offload_type)     \
 168                 RQ_CQE_OFFOLAD_TYPE_GET(offload_type, PKT_TYPE)
 169
 170 #define HINIC_GET_RX_PKT_UMBCAST(offload_type)  \
 171                 RQ_CQE_OFFOLAD_TYPE_GET(offload_type, PKT_UMBCAST)
 172
 173 #define RQ_CQE_STATUS_CSUM_BYPASS_VAL                   0x80U
 174 #define RQ_CQE_STATUS_CSUM_ERR_IP_MASK                  0x39U
 175 #define RQ_CQE_STATUS_CSUM_ERR_L4_MASK                  0x46U
 176 #define RQ_CQE_STATUS_CSUM_ERR_OTHER                    0x100U
 177
 178 #define HINIC_CSUM_ERR_BYPASSED(csum_err)        \
 179         ((csum_err) == RQ_CQE_STATUS_CSUM_BYPASS_VAL)
 180
 181 #define HINIC_CSUM_ERR_IP(csum_err)      \
 182         ((csum_err) & RQ_CQE_STATUS_CSUM_ERR_IP_MASK)
 183
 184 #define HINIC_CSUM_ERR_L4(csum_err)      \
 185         ((csum_err) & RQ_CQE_STATUS_CSUM_ERR_L4_MASK)
 186
 187 #define HINIC_CSUM_ERR_OTHER(csum_err)   \
 188         ((csum_err) == RQ_CQE_STATUS_CSUM_ERR_OTHER)
 189
 190
 191 void hinic_get_func_rx_buf_size(struct hinic_nic_dev *nic_dev)
 192 {
 193         struct hinic_rxq *rxq;
 194         u16 q_id;
 195         u16 buf_size = 0;
 196
 197         for (q_id = 0; q_id < nic_dev->num_rq; q_id++) {
 198                 rxq = nic_dev->rxqs[q_id];
 199
 200                 if (rxq == NULL)
 201                         continue;
 202
 203                 if (q_id == 0)
 204                         buf_size = rxq->buf_len;
 205
 206                 buf_size = buf_size > rxq->buf_len ? rxq->buf_len : buf_size;
 207         }
 208
 209         nic_dev->hwdev->nic_io->rq_buf_size = buf_size;
 210 }
 211
 212 int hinic_create_rq(struct hinic_hwdev *hwdev, u16 q_id, u16 rq_depth)
 213 {
 214         int err;
 215         struct hinic_nic_io *nic_io = hwdev->nic_io;
 216         struct hinic_qp *qp = &nic_io->qps[q_id];
 217         struct hinic_rq *rq = &qp->rq;
 218
 219         /* in case of hardware still generate interrupt, do not use msix 0 */
 220         rq->msix_entry_idx = 1;
 221         rq->q_id = q_id;
 222         rq->rq_depth = rq_depth;
 223         nic_io->rq_depth = rq_depth;
 224
 225         err = hinic_wq_allocate(hwdev, &nic_io->rq_wq[q_id],
 226                                 HINIC_RQ_WQEBB_SHIFT, nic_io->rq_depth);
 227         if (err) {
 228                 PMD_DRV_LOG(ERR, "Failed to allocate WQ for RQ");
 229                 return err;
 230         }
 231         rq->wq = &nic_io->rq_wq[q_id];
 232
 233         rq->pi_virt_addr =
 234                 (volatile u16 *)dma_zalloc_coherent(hwdev, HINIC_PAGE_SIZE,
 235                                                     &rq->pi_dma_addr,
 236                                                     GFP_KERNEL);
 237         if (!rq->pi_virt_addr) {
 238                 PMD_DRV_LOG(ERR, "Failed to allocate rq pi virt addr");
 239                 err = -ENOMEM;
 240                 goto rq_pi_alloc_err;
 241         }
 242
 243         return HINIC_OK;
 244
 245 rq_pi_alloc_err:
 246         hinic_wq_free(hwdev, &nic_io->rq_wq[q_id]);
 247
 248         return err;
 249 }
 250
 251 void hinic_destroy_rq(struct hinic_hwdev *hwdev, u16 q_id)
 252 {
 253         struct hinic_nic_io *nic_io = hwdev->nic_io;
 254         struct hinic_qp *qp = &nic_io->qps[q_id];
 255         struct hinic_rq *rq = &qp->rq;
 256
 257         if (qp->rq.wq == NULL)
 258                 return;
 259
 260         dma_free_coherent_volatile(hwdev, HINIC_PAGE_SIZE,
 261                                    (volatile void *)rq->pi_virt_addr,
 262                                    rq->pi_dma_addr);
 263         hinic_wq_free(nic_io->hwdev, qp->rq.wq);
 264         qp->rq.wq = NULL;
 265 }
 266
 267 static void
 268 hinic_prepare_rq_wqe(void *wqe, __rte_unused u16 pi, dma_addr_t buf_addr,
 269                         dma_addr_t cqe_dma)
 270 {
 271         struct hinic_rq_wqe *rq_wqe = wqe;
 272         struct hinic_rq_ctrl *ctrl = &rq_wqe->ctrl;
 273         struct hinic_rq_cqe_sect *cqe_sect = &rq_wqe->cqe_sect;
 274         struct hinic_rq_bufdesc *buf_desc = &rq_wqe->buf_desc;
 275         u32 rq_ceq_len = sizeof(struct hinic_rq_cqe);
 276
 277         ctrl->ctrl_fmt =
 278                 RQ_CTRL_SET(SIZE_8BYTES(sizeof(*ctrl)),  LEN) |
 279                 RQ_CTRL_SET(SIZE_8BYTES(sizeof(*cqe_sect)), COMPLETE_LEN) |
 280                 RQ_CTRL_SET(SIZE_8BYTES(sizeof(*buf_desc)), BUFDESC_SECT_LEN) |
 281                 RQ_CTRL_SET(RQ_COMPLETE_SGE, COMPLETE_FORMAT);
 282
 283         hinic_set_sge(&cqe_sect->sge, cqe_dma, rq_ceq_len);
 284
 285         buf_desc->addr_high = upper_32_bits(buf_addr);
 286         buf_desc->addr_low = lower_32_bits(buf_addr);
 287 }
 288
 289 void hinic_rxq_get_stats(struct hinic_rxq *rxq, struct hinic_rxq_stats *stats)
 290 {
 291         if (!rxq || !stats)
 292                 return;
 293
 294         memcpy(stats, &rxq->rxq_stats, sizeof(rxq->rxq_stats));
 295 }
 296
 297 void hinic_rxq_stats_reset(struct hinic_rxq *rxq)
 298 {
 299         struct hinic_rxq_stats *rxq_stats;
 300
 301         if (rxq == NULL)
 302                 return;
 303
 304         rxq_stats = &rxq->rxq_stats;
 305         memset(rxq_stats, 0, sizeof(*rxq_stats));
 306 }
 307
 308 static int hinic_rx_alloc_cqe(struct hinic_rxq *rxq)
 309 {
 310         size_t cqe_mem_size;
 311
 312         /* allocate continuous cqe memory for saving number of memory zone */
 313         cqe_mem_size = sizeof(struct hinic_rq_cqe) * rxq->q_depth;
 314         rxq->cqe_start_vaddr =
 315                 dma_zalloc_coherent(rxq->nic_dev->hwdev,
 316                                     cqe_mem_size, &rxq->cqe_start_paddr,
 317                                     GFP_KERNEL);
 318         if (!rxq->cqe_start_vaddr) {
 319                 PMD_DRV_LOG(ERR, "Allocate cqe dma memory failed");
 320                 return -ENOMEM;
 321         }
 322
 323         rxq->rx_cqe = (struct hinic_rq_cqe *)rxq->cqe_start_vaddr;
 324
 325         return HINIC_OK;
 326 }
 327
 328 static void hinic_rx_free_cqe(struct hinic_rxq *rxq)
 329 {
 330         size_t cqe_mem_size;
 331
 332         cqe_mem_size = sizeof(struct hinic_rq_cqe) * rxq->q_depth;
 333         dma_free_coherent(rxq->nic_dev->hwdev, cqe_mem_size,
 334                           rxq->cqe_start_vaddr, rxq->cqe_start_paddr);
 335         rxq->cqe_start_vaddr = NULL;
 336 }
 337
 338 static int hinic_rx_fill_wqe(struct hinic_rxq *rxq)
 339 {
 340         struct hinic_nic_dev *nic_dev = rxq->nic_dev;
 341         struct hinic_rq_wqe *rq_wqe;
 342         dma_addr_t buf_dma_addr, cqe_dma_addr;
 343         u16 pi = 0;
 344         int i;
 345
 346         buf_dma_addr = 0;
 347         cqe_dma_addr = rxq->cqe_start_paddr;
 348         for (i = 0; i < rxq->q_depth; i++) {
 349                 rq_wqe = hinic_get_rq_wqe(nic_dev->hwdev, rxq->q_id, &pi);
 350                 if (!rq_wqe) {
 351                         PMD_DRV_LOG(ERR, "Get rq wqe failed");
 352                         break;
 353                 }
 354
 355                 hinic_prepare_rq_wqe(rq_wqe, pi, buf_dma_addr, cqe_dma_addr);
 356                 cqe_dma_addr +=  sizeof(struct hinic_rq_cqe);
 357
 358                 hinic_cpu_to_be32(rq_wqe, sizeof(struct hinic_rq_wqe));
 359         }
 360
 361         hinic_return_rq_wqe(nic_dev->hwdev, rxq->q_id, i);
 362
 363         return i;
 364 }
 365
 366 /* alloc cqe and prepare rqe */
 367 int hinic_setup_rx_resources(struct hinic_rxq *rxq)
 368 {
 369         u64 rx_info_sz;
 370         int err, pkts;
 371
 372         rx_info_sz = rxq->q_depth * sizeof(*rxq->rx_info);
 373         rxq->rx_info = kzalloc_aligned(rx_info_sz, GFP_KERNEL);
 374         if (!rxq->rx_info)
 375                 return -ENOMEM;
 376
 377         err = hinic_rx_alloc_cqe(rxq);
 378         if (err) {
 379                 PMD_DRV_LOG(ERR, "Allocate rx cqe failed");
 380                 goto rx_cqe_err;
 381         }
 382
 383         pkts = hinic_rx_fill_wqe(rxq);
 384         if (pkts != rxq->q_depth) {
 385                 PMD_DRV_LOG(ERR, "Fill rx wqe failed");
 386                 err = -ENOMEM;
 387                 goto rx_fill_err;
 388         }
 389
 390         return 0;
 391
 392 rx_fill_err:
 393         hinic_rx_free_cqe(rxq);
 394
 395 rx_cqe_err:
 396         kfree(rxq->rx_info);
 397         rxq->rx_info = NULL;
 398
 399         return err;
 400 }
 401
 402 void hinic_free_rx_resources(struct hinic_rxq *rxq)
 403 {
 404         if (rxq->rx_info == NULL)
 405                 return;
 406
 407         hinic_rx_free_cqe(rxq);
 408         kfree(rxq->rx_info);
 409         rxq->rx_info = NULL;
 410 }
 411
 412 void hinic_free_all_rx_resources(struct rte_eth_dev *eth_dev)
 413 {
 414         u16 q_id;
 415         struct hinic_nic_dev *nic_dev =
 416                                 HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
 417
 418         for (q_id = 0; q_id < nic_dev->num_rq; q_id++) {
 419                 eth_dev->data->rx_queues[q_id] = NULL;
 420
 421                 if (nic_dev->rxqs[q_id] == NULL)
 422                         continue;
 423
 424                 hinic_free_all_rx_skbs(nic_dev->rxqs[q_id]);
 425                 hinic_free_rx_resources(nic_dev->rxqs[q_id]);
 426                 kfree(nic_dev->rxqs[q_id]);
 427                 nic_dev->rxqs[q_id] = NULL;
 428         }
 429 }
 430
 431 void hinic_free_all_rx_mbuf(struct rte_eth_dev *eth_dev)
 432 {
 433         struct hinic_nic_dev *nic_dev =
 434                                 HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
 435         u16 q_id;
 436
 437         for (q_id = 0; q_id < nic_dev->num_rq; q_id++)
 438                 hinic_free_all_rx_skbs(nic_dev->rxqs[q_id]);
 439 }
 440
 441 static void hinic_recv_jumbo_pkt(struct hinic_rxq *rxq,
 442                                  struct rte_mbuf *head_skb,
 443                                  u32 remain_pkt_len)
 444 {
 445         struct hinic_nic_dev *nic_dev = rxq->nic_dev;
 446         struct rte_mbuf *cur_mbuf, *rxm = NULL;
 447         struct hinic_rx_info *rx_info;
 448         u16 sw_ci, rx_buf_len = rxq->buf_len;
 449         u32 pkt_len;
 450
 451         while (remain_pkt_len > 0) {
 452                 sw_ci = hinic_get_rq_local_ci(nic_dev->hwdev, rxq->q_id);
 453                 rx_info = &rxq->rx_info[sw_ci];
 454
 455                 hinic_update_rq_local_ci(nic_dev->hwdev, rxq->q_id, 1);
 456
 457                 pkt_len = remain_pkt_len > rx_buf_len ?
 458                         rx_buf_len : remain_pkt_len;
 459                 remain_pkt_len -= pkt_len;
 460
 461                 cur_mbuf = rx_info->mbuf;
 462                 cur_mbuf->data_len = (u16)pkt_len;
 463                 cur_mbuf->next = NULL;
 464
 465                 head_skb->pkt_len += cur_mbuf->data_len;
 466                 head_skb->nb_segs++;
 467
 468                 if (!rxm)
 469                         head_skb->next = cur_mbuf;
 470                 else
 471                         rxm->next = cur_mbuf;
 472
 473                 rxm = cur_mbuf;
 474         }
 475 }
 476
 477 static void hinic_rss_deinit(struct hinic_nic_dev *nic_dev)
 478 {
 479         u8 prio_tc[HINIC_DCB_UP_MAX] = {0};
 480         (void)hinic_rss_cfg(nic_dev->hwdev, 0,
 481                             nic_dev->rss_tmpl_idx, 0, prio_tc);
 482 }
 483
 484 static int hinic_rss_key_init(struct hinic_nic_dev *nic_dev,
 485                               struct rte_eth_rss_conf *rss_conf)
 486 {
 487         u8 default_rss_key[HINIC_RSS_KEY_SIZE] = {
 488                          0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2,
 489                          0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0,
 490                          0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4,
 491                          0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c,
 492                          0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa};
 493         u8 hashkey[HINIC_RSS_KEY_SIZE] = {0};
 494         u8 tmpl_idx = nic_dev->rss_tmpl_idx;
 495
 496         if (rss_conf->rss_key == NULL)
 497                 memcpy(hashkey, default_rss_key, HINIC_RSS_KEY_SIZE);
 498         else
 499                 memcpy(hashkey, rss_conf->rss_key, rss_conf->rss_key_len);
 500
 501         return hinic_rss_set_template_tbl(nic_dev->hwdev, tmpl_idx, hashkey);
 502 }
 503
 504 static void hinic_fill_rss_type(struct nic_rss_type *rss_type,
 505                                 struct rte_eth_rss_conf *rss_conf)
 506 {
 507         u64 rss_hf = rss_conf->rss_hf;
 508
 509         rss_type->ipv4 = (rss_hf & (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4)) ? 1 : 0;
 510         rss_type->tcp_ipv4 = (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) ? 1 : 0;
 511         rss_type->ipv6 = (rss_hf & (ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6)) ? 1 : 0;
 512         rss_type->ipv6_ext = (rss_hf & ETH_RSS_IPV6_EX) ? 1 : 0;
 513         rss_type->tcp_ipv6 = (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP) ? 1 : 0;
 514         rss_type->tcp_ipv6_ext = (rss_hf & ETH_RSS_IPV6_TCP_EX) ? 1 : 0;
 515         rss_type->udp_ipv4 = (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) ? 1 : 0;
 516         rss_type->udp_ipv6 = (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP) ? 1 : 0;
 517 }
 518
 519 static void hinic_fillout_indir_tbl(struct hinic_nic_dev *nic_dev, u32 *indir)
 520 {
 521         u8 rss_queue_count = nic_dev->num_rss;
 522         int i = 0, j;
 523
 524         if (rss_queue_count == 0) {
 525                 /* delete q_id from indir tbl */
 526                 for (i = 0; i < HINIC_RSS_INDIR_SIZE; i++)
 527                         indir[i] = 0xFF;        /* Invalid value in indir tbl */
 528         } else {
 529                 while (i < HINIC_RSS_INDIR_SIZE)
 530                         for (j = 0; (j < rss_queue_count) &&
 531                              (i < HINIC_RSS_INDIR_SIZE); j++)
 532                                 indir[i++] = nic_dev->rx_queue_list[j];
 533         }
 534 }
 535
 536 static int hinic_rss_init(struct hinic_nic_dev *nic_dev,
 537                           __attribute__((unused)) u8 *rq2iq_map,
 538                           struct rte_eth_rss_conf *rss_conf)
 539 {
 540         u32 indir_tbl[HINIC_RSS_INDIR_SIZE] = {0};
 541         struct nic_rss_type rss_type = {0};
 542         u8 prio_tc[HINIC_DCB_UP_MAX] = {0};
 543         u8 tmpl_idx = 0xFF, num_tc = 0;
 544         int err;
 545
 546         tmpl_idx = nic_dev->rss_tmpl_idx;
 547
 548         err = hinic_rss_key_init(nic_dev, rss_conf);
 549         if (err)
 550                 return err;
 551
 552         if (!nic_dev->rss_indir_flag) {
 553                 hinic_fillout_indir_tbl(nic_dev, indir_tbl);
 554                 err = hinic_rss_set_indir_tbl(nic_dev->hwdev, tmpl_idx,
 555                                               indir_tbl);
 556                 if (err)
 557                         return err;
 558         }
 559
 560         hinic_fill_rss_type(&rss_type, rss_conf);
 561         err = hinic_set_rss_type(nic_dev->hwdev, tmpl_idx, rss_type);
 562         if (err)
 563                 return err;
 564
 565         err = hinic_rss_set_hash_engine(nic_dev->hwdev, tmpl_idx,
 566                                         HINIC_RSS_HASH_ENGINE_TYPE_TOEP);
 567         if (err)
 568                 return err;
 569
 570         return hinic_rss_cfg(nic_dev->hwdev, 1, tmpl_idx, num_tc, prio_tc);
 571 }
 572
 573 static void
 574 hinic_add_rq_to_rx_queue_list(struct hinic_nic_dev *nic_dev, u16 queue_id)
 575 {
 576         u8 rss_queue_count = nic_dev->num_rss;
 577
 578         RTE_ASSERT(rss_queue_count <= (RTE_DIM(nic_dev->rx_queue_list) - 1));
 579
 580         nic_dev->rx_queue_list[rss_queue_count] = queue_id;
 581         nic_dev->num_rss++;
 582 }
 583
 584 /**
 585  * hinic_setup_num_qps - determine num_qps from rss_tmpl_id
 586  * @nic_dev: pointer to the private ethernet device
 587  * Return: 0 on Success, error code otherwise.
 588  **/
 589 static int hinic_setup_num_qps(struct hinic_nic_dev *nic_dev)
 590 {
 591         int err, i;
 592
 593         if (!(nic_dev->flags & ETH_MQ_RX_RSS_FLAG)) {
 594                 nic_dev->flags &= ~ETH_MQ_RX_RSS_FLAG;
 595                 nic_dev->num_rss = 0;
 596                 if (nic_dev->num_rq > 1) {
 597                         /* get rss template id */
 598                         err = hinic_rss_template_alloc(nic_dev->hwdev,
 599                                                        &nic_dev->rss_tmpl_idx);
 600                         if (err) {
 601                                 PMD_DRV_LOG(WARNING, "Alloc rss template failed");
 602                                 return err;
 603                         }
 604                         nic_dev->flags |= ETH_MQ_RX_RSS_FLAG;
 605                         for (i = 0; i < nic_dev->num_rq; i++)
 606                                 hinic_add_rq_to_rx_queue_list(nic_dev, i);
 607                 }
 608         }
 609
 610         return 0;
 611 }
 612
 613 static void hinic_destroy_num_qps(struct hinic_nic_dev *nic_dev)
 614 {
 615         if (nic_dev->flags & ETH_MQ_RX_RSS_FLAG) {
 616                 if (hinic_rss_template_free(nic_dev->hwdev,
 617                                             nic_dev->rss_tmpl_idx))
 618                         PMD_DRV_LOG(WARNING, "Free rss template failed");
 619
 620                 nic_dev->flags &= ~ETH_MQ_RX_RSS_FLAG;
 621         }
 622 }
 623
 624 static int hinic_config_mq_rx_rss(struct hinic_nic_dev *nic_dev, bool on)
 625 {
 626         int ret = 0;
 627
 628         if (on) {
 629                 ret = hinic_setup_num_qps(nic_dev);
 630                 if (ret)
 631                         PMD_DRV_LOG(ERR, "Setup num_qps failed");
 632         } else {
 633                 hinic_destroy_num_qps(nic_dev);
 634         }
 635
 636         return ret;
 637 }
 638
 639 int hinic_config_mq_mode(struct rte_eth_dev *dev, bool on)
 640 {
 641         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 642         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 643         int ret = 0;
 644
 645         switch (dev_conf->rxmode.mq_mode) {
 646         case ETH_MQ_RX_RSS:
 647                 ret = hinic_config_mq_rx_rss(nic_dev, on);
 648                 break;
 649         default:
 650                 break;
 651         }
 652
 653         return ret;
 654 }
 655
 656 int hinic_rx_configure(struct rte_eth_dev *dev)
 657 {
 658         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 659         struct rte_eth_rss_conf rss_conf =
 660                 dev->data->dev_conf.rx_adv_conf.rss_conf;
 661         u32 csum_en = 0;
 662         int err;
 663
 664         if (nic_dev->flags & ETH_MQ_RX_RSS_FLAG) {
 665                 if (rss_conf.rss_hf == 0) {
 666                         rss_conf.rss_hf = HINIC_RSS_OFFLOAD_ALL;
 667                 } else if ((rss_conf.rss_hf & HINIC_RSS_OFFLOAD_ALL) == 0) {
 668                         PMD_DRV_LOG(ERR, "Do not support rss offload all");
 669                         goto rss_config_err;
 670                 }
 671
 672                 err = hinic_rss_init(nic_dev, NULL, &rss_conf);
 673                 if (err) {
 674                         PMD_DRV_LOG(ERR, "Init rss failed");
 675                         goto rss_config_err;
 676                 }
 677         }
 678
 679         /* Enable both L3/L4 rx checksum offload */
 680         if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_CHECKSUM)
 681                 csum_en = HINIC_RX_CSUM_OFFLOAD_EN;
 682
 683         err = hinic_set_rx_csum_offload(nic_dev->hwdev, csum_en);
 684         if (err)
 685                 goto rx_csum_ofl_err;
 686
 687         return 0;
 688
 689 rx_csum_ofl_err:
 690 rss_config_err:
 691         hinic_destroy_num_qps(nic_dev);
 692
 693         return HINIC_ERROR;
 694 }
 695
 696 void hinic_rx_remove_configure(struct rte_eth_dev *dev)
 697 {
 698         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 699
 700         if (nic_dev->flags & ETH_MQ_RX_RSS_FLAG) {
 701                 hinic_rss_deinit(nic_dev);
 702                 hinic_destroy_num_qps(nic_dev);
 703         }
 704 }
 705
 706 void hinic_free_all_rx_skbs(struct hinic_rxq *rxq)
 707 {
 708         struct hinic_nic_dev *nic_dev = rxq->nic_dev;
 709         struct hinic_rx_info *rx_info;
 710         int free_wqebbs =
 711                 hinic_get_rq_free_wqebbs(nic_dev->hwdev, rxq->q_id) + 1;
 712         volatile struct hinic_rq_cqe *rx_cqe;
 713         u16 ci;
 714
 715         while (free_wqebbs++ < rxq->q_depth) {
 716                 ci = hinic_get_rq_local_ci(nic_dev->hwdev, rxq->q_id);
 717
 718                 rx_cqe = &rxq->rx_cqe[ci];
 719
 720                 /* clear done bit */
 721                 rx_cqe->status = 0;
 722
 723                 rx_info = &rxq->rx_info[ci];
 724                 rte_pktmbuf_free(rx_info->mbuf);
 725                 rx_info->mbuf = NULL;
 726
 727                 hinic_update_rq_local_ci(nic_dev->hwdev, rxq->q_id, 1);
 728         }
 729 }
 730
 731 static inline void hinic_rq_cqe_be_to_cpu32(void *dst_le32,
 732                                             volatile void *src_be32)
 733 {
 734 #if defined(__X86_64_SSE__)
 735         volatile __m128i *wqe_be = (volatile __m128i *)src_be32;
 736         __m128i *wqe_le = (__m128i *)dst_le32;
 737         __m128i shuf_mask =  _mm_set_epi8(12, 13, 14, 15, 8, 9, 10,
 738                                         11, 4, 5, 6, 7, 0, 1, 2, 3);
 739
 740         /* l2nic just use first 128 bits */
 741         wqe_le[0] = _mm_shuffle_epi8(wqe_be[0], shuf_mask);
 742 #elif defined(__ARM64_NEON__)
 743         volatile uint8x16_t *wqe_be = (volatile uint8x16_t *)src_be32;
 744         uint8x16_t *wqe_le = (uint8x16_t *)dst_le32;
 745         const uint8x16_t shuf_mask = {3, 2, 1, 0, 7, 6, 5, 4, 11, 10,
 746                                         9, 8, 15, 14, 13, 12};
 747
 748         /* l2nic just use first 128 bits */
 749         wqe_le[0] = vqtbl1q_u8(wqe_be[0], shuf_mask);
 750 #else
 751         u32 i;
 752         volatile u32 *wqe_be = (volatile u32 *)src_be32;
 753         u32 *wqe_le = (u32 *)dst_le32;
 754
 755 #define HINIC_L2NIC_RQ_CQE_USED         4 /* 4Bytes unit */
 756
 757         for (i = 0; i < HINIC_L2NIC_RQ_CQE_USED; i++) {
 758                 *wqe_le = rte_be_to_cpu_32(*wqe_be);
 759                 wqe_be++;
 760                 wqe_le++;
 761         }
 762 #endif
 763 }
 764
 765 static inline uint64_t hinic_rx_rss_hash(uint32_t offload_type,
 766                                          uint32_t cqe_hass_val,
 767                                          uint32_t *rss_hash)
 768 {
 769         uint32_t rss_type;
 770
 771         rss_type = HINIC_GET_RSS_TYPES(offload_type);
 772         if (likely(rss_type != 0)) {
 773                 *rss_hash = cqe_hass_val;
 774                 return PKT_RX_RSS_HASH;
 775         }
 776
 777         return 0;
 778 }
 779
 780 static inline uint64_t hinic_rx_csum(uint32_t status, struct hinic_rxq *rxq)
 781 {
 782         uint32_t checksum_err;
 783         uint64_t flags;
 784
 785         /* most case checksum is ok */
 786         checksum_err = HINIC_GET_RX_CSUM_ERR(status);
 787         if (likely(checksum_err == 0))
 788                 return (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD);
 789
 790         /* If BYPASS bit set, all other status indications should be ignored */
 791         if (unlikely(HINIC_CSUM_ERR_BYPASSED(checksum_err)))
 792                 return PKT_RX_IP_CKSUM_UNKNOWN;
 793
 794         flags = 0;
 795
 796         /* IP checksum error */
 797         if (HINIC_CSUM_ERR_IP(checksum_err))
 798                 flags |= PKT_RX_IP_CKSUM_BAD;
 799         else
 800                 flags |= PKT_RX_IP_CKSUM_GOOD;
 801
 802         /* L4 checksum error */
 803         if (HINIC_CSUM_ERR_L4(checksum_err))
 804                 flags |= PKT_RX_L4_CKSUM_BAD;
 805         else
 806                 flags |= PKT_RX_L4_CKSUM_GOOD;
 807
 808         if (unlikely(HINIC_CSUM_ERR_OTHER(checksum_err)))
 809                 flags = PKT_RX_L4_CKSUM_NONE;
 810
 811         rxq->rxq_stats.errors++;
 812
 813         return flags;
 814 }
 815
 816 static inline uint64_t hinic_rx_vlan(uint32_t offload_type, uint32_t vlan_len,
 817                                      uint16_t *vlan_tci)
 818 {
 819         uint16_t vlan_tag;
 820
 821         vlan_tag = HINIC_GET_RX_VLAN_TAG(vlan_len);
 822         if (!HINIC_GET_RX_VLAN_OFFLOAD_EN(offload_type) || 0 == vlan_tag) {
 823                 *vlan_tci = 0;
 824                 return 0;
 825         }
 826
 827         *vlan_tci = vlan_tag;
 828
 829         return PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED;
 830 }
 831
 832 static inline u32 hinic_rx_alloc_mbuf_bulk(struct hinic_rxq *rxq,
 833                                            struct rte_mbuf **mbufs,
 834                                            u32 exp_mbuf_cnt)
 835 {
 836         int rc;
 837         u32 avail_cnt;
 838
 839         rc = rte_pktmbuf_alloc_bulk(rxq->mb_pool, mbufs, exp_mbuf_cnt);
 840         if (likely(rc == HINIC_OK)) {
 841                 avail_cnt = exp_mbuf_cnt;
 842         } else {
 843                 avail_cnt = 0;
 844                 rxq->rxq_stats.rx_nombuf += exp_mbuf_cnt;
 845         }
 846
 847         return avail_cnt;
 848 }
 849
 850 static struct rte_mbuf *hinic_rx_alloc_mbuf(struct hinic_rxq *rxq,
 851                                         dma_addr_t *dma_addr)
 852 {
 853         struct rte_mbuf *mbuf;
 854
 855         mbuf = rte_mbuf_raw_alloc(rxq->mb_pool);
 856         if (unlikely(!mbuf))
 857                 return NULL;
 858
 859         *dma_addr = rte_mbuf_data_iova_default(mbuf);
 860
 861         return mbuf;
 862 }
 863
 864 static inline void hinic_rearm_rxq_mbuf(struct hinic_rxq *rxq)
 865 {
 866         u16 pi;
 867         u32 i, free_wqebbs, rearm_wqebbs, exp_wqebbs;
 868         dma_addr_t dma_addr;
 869         struct hinic_rq_wqe *rq_wqe;
 870         struct rte_mbuf **rearm_mbufs;
 871
 872         /* check free wqebb fo rearm */
 873         free_wqebbs = HINIC_GET_RQ_FREE_WQEBBS(rxq);
 874         if (unlikely(free_wqebbs < rxq->rx_free_thresh))
 875                 return;
 876
 877         /* get rearm mbuf array */
 878         pi = HINIC_GET_RQ_LOCAL_PI(rxq);
 879         rearm_mbufs = (struct rte_mbuf **)(&rxq->rx_info[pi]);
 880
 881         /* check rxq free wqebbs turn around */
 882         exp_wqebbs = rxq->q_depth - pi;
 883         if (free_wqebbs < exp_wqebbs)
 884                 exp_wqebbs = free_wqebbs;
 885
 886         /* alloc mbuf in bulk */
 887         rearm_wqebbs = hinic_rx_alloc_mbuf_bulk(rxq, rearm_mbufs, exp_wqebbs);
 888         if (unlikely(rearm_wqebbs == 0))
 889                 return;
 890
 891         /* rearm rx mbuf */
 892         rq_wqe = WQ_WQE_ADDR(rxq->wq, (u32)pi);
 893         for (i = 0; i < rearm_wqebbs; i++) {
 894                 dma_addr = rte_mbuf_data_iova_default(rearm_mbufs[i]);
 895                 rq_wqe->buf_desc.addr_high =
 896                                         cpu_to_be32(upper_32_bits(dma_addr));
 897                 rq_wqe->buf_desc.addr_low =
 898                                         cpu_to_be32(lower_32_bits(dma_addr));
 899                 rq_wqe++;
 900         }
 901         rxq->wq->prod_idx += rearm_wqebbs;
 902         rxq->wq->delta -= rearm_wqebbs;
 903
 904         /* update rq hw_pi */
 905         rte_wmb();
 906         HINIC_UPDATE_RQ_HW_PI(rxq, pi + rearm_wqebbs);
 907 }
 908
 909 void hinic_rx_alloc_pkts(struct hinic_rxq *rxq)
 910 {
 911         struct hinic_nic_dev *nic_dev = rxq->nic_dev;
 912         struct hinic_rq_wqe *rq_wqe;
 913         struct hinic_rx_info *rx_info;
 914         struct rte_mbuf *mb;
 915         dma_addr_t dma_addr;
 916         u16 pi = 0;
 917         int i, free_wqebbs;
 918
 919         free_wqebbs = HINIC_GET_RQ_FREE_WQEBBS(rxq);
 920         for (i = 0; i < free_wqebbs; i++) {
 921                 mb = hinic_rx_alloc_mbuf(rxq, &dma_addr);
 922                 if (unlikely(!mb)) {
 923                         rxq->rxq_stats.rx_nombuf++;
 924                         break;
 925                 }
 926
 927                 rq_wqe = hinic_get_rq_wqe(nic_dev->hwdev, rxq->q_id, &pi);
 928                 if (unlikely(!rq_wqe)) {
 929                         rte_pktmbuf_free(mb);
 930                         break;
 931                 }
 932
 933                 /* fill buffer address only */
 934                 rq_wqe->buf_desc.addr_high =
 935                                 cpu_to_be32(upper_32_bits(dma_addr));
 936                 rq_wqe->buf_desc.addr_low =
 937                                 cpu_to_be32(lower_32_bits(dma_addr));
 938
 939                 rx_info = &rxq->rx_info[pi];
 940                 rx_info->mbuf = mb;
 941         }
 942
 943         if (likely(i > 0)) {
 944                 rte_wmb();
 945                 HINIC_UPDATE_RQ_HW_PI(rxq, pi + 1);
 946         }
 947 }
 948
 949 u16 hinic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, u16 nb_pkts)
 950 {
 951         struct rte_mbuf *rxm;
 952         struct hinic_rxq *rxq = rx_queue;
 953         struct hinic_rx_info *rx_info;
 954         volatile struct hinic_rq_cqe *rx_cqe;
 955         u16 rx_buf_len, pkts = 0;
 956         u16 sw_ci, ci_mask, wqebb_cnt = 0;
 957         u32 pkt_len, status, vlan_len;
 958         u64 rx_bytes = 0;
 959         struct hinic_rq_cqe cqe;
 960         u32 offload_type, rss_hash;
 961
 962         rx_buf_len = rxq->buf_len;
 963
 964         /* 1. get polling start ci */
 965         ci_mask = HINIC_GET_RQ_WQE_MASK(rxq);
 966         sw_ci = HINIC_GET_RQ_LOCAL_CI(rxq);
 967
 968         while (pkts < nb_pkts) {
 969                  /* 2. current ci is done */
 970                 rx_cqe = &rxq->rx_cqe[sw_ci];
 971                 status = rx_cqe->status;
 972                 if (!HINIC_GET_RX_DONE_BE(status))
 973                         break;
 974
 975                 /* read other cqe member after status */
 976                 rte_rmb();
 977
 978                 /* convert cqe and get packet length */
 979                 hinic_rq_cqe_be_to_cpu32(&cqe, (volatile void *)rx_cqe);
 980                 vlan_len = cqe.vlan_len;
 981
 982                 rx_info = &rxq->rx_info[sw_ci];
 983                 rxm = rx_info->mbuf;
 984
 985                 /* 3. next ci point and prefetch */
 986                 sw_ci++;
 987                 sw_ci &= ci_mask;
 988
 989                 /* prefetch next mbuf first 64B */
 990                 rte_prefetch0(rxq->rx_info[sw_ci].mbuf);
 991
 992                 /* 4. jumbo frame process */
 993                 pkt_len = HINIC_GET_RX_PKT_LEN(vlan_len);
 994                 if (likely(pkt_len <= rx_buf_len)) {
 995                         rxm->data_len = pkt_len;
 996                         rxm->pkt_len = pkt_len;
 997                         wqebb_cnt++;
 998                 } else {
 999                         rxm->data_len = rx_buf_len;
1000                         rxm->pkt_len = rx_buf_len;
1001
1002                         /* if jumbo use multi-wqebb update ci,
1003                          * recv_jumbo_pkt will also update ci
1004                          */
1005                         HINIC_UPDATE_RQ_LOCAL_CI(rxq, wqebb_cnt + 1);
1006                         wqebb_cnt = 0;
1007                         hinic_recv_jumbo_pkt(rxq, rxm, pkt_len - rx_buf_len);
1008                         sw_ci = HINIC_GET_RQ_LOCAL_CI(rxq);
1009                 }
1010
1011                 /* 5. vlan/checksum/rss/pkt_type/gro offload */
1012                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1013                 rxm->port = rxq->port_id;
1014                 offload_type = cqe.offload_type;
1015
1016                 /* vlan offload */
1017                 rxm->ol_flags |= hinic_rx_vlan(offload_type, vlan_len,
1018                                                &rxm->vlan_tci);
1019
1020                 /* checksum offload */
1021                 rxm->ol_flags |= hinic_rx_csum(cqe.status, rxq);
1022
1023                 /* rss hash offload */
1024                 rss_hash = cqe.rss_hash;
1025                 rxm->ol_flags |= hinic_rx_rss_hash(offload_type, rss_hash,
1026                                                    &rxm->hash.rss);
1027
1028                 /* 6. clear done bit */
1029                 rx_cqe->status = 0;
1030
1031                 rx_bytes += pkt_len;
1032                 rx_pkts[pkts++] = rxm;
1033         }
1034
1035         if (pkts) {
1036                 /* 7. update ci */
1037                 HINIC_UPDATE_RQ_LOCAL_CI(rxq, wqebb_cnt);
1038
1039                 /* do packet stats */
1040                 rxq->rxq_stats.packets += pkts;
1041                 rxq->rxq_stats.bytes += rx_bytes;
1042         }
1043         rxq->rxq_stats.burst_pkts = pkts;
1044
1045         /* 8. rearm mbuf to rxq */
1046         hinic_rearm_rxq_mbuf(rxq);
1047
1048         return pkts;
1049 }