drivers/net/hinic/hinic_pmd_rx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Huawei Technologies Co., Ltd
   3  */
   4
   5 #include <rte_ether.h>
   6 #include <rte_mbuf.h>
   7 #ifdef __ARM64_NEON__
   8 #include <arm_neon.h>
   9 #endif
  10
  11 #include "base/hinic_compat.h"
  12 #include "base/hinic_pmd_hwdev.h"
  13 #include "base/hinic_pmd_wq.h"
  14 #include "base/hinic_pmd_niccfg.h"
  15 #include "base/hinic_pmd_nicio.h"
  16 #include "hinic_pmd_ethdev.h"
  17 #include "hinic_pmd_rx.h"
  18
  19 /* rxq wq operations */
  20 #define HINIC_GET_RQ_WQE_MASK(rxq)      \
  21         ((rxq)->wq->mask)
  22
  23 #define HINIC_GET_RQ_LOCAL_CI(rxq)      \
  24         (((rxq)->wq->cons_idx) & HINIC_GET_RQ_WQE_MASK(rxq))
  25
  26 #define HINIC_GET_RQ_LOCAL_PI(rxq)      \
  27         (((rxq)->wq->prod_idx) & HINIC_GET_RQ_WQE_MASK(rxq))
  28
  29 #define HINIC_UPDATE_RQ_LOCAL_CI(rxq, wqebb_cnt)        \
  30         do {                                            \
  31                 (rxq)->wq->cons_idx += (wqebb_cnt);     \
  32                 (rxq)->wq->delta += (wqebb_cnt);        \
  33         } while (0)
  34
  35 #define HINIC_UPDATE_RQ_HW_PI(rxq, pi)  \
  36         (*((rxq)->pi_virt_addr) =       \
  37                 cpu_to_be16((pi) & HINIC_GET_RQ_WQE_MASK(rxq)))
  38
  39 #define HINIC_GET_RQ_FREE_WQEBBS(rxq)   ((rxq)->wq->delta - 1)
  40
  41 /* rxq cqe done and status bit */
  42 #define HINIC_GET_RX_DONE_BE(status)    \
  43         ((status) & 0x80U)
  44
  45 #define HINIC_RX_CSUM_OFFLOAD_EN        0xFFF
  46
  47 #define RQ_CQE_SGE_VLAN_SHIFT                   0
  48 #define RQ_CQE_SGE_LEN_SHIFT                    16
  49
  50 #define RQ_CQE_SGE_VLAN_MASK                    0xFFFFU
  51 #define RQ_CQE_SGE_LEN_MASK                     0xFFFFU
  52
  53 #define RQ_CQE_SGE_GET(val, member)             \
  54         (((val) >> RQ_CQE_SGE_##member##_SHIFT) & RQ_CQE_SGE_##member##_MASK)
  55
  56 #define HINIC_GET_RX_VLAN_TAG(vlan_len) \
  57                 RQ_CQE_SGE_GET(vlan_len, VLAN)
  58
  59 #define HINIC_GET_RX_PKT_LEN(vlan_len)  \
  60                 RQ_CQE_SGE_GET(vlan_len, LEN)
  61
  62 #define RQ_CQE_STATUS_CSUM_ERR_SHIFT            0
  63 #define RQ_CQE_STATUS_NUM_LRO_SHIFT             16
  64 #define RQ_CQE_STATUS_LRO_PUSH_SHIFT            25
  65 #define RQ_CQE_STATUS_LRO_ENTER_SHIFT           26
  66 #define RQ_CQE_STATUS_LRO_INTR_SHIFT            27
  67
  68 #define RQ_CQE_STATUS_BP_EN_SHIFT               30
  69 #define RQ_CQE_STATUS_RXDONE_SHIFT              31
  70 #define RQ_CQE_STATUS_FLUSH_SHIFT               28
  71
  72 #define RQ_CQE_STATUS_CSUM_ERR_MASK             0xFFFFU
  73 #define RQ_CQE_STATUS_NUM_LRO_MASK              0xFFU
  74 #define RQ_CQE_STATUS_LRO_PUSH_MASK             0X1U
  75 #define RQ_CQE_STATUS_LRO_ENTER_MASK            0X1U
  76 #define RQ_CQE_STATUS_LRO_INTR_MASK             0X1U
  77 #define RQ_CQE_STATUS_BP_EN_MASK                0X1U
  78 #define RQ_CQE_STATUS_RXDONE_MASK               0x1U
  79 #define RQ_CQE_STATUS_FLUSH_MASK                0x1U
  80
  81 #define RQ_CQE_STATUS_GET(val, member)          \
  82                 (((val) >> RQ_CQE_STATUS_##member##_SHIFT) & \
  83                                 RQ_CQE_STATUS_##member##_MASK)
  84
  85 #define RQ_CQE_STATUS_CLEAR(val, member)        \
  86                 ((val) & (~(RQ_CQE_STATUS_##member##_MASK << \
  87                                 RQ_CQE_STATUS_##member##_SHIFT)))
  88
  89 #define HINIC_GET_RX_CSUM_ERR(status)   \
  90                 RQ_CQE_STATUS_GET(status, CSUM_ERR)
  91
  92 #define HINIC_GET_RX_DONE(status)       \
  93                 RQ_CQE_STATUS_GET(status, RXDONE)
  94
  95 #define HINIC_GET_RX_FLUSH(status)      \
  96                 RQ_CQE_STATUS_GET(status, FLUSH)
  97
  98 #define HINIC_GET_RX_BP_EN(status)      \
  99                 RQ_CQE_STATUS_GET(status, BP_EN)
 100
 101 #define HINIC_GET_RX_NUM_LRO(status)    \
 102                 RQ_CQE_STATUS_GET(status, NUM_LRO)
 103
 104 /* RQ_CTRL */
 105 #define RQ_CTRL_BUFDESC_SECT_LEN_SHIFT          0
 106 #define RQ_CTRL_COMPLETE_FORMAT_SHIFT           15
 107 #define RQ_CTRL_COMPLETE_LEN_SHIFT              27
 108 #define RQ_CTRL_LEN_SHIFT                       29
 109
 110 #define RQ_CTRL_BUFDESC_SECT_LEN_MASK           0xFFU
 111 #define RQ_CTRL_COMPLETE_FORMAT_MASK            0x1U
 112 #define RQ_CTRL_COMPLETE_LEN_MASK               0x3U
 113 #define RQ_CTRL_LEN_MASK                        0x3U
 114
 115 #define RQ_CTRL_SET(val, member)                \
 116         (((val) & RQ_CTRL_##member##_MASK) << RQ_CTRL_##member##_SHIFT)
 117
 118 #define RQ_CTRL_GET(val, member)                \
 119         (((val) >> RQ_CTRL_##member##_SHIFT) & RQ_CTRL_##member##_MASK)
 120
 121 #define RQ_CTRL_CLEAR(val, member)              \
 122         ((val) & (~(RQ_CTRL_##member##_MASK << RQ_CTRL_##member##_SHIFT)))
 123
 124 #define RQ_CQE_PKT_NUM_SHIFT                    1
 125 #define RQ_CQE_PKT_FIRST_LEN_SHIFT              19
 126 #define RQ_CQE_PKT_LAST_LEN_SHIFT               6
 127 #define RQ_CQE_SUPER_CQE_EN_SHIFT               0
 128
 129 #define RQ_CQE_PKT_FIRST_LEN_MASK               0x1FFFU
 130 #define RQ_CQE_PKT_LAST_LEN_MASK                0x1FFFU
 131 #define RQ_CQE_PKT_NUM_MASK                     0x1FU
 132 #define RQ_CQE_SUPER_CQE_EN_MASK                0x1
 133
 134 #define RQ_CQE_PKT_NUM_GET(val, member)         \
 135         (((val) >> RQ_CQE_PKT_##member##_SHIFT) & RQ_CQE_PKT_##member##_MASK)
 136
 137 #define HINIC_GET_RQ_CQE_PKT_NUM(pkt_info) RQ_CQE_PKT_NUM_GET(pkt_info, NUM)
 138
 139 #define RQ_CQE_SUPER_CQE_EN_GET(val, member)    \
 140         (((val) >> RQ_CQE_##member##_SHIFT) & RQ_CQE_##member##_MASK)
 141
 142 #define HINIC_GET_SUPER_CQE_EN(pkt_info)        \
 143         RQ_CQE_SUPER_CQE_EN_GET(pkt_info, SUPER_CQE_EN)
 144
 145 #define RQ_CQE_OFFOLAD_TYPE_VLAN_EN_SHIFT               21
 146 #define RQ_CQE_OFFOLAD_TYPE_VLAN_EN_MASK                0x1U
 147
 148 #define RQ_CQE_OFFOLAD_TYPE_PKT_TYPE_SHIFT              0
 149 #define RQ_CQE_OFFOLAD_TYPE_PKT_TYPE_MASK               0xFFFU
 150
 151 #define RQ_CQE_OFFOLAD_TYPE_PKT_UMBCAST_SHIFT           19
 152 #define RQ_CQE_OFFOLAD_TYPE_PKT_UMBCAST_MASK            0x3U
 153
 154 #define RQ_CQE_OFFOLAD_TYPE_RSS_TYPE_SHIFT              24
 155 #define RQ_CQE_OFFOLAD_TYPE_RSS_TYPE_MASK               0xFFU
 156
 157 #define RQ_CQE_OFFOLAD_TYPE_GET(val, member)            (((val) >> \
 158                                 RQ_CQE_OFFOLAD_TYPE_##member##_SHIFT) & \
 159                                 RQ_CQE_OFFOLAD_TYPE_##member##_MASK)
 160
 161 #define HINIC_GET_RX_VLAN_OFFLOAD_EN(offload_type)      \
 162                 RQ_CQE_OFFOLAD_TYPE_GET(offload_type, VLAN_EN)
 163
 164 #define HINIC_GET_RSS_TYPES(offload_type)       \
 165                 RQ_CQE_OFFOLAD_TYPE_GET(offload_type, RSS_TYPE)
 166
 167 #define HINIC_GET_RX_PKT_TYPE(offload_type)     \
 168                 RQ_CQE_OFFOLAD_TYPE_GET(offload_type, PKT_TYPE)
 169
 170 #define HINIC_GET_RX_PKT_UMBCAST(offload_type)  \
 171                 RQ_CQE_OFFOLAD_TYPE_GET(offload_type, PKT_UMBCAST)
 172
 173 #define RQ_CQE_STATUS_CSUM_BYPASS_VAL                   0x80U
 174 #define RQ_CQE_STATUS_CSUM_ERR_IP_MASK                  0x39U
 175 #define RQ_CQE_STATUS_CSUM_ERR_L4_MASK                  0x46U
 176 #define RQ_CQE_STATUS_CSUM_ERR_OTHER                    0x100U
 177
 178 #define HINIC_CSUM_ERR_BYPASSED(csum_err)        \
 179         ((csum_err) == RQ_CQE_STATUS_CSUM_BYPASS_VAL)
 180
 181 #define HINIC_CSUM_ERR_IP(csum_err)      \
 182         ((csum_err) & RQ_CQE_STATUS_CSUM_ERR_IP_MASK)
 183
 184 #define HINIC_CSUM_ERR_L4(csum_err)      \
 185         ((csum_err) & RQ_CQE_STATUS_CSUM_ERR_L4_MASK)
 186
 187 #define HINIC_CSUM_ERR_OTHER(csum_err)   \
 188         ((csum_err) == RQ_CQE_STATUS_CSUM_ERR_OTHER)
 189
 190
 191 void hinic_get_func_rx_buf_size(struct hinic_nic_dev *nic_dev)
 192 {
 193         struct hinic_rxq *rxq;
 194         u16 q_id;
 195         u16 buf_size = 0;
 196
 197         for (q_id = 0; q_id < nic_dev->num_rq; q_id++) {
 198                 rxq = nic_dev->rxqs[q_id];
 199
 200                 if (rxq == NULL)
 201                         continue;
 202
 203                 if (q_id == 0)
 204                         buf_size = rxq->buf_len;
 205
 206                 buf_size = buf_size > rxq->buf_len ? rxq->buf_len : buf_size;
 207         }
 208
 209         nic_dev->hwdev->nic_io->rq_buf_size = buf_size;
 210 }
 211
 212 int hinic_create_rq(struct hinic_hwdev *hwdev, u16 q_id,
 213                         u16 rq_depth, unsigned int socket_id)
 214 {
 215         int err;
 216         struct hinic_nic_io *nic_io = hwdev->nic_io;
 217         struct hinic_qp *qp = &nic_io->qps[q_id];
 218         struct hinic_rq *rq = &qp->rq;
 219
 220         /* in case of hardware still generate interrupt, do not use msix 0 */
 221         rq->msix_entry_idx = 1;
 222         rq->q_id = q_id;
 223         rq->rq_depth = rq_depth;
 224         nic_io->rq_depth = rq_depth;
 225
 226         err = hinic_wq_allocate(hwdev, &nic_io->rq_wq[q_id],
 227                         HINIC_RQ_WQEBB_SHIFT, nic_io->rq_depth, socket_id);
 228         if (err) {
 229                 PMD_DRV_LOG(ERR, "Failed to allocate WQ for RQ");
 230                 return err;
 231         }
 232         rq->wq = &nic_io->rq_wq[q_id];
 233
 234         rq->pi_virt_addr = (volatile u16 *)dma_zalloc_coherent(hwdev,
 235                         HINIC_PAGE_SIZE, &rq->pi_dma_addr, socket_id);
 236         if (!rq->pi_virt_addr) {
 237                 PMD_DRV_LOG(ERR, "Failed to allocate rq pi virt addr");
 238                 err = -ENOMEM;
 239                 goto rq_pi_alloc_err;
 240         }
 241
 242         return HINIC_OK;
 243
 244 rq_pi_alloc_err:
 245         hinic_wq_free(hwdev, &nic_io->rq_wq[q_id]);
 246
 247         return err;
 248 }
 249
 250 void hinic_destroy_rq(struct hinic_hwdev *hwdev, u16 q_id)
 251 {
 252         struct hinic_nic_io *nic_io = hwdev->nic_io;
 253         struct hinic_qp *qp = &nic_io->qps[q_id];
 254         struct hinic_rq *rq = &qp->rq;
 255
 256         if (qp->rq.wq == NULL)
 257                 return;
 258
 259         dma_free_coherent_volatile(hwdev, HINIC_PAGE_SIZE,
 260                                    (volatile void *)rq->pi_virt_addr,
 261                                    rq->pi_dma_addr);
 262         hinic_wq_free(nic_io->hwdev, qp->rq.wq);
 263         qp->rq.wq = NULL;
 264 }
 265
 266 static void
 267 hinic_prepare_rq_wqe(void *wqe, __rte_unused u16 pi, dma_addr_t buf_addr,
 268                         dma_addr_t cqe_dma)
 269 {
 270         struct hinic_rq_wqe *rq_wqe = wqe;
 271         struct hinic_rq_ctrl *ctrl = &rq_wqe->ctrl;
 272         struct hinic_rq_cqe_sect *cqe_sect = &rq_wqe->cqe_sect;
 273         struct hinic_rq_bufdesc *buf_desc = &rq_wqe->buf_desc;
 274         u32 rq_ceq_len = sizeof(struct hinic_rq_cqe);
 275
 276         ctrl->ctrl_fmt =
 277                 RQ_CTRL_SET(SIZE_8BYTES(sizeof(*ctrl)),  LEN) |
 278                 RQ_CTRL_SET(SIZE_8BYTES(sizeof(*cqe_sect)), COMPLETE_LEN) |
 279                 RQ_CTRL_SET(SIZE_8BYTES(sizeof(*buf_desc)), BUFDESC_SECT_LEN) |
 280                 RQ_CTRL_SET(RQ_COMPLETE_SGE, COMPLETE_FORMAT);
 281
 282         hinic_set_sge(&cqe_sect->sge, cqe_dma, rq_ceq_len);
 283
 284         buf_desc->addr_high = upper_32_bits(buf_addr);
 285         buf_desc->addr_low = lower_32_bits(buf_addr);
 286 }
 287
 288 void hinic_rxq_get_stats(struct hinic_rxq *rxq, struct hinic_rxq_stats *stats)
 289 {
 290         if (!rxq || !stats)
 291                 return;
 292
 293         memcpy(stats, &rxq->rxq_stats, sizeof(rxq->rxq_stats));
 294 }
 295
 296 void hinic_rxq_stats_reset(struct hinic_rxq *rxq)
 297 {
 298         struct hinic_rxq_stats *rxq_stats;
 299
 300         if (rxq == NULL)
 301                 return;
 302
 303         rxq_stats = &rxq->rxq_stats;
 304         memset(rxq_stats, 0, sizeof(*rxq_stats));
 305 }
 306
 307 static int hinic_rx_alloc_cqe(struct hinic_rxq *rxq, unsigned int socket_id)
 308 {
 309         size_t cqe_mem_size;
 310
 311         cqe_mem_size = sizeof(struct hinic_rq_cqe) * rxq->q_depth;
 312         rxq->cqe_start_vaddr = dma_zalloc_coherent(rxq->nic_dev->hwdev,
 313                                 cqe_mem_size, &rxq->cqe_start_paddr, socket_id);
 314         if (!rxq->cqe_start_vaddr) {
 315                 PMD_DRV_LOG(ERR, "Allocate cqe dma memory failed");
 316                 return -ENOMEM;
 317         }
 318
 319         rxq->rx_cqe = (struct hinic_rq_cqe *)rxq->cqe_start_vaddr;
 320
 321         return HINIC_OK;
 322 }
 323
 324 static void hinic_rx_free_cqe(struct hinic_rxq *rxq)
 325 {
 326         size_t cqe_mem_size;
 327
 328         cqe_mem_size = sizeof(struct hinic_rq_cqe) * rxq->q_depth;
 329         dma_free_coherent(rxq->nic_dev->hwdev, cqe_mem_size,
 330                           rxq->cqe_start_vaddr, rxq->cqe_start_paddr);
 331         rxq->cqe_start_vaddr = NULL;
 332 }
 333
 334 static int hinic_rx_fill_wqe(struct hinic_rxq *rxq)
 335 {
 336         struct hinic_nic_dev *nic_dev = rxq->nic_dev;
 337         struct hinic_rq_wqe *rq_wqe;
 338         dma_addr_t buf_dma_addr, cqe_dma_addr;
 339         u16 pi = 0;
 340         int i;
 341
 342         buf_dma_addr = 0;
 343         cqe_dma_addr = rxq->cqe_start_paddr;
 344         for (i = 0; i < rxq->q_depth; i++) {
 345                 rq_wqe = hinic_get_rq_wqe(nic_dev->hwdev, rxq->q_id, &pi);
 346                 if (!rq_wqe) {
 347                         PMD_DRV_LOG(ERR, "Get rq wqe failed");
 348                         break;
 349                 }
 350
 351                 hinic_prepare_rq_wqe(rq_wqe, pi, buf_dma_addr, cqe_dma_addr);
 352                 cqe_dma_addr +=  sizeof(struct hinic_rq_cqe);
 353
 354                 hinic_cpu_to_be32(rq_wqe, sizeof(struct hinic_rq_wqe));
 355         }
 356
 357         hinic_return_rq_wqe(nic_dev->hwdev, rxq->q_id, i);
 358
 359         return i;
 360 }
 361
 362 /* alloc cqe and prepare rqe */
 363 int hinic_setup_rx_resources(struct hinic_rxq *rxq)
 364 {
 365         u64 rx_info_sz;
 366         int err, pkts;
 367
 368         rx_info_sz = rxq->q_depth * sizeof(*rxq->rx_info);
 369         rxq->rx_info = rte_zmalloc_socket("rx_info", rx_info_sz,
 370                                 RTE_CACHE_LINE_SIZE, rxq->socket_id);
 371         if (!rxq->rx_info)
 372                 return -ENOMEM;
 373
 374         err = hinic_rx_alloc_cqe(rxq, rxq->socket_id);
 375         if (err) {
 376                 PMD_DRV_LOG(ERR, "Allocate rx cqe failed");
 377                 goto rx_cqe_err;
 378         }
 379
 380         pkts = hinic_rx_fill_wqe(rxq);
 381         if (pkts != rxq->q_depth) {
 382                 PMD_DRV_LOG(ERR, "Fill rx wqe failed");
 383                 err = -ENOMEM;
 384                 goto rx_fill_err;
 385         }
 386
 387         return 0;
 388
 389 rx_fill_err:
 390         hinic_rx_free_cqe(rxq);
 391
 392 rx_cqe_err:
 393         rte_free(rxq->rx_info);
 394         rxq->rx_info = NULL;
 395
 396         return err;
 397 }
 398
 399 void hinic_free_rx_resources(struct hinic_rxq *rxq)
 400 {
 401         if (rxq->rx_info == NULL)
 402                 return;
 403
 404         hinic_rx_free_cqe(rxq);
 405         rte_free(rxq->rx_info);
 406         rxq->rx_info = NULL;
 407 }
 408
 409 void hinic_free_all_rx_resources(struct rte_eth_dev *eth_dev)
 410 {
 411         u16 q_id;
 412         struct hinic_nic_dev *nic_dev =
 413                                 HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
 414
 415         for (q_id = 0; q_id < nic_dev->num_rq; q_id++) {
 416                 eth_dev->data->rx_queues[q_id] = NULL;
 417
 418                 if (nic_dev->rxqs[q_id] == NULL)
 419                         continue;
 420
 421                 hinic_free_all_rx_mbufs(nic_dev->rxqs[q_id]);
 422                 hinic_free_rx_resources(nic_dev->rxqs[q_id]);
 423                 kfree(nic_dev->rxqs[q_id]);
 424                 nic_dev->rxqs[q_id] = NULL;
 425         }
 426 }
 427
 428 void hinic_free_all_rx_mbuf(struct rte_eth_dev *eth_dev)
 429 {
 430         struct hinic_nic_dev *nic_dev =
 431                                 HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
 432         u16 q_id;
 433
 434         for (q_id = 0; q_id < nic_dev->num_rq; q_id++)
 435                 hinic_free_all_rx_mbufs(nic_dev->rxqs[q_id]);
 436 }
 437
 438 static void hinic_recv_jumbo_pkt(struct hinic_rxq *rxq,
 439                                  struct rte_mbuf *head_mbuf,
 440                                  u32 remain_pkt_len)
 441 {
 442         struct hinic_nic_dev *nic_dev = rxq->nic_dev;
 443         struct rte_mbuf *cur_mbuf, *rxm = NULL;
 444         struct hinic_rx_info *rx_info;
 445         u16 sw_ci, rx_buf_len = rxq->buf_len;
 446         u32 pkt_len;
 447
 448         while (remain_pkt_len > 0) {
 449                 sw_ci = hinic_get_rq_local_ci(nic_dev->hwdev, rxq->q_id);
 450                 rx_info = &rxq->rx_info[sw_ci];
 451
 452                 hinic_update_rq_local_ci(nic_dev->hwdev, rxq->q_id, 1);
 453
 454                 pkt_len = remain_pkt_len > rx_buf_len ?
 455                         rx_buf_len : remain_pkt_len;
 456                 remain_pkt_len -= pkt_len;
 457
 458                 cur_mbuf = rx_info->mbuf;
 459                 cur_mbuf->data_len = (u16)pkt_len;
 460                 cur_mbuf->next = NULL;
 461
 462                 head_mbuf->pkt_len += cur_mbuf->data_len;
 463                 head_mbuf->nb_segs++;
 464
 465                 if (!rxm)
 466                         head_mbuf->next = cur_mbuf;
 467                 else
 468                         rxm->next = cur_mbuf;
 469
 470                 rxm = cur_mbuf;
 471         }
 472 }
 473
 474 static void hinic_rss_deinit(struct hinic_nic_dev *nic_dev)
 475 {
 476         u8 prio_tc[HINIC_DCB_UP_MAX] = {0};
 477         (void)hinic_rss_cfg(nic_dev->hwdev, 0,
 478                             nic_dev->rss_tmpl_idx, 0, prio_tc);
 479 }
 480
 481 static int hinic_rss_key_init(struct hinic_nic_dev *nic_dev,
 482                               struct rte_eth_rss_conf *rss_conf)
 483 {
 484         u8 default_rss_key[HINIC_RSS_KEY_SIZE] = {
 485                          0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2,
 486                          0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0,
 487                          0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4,
 488                          0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c,
 489                          0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa};
 490         u8 hashkey[HINIC_RSS_KEY_SIZE] = {0};
 491         u8 tmpl_idx = nic_dev->rss_tmpl_idx;
 492
 493         if (rss_conf->rss_key == NULL)
 494                 memcpy(hashkey, default_rss_key, HINIC_RSS_KEY_SIZE);
 495         else
 496                 memcpy(hashkey, rss_conf->rss_key, rss_conf->rss_key_len);
 497
 498         return hinic_rss_set_template_tbl(nic_dev->hwdev, tmpl_idx, hashkey);
 499 }
 500
 501 static void hinic_fill_rss_type(struct nic_rss_type *rss_type,
 502                                 struct rte_eth_rss_conf *rss_conf)
 503 {
 504         u64 rss_hf = rss_conf->rss_hf;
 505
 506         rss_type->ipv4 = (rss_hf & (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4)) ? 1 : 0;
 507         rss_type->tcp_ipv4 = (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) ? 1 : 0;
 508         rss_type->ipv6 = (rss_hf & (ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6)) ? 1 : 0;
 509         rss_type->ipv6_ext = (rss_hf & ETH_RSS_IPV6_EX) ? 1 : 0;
 510         rss_type->tcp_ipv6 = (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP) ? 1 : 0;
 511         rss_type->tcp_ipv6_ext = (rss_hf & ETH_RSS_IPV6_TCP_EX) ? 1 : 0;
 512         rss_type->udp_ipv4 = (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) ? 1 : 0;
 513         rss_type->udp_ipv6 = (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP) ? 1 : 0;
 514 }
 515
 516 static void hinic_fillout_indir_tbl(struct hinic_nic_dev *nic_dev, u32 *indir)
 517 {
 518         u8 rss_queue_count = nic_dev->num_rss;
 519         int i = 0, j;
 520
 521         if (rss_queue_count == 0) {
 522                 /* delete q_id from indir tbl */
 523                 for (i = 0; i < HINIC_RSS_INDIR_SIZE; i++)
 524                         indir[i] = 0xFF;        /* Invalid value in indir tbl */
 525         } else {
 526                 while (i < HINIC_RSS_INDIR_SIZE)
 527                         for (j = 0; (j < rss_queue_count) &&
 528                              (i < HINIC_RSS_INDIR_SIZE); j++)
 529                                 indir[i++] = nic_dev->rx_queue_list[j];
 530         }
 531 }
 532
 533 static int hinic_rss_init(struct hinic_nic_dev *nic_dev,
 534                           __rte_unused u8 *rq2iq_map,
 535                           struct rte_eth_rss_conf *rss_conf)
 536 {
 537         u32 indir_tbl[HINIC_RSS_INDIR_SIZE] = {0};
 538         struct nic_rss_type rss_type = {0};
 539         u8 prio_tc[HINIC_DCB_UP_MAX] = {0};
 540         u8 tmpl_idx = 0xFF, num_tc = 0;
 541         int err;
 542
 543         tmpl_idx = nic_dev->rss_tmpl_idx;
 544
 545         err = hinic_rss_key_init(nic_dev, rss_conf);
 546         if (err)
 547                 return err;
 548
 549         if (!nic_dev->rss_indir_flag) {
 550                 hinic_fillout_indir_tbl(nic_dev, indir_tbl);
 551                 err = hinic_rss_set_indir_tbl(nic_dev->hwdev, tmpl_idx,
 552                                               indir_tbl);
 553                 if (err)
 554                         return err;
 555         }
 556
 557         hinic_fill_rss_type(&rss_type, rss_conf);
 558         err = hinic_set_rss_type(nic_dev->hwdev, tmpl_idx, rss_type);
 559         if (err)
 560                 return err;
 561
 562         err = hinic_rss_set_hash_engine(nic_dev->hwdev, tmpl_idx,
 563                                         HINIC_RSS_HASH_ENGINE_TYPE_TOEP);
 564         if (err)
 565                 return err;
 566
 567         return hinic_rss_cfg(nic_dev->hwdev, 1, tmpl_idx, num_tc, prio_tc);
 568 }
 569
 570 static void
 571 hinic_add_rq_to_rx_queue_list(struct hinic_nic_dev *nic_dev, u16 queue_id)
 572 {
 573         u8 rss_queue_count = nic_dev->num_rss;
 574
 575         RTE_ASSERT(rss_queue_count <= (RTE_DIM(nic_dev->rx_queue_list) - 1));
 576
 577         nic_dev->rx_queue_list[rss_queue_count] = queue_id;
 578         nic_dev->num_rss++;
 579 }
 580
 581 /**
 582  * hinic_setup_num_qps - determine num_qps from rss_tmpl_id
 583  * @nic_dev: pointer to the private ethernet device
 584  * Return: 0 on Success, error code otherwise.
 585  **/
 586 static int hinic_setup_num_qps(struct hinic_nic_dev *nic_dev)
 587 {
 588         int err, i;
 589
 590         if (!(nic_dev->flags & ETH_MQ_RX_RSS_FLAG)) {
 591                 nic_dev->flags &= ~ETH_MQ_RX_RSS_FLAG;
 592                 nic_dev->num_rss = 0;
 593                 if (nic_dev->num_rq > 1) {
 594                         /* get rss template id */
 595                         err = hinic_rss_template_alloc(nic_dev->hwdev,
 596                                                        &nic_dev->rss_tmpl_idx);
 597                         if (err) {
 598                                 PMD_DRV_LOG(WARNING, "Alloc rss template failed");
 599                                 return err;
 600                         }
 601                         nic_dev->flags |= ETH_MQ_RX_RSS_FLAG;
 602                         for (i = 0; i < nic_dev->num_rq; i++)
 603                                 hinic_add_rq_to_rx_queue_list(nic_dev, i);
 604                 }
 605         }
 606
 607         return 0;
 608 }
 609
 610 static void hinic_destroy_num_qps(struct hinic_nic_dev *nic_dev)
 611 {
 612         if (nic_dev->flags & ETH_MQ_RX_RSS_FLAG) {
 613                 if (hinic_rss_template_free(nic_dev->hwdev,
 614                                             nic_dev->rss_tmpl_idx))
 615                         PMD_DRV_LOG(WARNING, "Free rss template failed");
 616
 617                 nic_dev->flags &= ~ETH_MQ_RX_RSS_FLAG;
 618         }
 619 }
 620
 621 static int hinic_config_mq_rx_rss(struct hinic_nic_dev *nic_dev, bool on)
 622 {
 623         int ret = 0;
 624
 625         if (on) {
 626                 ret = hinic_setup_num_qps(nic_dev);
 627                 if (ret)
 628                         PMD_DRV_LOG(ERR, "Setup num_qps failed");
 629         } else {
 630                 hinic_destroy_num_qps(nic_dev);
 631         }
 632
 633         return ret;
 634 }
 635
 636 int hinic_config_mq_mode(struct rte_eth_dev *dev, bool on)
 637 {
 638         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 639         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 640         int ret = 0;
 641
 642         switch (dev_conf->rxmode.mq_mode) {
 643         case ETH_MQ_RX_RSS:
 644                 ret = hinic_config_mq_rx_rss(nic_dev, on);
 645                 break;
 646         default:
 647                 break;
 648         }
 649
 650         return ret;
 651 }
 652
 653 int hinic_rx_configure(struct rte_eth_dev *dev)
 654 {
 655         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 656         struct rte_eth_rss_conf rss_conf =
 657                 dev->data->dev_conf.rx_adv_conf.rss_conf;
 658         int err;
 659
 660         if (nic_dev->flags & ETH_MQ_RX_RSS_FLAG) {
 661                 if (rss_conf.rss_hf == 0) {
 662                         rss_conf.rss_hf = HINIC_RSS_OFFLOAD_ALL;
 663                 } else if ((rss_conf.rss_hf & HINIC_RSS_OFFLOAD_ALL) == 0) {
 664                         PMD_DRV_LOG(ERR, "Do not support rss offload all");
 665                         goto rss_config_err;
 666                 }
 667
 668                 err = hinic_rss_init(nic_dev, NULL, &rss_conf);
 669                 if (err) {
 670                         PMD_DRV_LOG(ERR, "Init rss failed");
 671                         goto rss_config_err;
 672                 }
 673         }
 674
 675         /* Enable both L3/L4 rx checksum offload */
 676         if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_CHECKSUM)
 677                 nic_dev->rx_csum_en = HINIC_RX_CSUM_OFFLOAD_EN;
 678
 679         err = hinic_set_rx_csum_offload(nic_dev->hwdev,
 680                                         HINIC_RX_CSUM_OFFLOAD_EN);
 681         if (err)
 682                 goto rx_csum_ofl_err;
 683
 684         return 0;
 685
 686 rx_csum_ofl_err:
 687 rss_config_err:
 688         hinic_destroy_num_qps(nic_dev);
 689
 690         return HINIC_ERROR;
 691 }
 692
 693 void hinic_rx_remove_configure(struct rte_eth_dev *dev)
 694 {
 695         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 696
 697         if (nic_dev->flags & ETH_MQ_RX_RSS_FLAG) {
 698                 hinic_rss_deinit(nic_dev);
 699                 hinic_destroy_num_qps(nic_dev);
 700         }
 701 }
 702
 703 void hinic_free_all_rx_mbufs(struct hinic_rxq *rxq)
 704 {
 705         struct hinic_nic_dev *nic_dev = rxq->nic_dev;
 706         struct hinic_rx_info *rx_info;
 707         int free_wqebbs =
 708                 hinic_get_rq_free_wqebbs(nic_dev->hwdev, rxq->q_id) + 1;
 709         volatile struct hinic_rq_cqe *rx_cqe;
 710         u16 ci;
 711
 712         while (free_wqebbs++ < rxq->q_depth) {
 713                 ci = hinic_get_rq_local_ci(nic_dev->hwdev, rxq->q_id);
 714
 715                 rx_cqe = &rxq->rx_cqe[ci];
 716
 717                 /* clear done bit */
 718                 rx_cqe->status = 0;
 719
 720                 rx_info = &rxq->rx_info[ci];
 721                 rte_pktmbuf_free(rx_info->mbuf);
 722                 rx_info->mbuf = NULL;
 723
 724                 hinic_update_rq_local_ci(nic_dev->hwdev, rxq->q_id, 1);
 725         }
 726 }
 727
 728 static inline void hinic_rq_cqe_be_to_cpu32(void *dst_le32,
 729                                             volatile void *src_be32)
 730 {
 731 #if defined(__X86_64_SSE__)
 732         volatile __m128i *wqe_be = (volatile __m128i *)src_be32;
 733         __m128i *wqe_le = (__m128i *)dst_le32;
 734         __m128i shuf_mask =  _mm_set_epi8(12, 13, 14, 15, 8, 9, 10,
 735                                         11, 4, 5, 6, 7, 0, 1, 2, 3);
 736
 737         /* l2nic just use first 128 bits */
 738         wqe_le[0] = _mm_shuffle_epi8(wqe_be[0], shuf_mask);
 739 #elif defined(__ARM64_NEON__)
 740         volatile uint8x16_t *wqe_be = (volatile uint8x16_t *)src_be32;
 741         uint8x16_t *wqe_le = (uint8x16_t *)dst_le32;
 742         const uint8x16_t shuf_mask = {3, 2, 1, 0, 7, 6, 5, 4, 11, 10,
 743                                         9, 8, 15, 14, 13, 12};
 744
 745         /* l2nic just use first 128 bits */
 746         wqe_le[0] = vqtbl1q_u8(wqe_be[0], shuf_mask);
 747 #else
 748         u32 i;
 749         volatile u32 *wqe_be = (volatile u32 *)src_be32;
 750         u32 *wqe_le = (u32 *)dst_le32;
 751
 752 #define HINIC_L2NIC_RQ_CQE_USED         4 /* 4Bytes unit */
 753
 754         for (i = 0; i < HINIC_L2NIC_RQ_CQE_USED; i++) {
 755                 *wqe_le = rte_be_to_cpu_32(*wqe_be);
 756                 wqe_be++;
 757                 wqe_le++;
 758         }
 759 #endif
 760 }
 761
 762 static inline uint64_t hinic_rx_rss_hash(uint32_t offload_type,
 763                                          uint32_t cqe_hass_val,
 764                                          uint32_t *rss_hash)
 765 {
 766         uint32_t rss_type;
 767
 768         rss_type = HINIC_GET_RSS_TYPES(offload_type);
 769         if (likely(rss_type != 0)) {
 770                 *rss_hash = cqe_hass_val;
 771                 return PKT_RX_RSS_HASH;
 772         }
 773
 774         return 0;
 775 }
 776
 777 static inline uint64_t hinic_rx_csum(uint32_t status, struct hinic_rxq *rxq)
 778 {
 779         uint32_t checksum_err;
 780         uint64_t flags;
 781         struct hinic_nic_dev *nic_dev = rxq->nic_dev;
 782
 783         if (unlikely(!(nic_dev->rx_csum_en & HINIC_RX_CSUM_OFFLOAD_EN)))
 784                 return PKT_RX_IP_CKSUM_UNKNOWN;
 785
 786         /* most case checksum is ok */
 787         checksum_err = HINIC_GET_RX_CSUM_ERR(status);
 788         if (likely(checksum_err == 0))
 789                 return (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD);
 790
 791         /* If BYPASS bit set, all other status indications should be ignored */
 792         if (unlikely(HINIC_CSUM_ERR_BYPASSED(checksum_err)))
 793                 return PKT_RX_IP_CKSUM_UNKNOWN;
 794
 795         flags = 0;
 796
 797         /* IP checksum error */
 798         if (HINIC_CSUM_ERR_IP(checksum_err))
 799                 flags |= PKT_RX_IP_CKSUM_BAD;
 800         else
 801                 flags |= PKT_RX_IP_CKSUM_GOOD;
 802
 803         /* L4 checksum error */
 804         if (HINIC_CSUM_ERR_L4(checksum_err))
 805                 flags |= PKT_RX_L4_CKSUM_BAD;
 806         else
 807                 flags |= PKT_RX_L4_CKSUM_GOOD;
 808
 809         if (unlikely(HINIC_CSUM_ERR_OTHER(checksum_err)))
 810                 flags = PKT_RX_L4_CKSUM_NONE;
 811
 812         rxq->rxq_stats.errors++;
 813
 814         return flags;
 815 }
 816
 817 static inline uint64_t hinic_rx_vlan(uint32_t offload_type, uint32_t vlan_len,
 818                                      uint16_t *vlan_tci)
 819 {
 820         uint16_t vlan_tag;
 821
 822         vlan_tag = HINIC_GET_RX_VLAN_TAG(vlan_len);
 823         if (!HINIC_GET_RX_VLAN_OFFLOAD_EN(offload_type) || 0 == vlan_tag) {
 824                 *vlan_tci = 0;
 825                 return 0;
 826         }
 827
 828         *vlan_tci = vlan_tag;
 829
 830         return PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED;
 831 }
 832
 833 static inline u32 hinic_rx_alloc_mbuf_bulk(struct hinic_rxq *rxq,
 834                                            struct rte_mbuf **mbufs,
 835                                            u32 exp_mbuf_cnt)
 836 {
 837         int rc;
 838         u32 avail_cnt;
 839
 840         rc = rte_pktmbuf_alloc_bulk(rxq->mb_pool, mbufs, exp_mbuf_cnt);
 841         if (likely(rc == HINIC_OK)) {
 842                 avail_cnt = exp_mbuf_cnt;
 843         } else {
 844                 avail_cnt = 0;
 845                 rxq->rxq_stats.rx_nombuf += exp_mbuf_cnt;
 846         }
 847
 848         return avail_cnt;
 849 }
 850
 851 static struct rte_mbuf *hinic_rx_alloc_mbuf(struct hinic_rxq *rxq,
 852                                         dma_addr_t *dma_addr)
 853 {
 854         struct rte_mbuf *mbuf = NULL;
 855         int rc;
 856
 857         rc = rte_pktmbuf_alloc_bulk(rxq->mb_pool, &mbuf, 1);
 858         if (unlikely(rc != HINIC_OK))
 859                 return NULL;
 860
 861         *dma_addr = rte_mbuf_data_iova_default(mbuf);
 862
 863         return mbuf;
 864 }
 865
 866 static inline void hinic_rearm_rxq_mbuf(struct hinic_rxq *rxq)
 867 {
 868         u16 pi;
 869         u32 i, free_wqebbs, rearm_wqebbs, exp_wqebbs;
 870         dma_addr_t dma_addr;
 871         struct hinic_rq_wqe *rq_wqe;
 872         struct rte_mbuf **rearm_mbufs;
 873
 874         /* check free wqebb fo rearm */
 875         free_wqebbs = HINIC_GET_RQ_FREE_WQEBBS(rxq);
 876         if (unlikely(free_wqebbs < rxq->rx_free_thresh))
 877                 return;
 878
 879         /* get rearm mbuf array */
 880         pi = HINIC_GET_RQ_LOCAL_PI(rxq);
 881         rearm_mbufs = (struct rte_mbuf **)(&rxq->rx_info[pi]);
 882
 883         /* check rxq free wqebbs turn around */
 884         exp_wqebbs = rxq->q_depth - pi;
 885         if (free_wqebbs < exp_wqebbs)
 886                 exp_wqebbs = free_wqebbs;
 887
 888         /* alloc mbuf in bulk */
 889         rearm_wqebbs = hinic_rx_alloc_mbuf_bulk(rxq, rearm_mbufs, exp_wqebbs);
 890         if (unlikely(rearm_wqebbs == 0))
 891                 return;
 892
 893         /* rearm rx mbuf */
 894         rq_wqe = WQ_WQE_ADDR(rxq->wq, (u32)pi);
 895         for (i = 0; i < rearm_wqebbs; i++) {
 896                 dma_addr = rte_mbuf_data_iova_default(rearm_mbufs[i]);
 897                 rq_wqe->buf_desc.addr_high =
 898                                         cpu_to_be32(upper_32_bits(dma_addr));
 899                 rq_wqe->buf_desc.addr_low =
 900                                         cpu_to_be32(lower_32_bits(dma_addr));
 901                 rq_wqe++;
 902         }
 903         rxq->wq->prod_idx += rearm_wqebbs;
 904         rxq->wq->delta -= rearm_wqebbs;
 905
 906         /* update rq hw_pi */
 907         rte_wmb();
 908         HINIC_UPDATE_RQ_HW_PI(rxq, pi + rearm_wqebbs);
 909 }
 910
 911 void hinic_rx_alloc_pkts(struct hinic_rxq *rxq)
 912 {
 913         struct hinic_nic_dev *nic_dev = rxq->nic_dev;
 914         struct hinic_rq_wqe *rq_wqe;
 915         struct hinic_rx_info *rx_info;
 916         struct rte_mbuf *mb;
 917         dma_addr_t dma_addr;
 918         u16 pi = 0;
 919         int i, free_wqebbs;
 920
 921         free_wqebbs = HINIC_GET_RQ_FREE_WQEBBS(rxq);
 922         for (i = 0; i < free_wqebbs; i++) {
 923                 mb = hinic_rx_alloc_mbuf(rxq, &dma_addr);
 924                 if (unlikely(!mb)) {
 925                         rxq->rxq_stats.rx_nombuf++;
 926                         break;
 927                 }
 928
 929                 rq_wqe = hinic_get_rq_wqe(nic_dev->hwdev, rxq->q_id, &pi);
 930                 if (unlikely(!rq_wqe)) {
 931                         rte_pktmbuf_free(mb);
 932                         break;
 933                 }
 934
 935                 /* fill buffer address only */
 936                 rq_wqe->buf_desc.addr_high =
 937                                 cpu_to_be32(upper_32_bits(dma_addr));
 938                 rq_wqe->buf_desc.addr_low =
 939                                 cpu_to_be32(lower_32_bits(dma_addr));
 940
 941                 rx_info = &rxq->rx_info[pi];
 942                 rx_info->mbuf = mb;
 943         }
 944
 945         if (likely(i > 0)) {
 946                 rte_wmb();
 947                 HINIC_UPDATE_RQ_HW_PI(rxq, pi + 1);
 948         }
 949 }
 950
 951 u16 hinic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, u16 nb_pkts)
 952 {
 953         struct rte_mbuf *rxm;
 954         struct hinic_rxq *rxq = rx_queue;
 955         struct hinic_rx_info *rx_info;
 956         volatile struct hinic_rq_cqe *rx_cqe;
 957         u16 rx_buf_len, pkts = 0;
 958         u16 sw_ci, ci_mask, wqebb_cnt = 0;
 959         u32 pkt_len, status, vlan_len;
 960         u64 rx_bytes = 0;
 961         struct hinic_rq_cqe cqe;
 962         u32 offload_type, rss_hash;
 963
 964         rx_buf_len = rxq->buf_len;
 965
 966         /* 1. get polling start ci */
 967         ci_mask = HINIC_GET_RQ_WQE_MASK(rxq);
 968         sw_ci = HINIC_GET_RQ_LOCAL_CI(rxq);
 969
 970         while (pkts < nb_pkts) {
 971                  /* 2. current ci is done */
 972                 rx_cqe = &rxq->rx_cqe[sw_ci];
 973                 status = __atomic_load_n(&rx_cqe->status, __ATOMIC_ACQUIRE);
 974                 if (!HINIC_GET_RX_DONE_BE(status))
 975                         break;
 976
 977                 /* convert cqe and get packet length */
 978                 hinic_rq_cqe_be_to_cpu32(&cqe, (volatile void *)rx_cqe);
 979                 vlan_len = cqe.vlan_len;
 980
 981                 rx_info = &rxq->rx_info[sw_ci];
 982                 rxm = rx_info->mbuf;
 983
 984                 /* 3. next ci point and prefetch */
 985                 sw_ci++;
 986                 sw_ci &= ci_mask;
 987
 988                 /* prefetch next mbuf first 64B */
 989                 rte_prefetch0(rxq->rx_info[sw_ci].mbuf);
 990
 991                 /* 4. jumbo frame process */
 992                 pkt_len = HINIC_GET_RX_PKT_LEN(vlan_len);
 993                 if (likely(pkt_len <= rx_buf_len)) {
 994                         rxm->data_len = pkt_len;
 995                         rxm->pkt_len = pkt_len;
 996                         wqebb_cnt++;
 997                 } else {
 998                         rxm->data_len = rx_buf_len;
 999                         rxm->pkt_len = rx_buf_len;
1000
1001                         /* if receive jumbo, updating ci will be done by
1002                          * hinic_recv_jumbo_pkt function.
1003                          */
1004                         HINIC_UPDATE_RQ_LOCAL_CI(rxq, wqebb_cnt + 1);
1005                         wqebb_cnt = 0;
1006                         hinic_recv_jumbo_pkt(rxq, rxm, pkt_len - rx_buf_len);
1007                         sw_ci = HINIC_GET_RQ_LOCAL_CI(rxq);
1008                 }
1009
1010                 /* 5. vlan/checksum/rss/pkt_type/gro offload */
1011                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1012                 rxm->port = rxq->port_id;
1013                 offload_type = cqe.offload_type;
1014
1015                 /* vlan offload */
1016                 rxm->ol_flags |= hinic_rx_vlan(offload_type, vlan_len,
1017                                                &rxm->vlan_tci);
1018
1019                 /* checksum offload */
1020                 rxm->ol_flags |= hinic_rx_csum(cqe.status, rxq);
1021
1022                 /* rss hash offload */
1023                 rss_hash = cqe.rss_hash;
1024                 rxm->ol_flags |= hinic_rx_rss_hash(offload_type, rss_hash,
1025                                                    &rxm->hash.rss);
1026
1027                 /* 6. clear done bit */
1028                 rx_cqe->status = 0;
1029
1030                 rx_bytes += pkt_len;
1031                 rx_pkts[pkts++] = rxm;
1032         }
1033
1034         if (pkts) {
1035                 /* 7. update ci */
1036                 HINIC_UPDATE_RQ_LOCAL_CI(rxq, wqebb_cnt);
1037
1038                 /* do packet stats */
1039                 rxq->rxq_stats.packets += pkts;
1040                 rxq->rxq_stats.bytes += rx_bytes;
1041         }
1042         rxq->rxq_stats.burst_pkts = pkts;
1043
1044         /* 8. rearm mbuf to rxq */
1045         hinic_rearm_rxq_mbuf(rxq);
1046
1047         return pkts;
1048 }