net/bnxt: fix using RSS config struct
[dpdk.git] / drivers / net / hinic / hinic_pmd_rx.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2017 Huawei Technologies Co., Ltd
3  */
4
5 #include <rte_ether.h>
6 #include <rte_mbuf.h>
7 #ifdef __ARM64_NEON__
8 #include <arm_neon.h>
9 #endif
10
11 #include "base/hinic_compat.h"
12 #include "base/hinic_pmd_hwdev.h"
13 #include "base/hinic_pmd_wq.h"
14 #include "base/hinic_pmd_niccfg.h"
15 #include "base/hinic_pmd_nicio.h"
16 #include "hinic_pmd_ethdev.h"
17 #include "hinic_pmd_rx.h"
18
19 /* rxq wq operations */
20 #define HINIC_GET_RQ_WQE_MASK(rxq)      \
21         ((rxq)->wq->mask)
22
23 #define HINIC_GET_RQ_LOCAL_CI(rxq)      \
24         (((rxq)->wq->cons_idx) & HINIC_GET_RQ_WQE_MASK(rxq))
25
26 #define HINIC_GET_RQ_LOCAL_PI(rxq)      \
27         (((rxq)->wq->prod_idx) & HINIC_GET_RQ_WQE_MASK(rxq))
28
29 #define HINIC_UPDATE_RQ_LOCAL_CI(rxq, wqebb_cnt)        \
30         do {                                            \
31                 (rxq)->wq->cons_idx += (wqebb_cnt);     \
32                 (rxq)->wq->delta += (wqebb_cnt);        \
33         } while (0)
34
35 #define HINIC_UPDATE_RQ_HW_PI(rxq, pi)  \
36         (*((rxq)->pi_virt_addr) =       \
37                 cpu_to_be16((pi) & HINIC_GET_RQ_WQE_MASK(rxq)))
38
39 #define HINIC_GET_RQ_FREE_WQEBBS(rxq)   ((rxq)->wq->delta - 1)
40
41 /* rxq cqe done and status bit */
42 #define HINIC_GET_RX_DONE_BE(status)    \
43         ((status) & 0x80U)
44
45 #define HINIC_RX_CSUM_OFFLOAD_EN        0xFFF
46
47 #define RQ_CQE_SGE_VLAN_SHIFT                   0
48 #define RQ_CQE_SGE_LEN_SHIFT                    16
49
50 #define RQ_CQE_SGE_VLAN_MASK                    0xFFFFU
51 #define RQ_CQE_SGE_LEN_MASK                     0xFFFFU
52
53 #define RQ_CQE_SGE_GET(val, member)             \
54         (((val) >> RQ_CQE_SGE_##member##_SHIFT) & RQ_CQE_SGE_##member##_MASK)
55
56 #define HINIC_GET_RX_VLAN_TAG(vlan_len) \
57                 RQ_CQE_SGE_GET(vlan_len, VLAN)
58
59 #define HINIC_GET_RX_PKT_LEN(vlan_len)  \
60                 RQ_CQE_SGE_GET(vlan_len, LEN)
61
62 #define RQ_CQE_STATUS_CSUM_ERR_SHIFT            0
63 #define RQ_CQE_STATUS_NUM_LRO_SHIFT             16
64 #define RQ_CQE_STATUS_LRO_PUSH_SHIFT            25
65 #define RQ_CQE_STATUS_LRO_ENTER_SHIFT           26
66 #define RQ_CQE_STATUS_LRO_INTR_SHIFT            27
67
68 #define RQ_CQE_STATUS_BP_EN_SHIFT               30
69 #define RQ_CQE_STATUS_RXDONE_SHIFT              31
70 #define RQ_CQE_STATUS_FLUSH_SHIFT               28
71
72 #define RQ_CQE_STATUS_CSUM_ERR_MASK             0xFFFFU
73 #define RQ_CQE_STATUS_NUM_LRO_MASK              0xFFU
74 #define RQ_CQE_STATUS_LRO_PUSH_MASK             0X1U
75 #define RQ_CQE_STATUS_LRO_ENTER_MASK            0X1U
76 #define RQ_CQE_STATUS_LRO_INTR_MASK             0X1U
77 #define RQ_CQE_STATUS_BP_EN_MASK                0X1U
78 #define RQ_CQE_STATUS_RXDONE_MASK               0x1U
79 #define RQ_CQE_STATUS_FLUSH_MASK                0x1U
80
81 #define RQ_CQE_STATUS_GET(val, member)          \
82                 (((val) >> RQ_CQE_STATUS_##member##_SHIFT) & \
83                                 RQ_CQE_STATUS_##member##_MASK)
84
85 #define RQ_CQE_STATUS_CLEAR(val, member)        \
86                 ((val) & (~(RQ_CQE_STATUS_##member##_MASK << \
87                                 RQ_CQE_STATUS_##member##_SHIFT)))
88
89 #define HINIC_GET_RX_CSUM_ERR(status)   \
90                 RQ_CQE_STATUS_GET(status, CSUM_ERR)
91
92 #define HINIC_GET_RX_DONE(status)       \
93                 RQ_CQE_STATUS_GET(status, RXDONE)
94
95 #define HINIC_GET_RX_FLUSH(status)      \
96                 RQ_CQE_STATUS_GET(status, FLUSH)
97
98 #define HINIC_GET_RX_BP_EN(status)      \
99                 RQ_CQE_STATUS_GET(status, BP_EN)
100
101 #define HINIC_GET_RX_NUM_LRO(status)    \
102                 RQ_CQE_STATUS_GET(status, NUM_LRO)
103
104 /* RQ_CTRL */
105 #define RQ_CTRL_BUFDESC_SECT_LEN_SHIFT          0
106 #define RQ_CTRL_COMPLETE_FORMAT_SHIFT           15
107 #define RQ_CTRL_COMPLETE_LEN_SHIFT              27
108 #define RQ_CTRL_LEN_SHIFT                       29
109
110 #define RQ_CTRL_BUFDESC_SECT_LEN_MASK           0xFFU
111 #define RQ_CTRL_COMPLETE_FORMAT_MASK            0x1U
112 #define RQ_CTRL_COMPLETE_LEN_MASK               0x3U
113 #define RQ_CTRL_LEN_MASK                        0x3U
114
115 #define RQ_CTRL_SET(val, member)                \
116         (((val) & RQ_CTRL_##member##_MASK) << RQ_CTRL_##member##_SHIFT)
117
118 #define RQ_CTRL_GET(val, member)                \
119         (((val) >> RQ_CTRL_##member##_SHIFT) & RQ_CTRL_##member##_MASK)
120
121 #define RQ_CTRL_CLEAR(val, member)              \
122         ((val) & (~(RQ_CTRL_##member##_MASK << RQ_CTRL_##member##_SHIFT)))
123
124 #define RQ_CQE_PKT_NUM_SHIFT                    1
125 #define RQ_CQE_PKT_FIRST_LEN_SHIFT              19
126 #define RQ_CQE_PKT_LAST_LEN_SHIFT               6
127 #define RQ_CQE_SUPER_CQE_EN_SHIFT               0
128
129 #define RQ_CQE_PKT_FIRST_LEN_MASK               0x1FFFU
130 #define RQ_CQE_PKT_LAST_LEN_MASK                0x1FFFU
131 #define RQ_CQE_PKT_NUM_MASK                     0x1FU
132 #define RQ_CQE_SUPER_CQE_EN_MASK                0x1
133
134 #define RQ_CQE_PKT_NUM_GET(val, member)         \
135         (((val) >> RQ_CQE_PKT_##member##_SHIFT) & RQ_CQE_PKT_##member##_MASK)
136
137 #define HINIC_GET_RQ_CQE_PKT_NUM(pkt_info) RQ_CQE_PKT_NUM_GET(pkt_info, NUM)
138
139 #define RQ_CQE_SUPER_CQE_EN_GET(val, member)    \
140         (((val) >> RQ_CQE_##member##_SHIFT) & RQ_CQE_##member##_MASK)
141
142 #define HINIC_GET_SUPER_CQE_EN(pkt_info)        \
143         RQ_CQE_SUPER_CQE_EN_GET(pkt_info, SUPER_CQE_EN)
144
145 #define RQ_CQE_OFFOLAD_TYPE_VLAN_EN_SHIFT               21
146 #define RQ_CQE_OFFOLAD_TYPE_VLAN_EN_MASK                0x1U
147
148 #define RQ_CQE_OFFOLAD_TYPE_PKT_TYPE_SHIFT              0
149 #define RQ_CQE_OFFOLAD_TYPE_PKT_TYPE_MASK               0xFFFU
150
151 #define RQ_CQE_OFFOLAD_TYPE_PKT_UMBCAST_SHIFT           19
152 #define RQ_CQE_OFFOLAD_TYPE_PKT_UMBCAST_MASK            0x3U
153
154 #define RQ_CQE_OFFOLAD_TYPE_RSS_TYPE_SHIFT              24
155 #define RQ_CQE_OFFOLAD_TYPE_RSS_TYPE_MASK               0xFFU
156
157 #define RQ_CQE_OFFOLAD_TYPE_GET(val, member)            (((val) >> \
158                                 RQ_CQE_OFFOLAD_TYPE_##member##_SHIFT) & \
159                                 RQ_CQE_OFFOLAD_TYPE_##member##_MASK)
160
161 #define HINIC_GET_RX_VLAN_OFFLOAD_EN(offload_type)      \
162                 RQ_CQE_OFFOLAD_TYPE_GET(offload_type, VLAN_EN)
163
164 #define HINIC_GET_RSS_TYPES(offload_type)       \
165                 RQ_CQE_OFFOLAD_TYPE_GET(offload_type, RSS_TYPE)
166
167 #define HINIC_GET_RX_PKT_TYPE(offload_type)     \
168                 RQ_CQE_OFFOLAD_TYPE_GET(offload_type, PKT_TYPE)
169
170 #define HINIC_GET_RX_PKT_UMBCAST(offload_type)  \
171                 RQ_CQE_OFFOLAD_TYPE_GET(offload_type, PKT_UMBCAST)
172
173 #define RQ_CQE_STATUS_CSUM_BYPASS_VAL                   0x80U
174 #define RQ_CQE_STATUS_CSUM_ERR_IP_MASK                  0x39U
175 #define RQ_CQE_STATUS_CSUM_ERR_L4_MASK                  0x46U
176 #define RQ_CQE_STATUS_CSUM_ERR_OTHER                    0x100U
177
178 #define HINIC_CSUM_ERR_BYPASSED(csum_err)        \
179         ((csum_err) == RQ_CQE_STATUS_CSUM_BYPASS_VAL)
180
181 #define HINIC_CSUM_ERR_IP(csum_err)      \
182         ((csum_err) & RQ_CQE_STATUS_CSUM_ERR_IP_MASK)
183
184 #define HINIC_CSUM_ERR_L4(csum_err)      \
185         ((csum_err) & RQ_CQE_STATUS_CSUM_ERR_L4_MASK)
186
187 #define HINIC_CSUM_ERR_OTHER(csum_err)   \
188         ((csum_err) == RQ_CQE_STATUS_CSUM_ERR_OTHER)
189
190
191 void hinic_get_func_rx_buf_size(struct hinic_nic_dev *nic_dev)
192 {
193         struct hinic_rxq *rxq;
194         u16 q_id;
195         u16 buf_size = 0;
196
197         for (q_id = 0; q_id < nic_dev->num_rq; q_id++) {
198                 rxq = nic_dev->rxqs[q_id];
199
200                 if (rxq == NULL)
201                         continue;
202
203                 if (q_id == 0)
204                         buf_size = rxq->buf_len;
205
206                 buf_size = buf_size > rxq->buf_len ? rxq->buf_len : buf_size;
207         }
208
209         nic_dev->hwdev->nic_io->rq_buf_size = buf_size;
210 }
211
212 int hinic_create_rq(struct hinic_hwdev *hwdev, u16 q_id,
213                         u16 rq_depth, unsigned int socket_id)
214 {
215         int err;
216         struct hinic_nic_io *nic_io = hwdev->nic_io;
217         struct hinic_qp *qp = &nic_io->qps[q_id];
218         struct hinic_rq *rq = &qp->rq;
219
220         /* in case of hardware still generate interrupt, do not use msix 0 */
221         rq->msix_entry_idx = 1;
222         rq->q_id = q_id;
223         rq->rq_depth = rq_depth;
224         nic_io->rq_depth = rq_depth;
225
226         err = hinic_wq_allocate(hwdev, &nic_io->rq_wq[q_id],
227                         HINIC_RQ_WQEBB_SHIFT, nic_io->rq_depth, socket_id);
228         if (err) {
229                 PMD_DRV_LOG(ERR, "Failed to allocate WQ for RQ");
230                 return err;
231         }
232         rq->wq = &nic_io->rq_wq[q_id];
233
234         rq->pi_virt_addr = (volatile u16 *)dma_zalloc_coherent(hwdev,
235                         HINIC_PAGE_SIZE, &rq->pi_dma_addr, socket_id);
236         if (!rq->pi_virt_addr) {
237                 PMD_DRV_LOG(ERR, "Failed to allocate rq pi virt addr");
238                 err = -ENOMEM;
239                 goto rq_pi_alloc_err;
240         }
241
242         return HINIC_OK;
243
244 rq_pi_alloc_err:
245         hinic_wq_free(hwdev, &nic_io->rq_wq[q_id]);
246
247         return err;
248 }
249
250 void hinic_destroy_rq(struct hinic_hwdev *hwdev, u16 q_id)
251 {
252         struct hinic_nic_io *nic_io = hwdev->nic_io;
253         struct hinic_qp *qp = &nic_io->qps[q_id];
254         struct hinic_rq *rq = &qp->rq;
255
256         if (qp->rq.wq == NULL)
257                 return;
258
259         dma_free_coherent_volatile(hwdev, HINIC_PAGE_SIZE,
260                                    (volatile void *)rq->pi_virt_addr,
261                                    rq->pi_dma_addr);
262         hinic_wq_free(nic_io->hwdev, qp->rq.wq);
263         qp->rq.wq = NULL;
264 }
265
266 static void
267 hinic_prepare_rq_wqe(void *wqe, __rte_unused u16 pi, dma_addr_t buf_addr,
268                         dma_addr_t cqe_dma)
269 {
270         struct hinic_rq_wqe *rq_wqe = wqe;
271         struct hinic_rq_ctrl *ctrl = &rq_wqe->ctrl;
272         struct hinic_rq_cqe_sect *cqe_sect = &rq_wqe->cqe_sect;
273         struct hinic_rq_bufdesc *buf_desc = &rq_wqe->buf_desc;
274         u32 rq_ceq_len = sizeof(struct hinic_rq_cqe);
275
276         ctrl->ctrl_fmt =
277                 RQ_CTRL_SET(SIZE_8BYTES(sizeof(*ctrl)),  LEN) |
278                 RQ_CTRL_SET(SIZE_8BYTES(sizeof(*cqe_sect)), COMPLETE_LEN) |
279                 RQ_CTRL_SET(SIZE_8BYTES(sizeof(*buf_desc)), BUFDESC_SECT_LEN) |
280                 RQ_CTRL_SET(RQ_COMPLETE_SGE, COMPLETE_FORMAT);
281
282         hinic_set_sge(&cqe_sect->sge, cqe_dma, rq_ceq_len);
283
284         buf_desc->addr_high = upper_32_bits(buf_addr);
285         buf_desc->addr_low = lower_32_bits(buf_addr);
286 }
287
288 void hinic_rxq_get_stats(struct hinic_rxq *rxq, struct hinic_rxq_stats *stats)
289 {
290         if (!rxq || !stats)
291                 return;
292
293         memcpy(stats, &rxq->rxq_stats, sizeof(rxq->rxq_stats));
294 }
295
296 void hinic_rxq_stats_reset(struct hinic_rxq *rxq)
297 {
298         struct hinic_rxq_stats *rxq_stats;
299
300         if (rxq == NULL)
301                 return;
302
303         rxq_stats = &rxq->rxq_stats;
304         memset(rxq_stats, 0, sizeof(*rxq_stats));
305 }
306
307 static int hinic_rx_alloc_cqe(struct hinic_rxq *rxq, unsigned int socket_id)
308 {
309         size_t cqe_mem_size;
310
311         cqe_mem_size = sizeof(struct hinic_rq_cqe) * rxq->q_depth;
312         rxq->cqe_start_vaddr = dma_zalloc_coherent(rxq->nic_dev->hwdev,
313                                 cqe_mem_size, &rxq->cqe_start_paddr, socket_id);
314         if (!rxq->cqe_start_vaddr) {
315                 PMD_DRV_LOG(ERR, "Allocate cqe dma memory failed");
316                 return -ENOMEM;
317         }
318
319         rxq->rx_cqe = (struct hinic_rq_cqe *)rxq->cqe_start_vaddr;
320
321         return HINIC_OK;
322 }
323
324 static void hinic_rx_free_cqe(struct hinic_rxq *rxq)
325 {
326         size_t cqe_mem_size;
327
328         cqe_mem_size = sizeof(struct hinic_rq_cqe) * rxq->q_depth;
329         dma_free_coherent(rxq->nic_dev->hwdev, cqe_mem_size,
330                           rxq->cqe_start_vaddr, rxq->cqe_start_paddr);
331         rxq->cqe_start_vaddr = NULL;
332 }
333
334 static int hinic_rx_fill_wqe(struct hinic_rxq *rxq)
335 {
336         struct hinic_nic_dev *nic_dev = rxq->nic_dev;
337         struct hinic_rq_wqe *rq_wqe;
338         dma_addr_t buf_dma_addr, cqe_dma_addr;
339         u16 pi = 0;
340         int i;
341
342         buf_dma_addr = 0;
343         cqe_dma_addr = rxq->cqe_start_paddr;
344         for (i = 0; i < rxq->q_depth; i++) {
345                 rq_wqe = hinic_get_rq_wqe(nic_dev->hwdev, rxq->q_id, &pi);
346                 if (!rq_wqe) {
347                         PMD_DRV_LOG(ERR, "Get rq wqe failed");
348                         break;
349                 }
350
351                 hinic_prepare_rq_wqe(rq_wqe, pi, buf_dma_addr, cqe_dma_addr);
352                 cqe_dma_addr +=  sizeof(struct hinic_rq_cqe);
353
354                 hinic_cpu_to_be32(rq_wqe, sizeof(struct hinic_rq_wqe));
355         }
356
357         hinic_return_rq_wqe(nic_dev->hwdev, rxq->q_id, i);
358
359         return i;
360 }
361
362 /* alloc cqe and prepare rqe */
363 int hinic_setup_rx_resources(struct hinic_rxq *rxq)
364 {
365         u64 rx_info_sz;
366         int err, pkts;
367
368         rx_info_sz = rxq->q_depth * sizeof(*rxq->rx_info);
369         rxq->rx_info = rte_zmalloc_socket("rx_info", rx_info_sz,
370                                 RTE_CACHE_LINE_SIZE, rxq->socket_id);
371         if (!rxq->rx_info)
372                 return -ENOMEM;
373
374         err = hinic_rx_alloc_cqe(rxq, rxq->socket_id);
375         if (err) {
376                 PMD_DRV_LOG(ERR, "Allocate rx cqe failed");
377                 goto rx_cqe_err;
378         }
379
380         pkts = hinic_rx_fill_wqe(rxq);
381         if (pkts != rxq->q_depth) {
382                 PMD_DRV_LOG(ERR, "Fill rx wqe failed");
383                 err = -ENOMEM;
384                 goto rx_fill_err;
385         }
386
387         return 0;
388
389 rx_fill_err:
390         hinic_rx_free_cqe(rxq);
391
392 rx_cqe_err:
393         rte_free(rxq->rx_info);
394         rxq->rx_info = NULL;
395
396         return err;
397 }
398
399 void hinic_free_rx_resources(struct hinic_rxq *rxq)
400 {
401         if (rxq->rx_info == NULL)
402                 return;
403
404         hinic_rx_free_cqe(rxq);
405         rte_free(rxq->rx_info);
406         rxq->rx_info = NULL;
407 }
408
409 void hinic_free_all_rx_resources(struct rte_eth_dev *eth_dev)
410 {
411         u16 q_id;
412         struct hinic_nic_dev *nic_dev =
413                                 HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
414
415         for (q_id = 0; q_id < nic_dev->num_rq; q_id++) {
416                 if (eth_dev->data->rx_queues != NULL)
417                         eth_dev->data->rx_queues[q_id] = NULL;
418
419                 if (nic_dev->rxqs[q_id] == NULL)
420                         continue;
421
422                 hinic_free_all_rx_mbufs(nic_dev->rxqs[q_id]);
423                 hinic_free_rx_resources(nic_dev->rxqs[q_id]);
424                 kfree(nic_dev->rxqs[q_id]);
425                 nic_dev->rxqs[q_id] = NULL;
426         }
427 }
428
429 void hinic_free_all_rx_mbuf(struct rte_eth_dev *eth_dev)
430 {
431         struct hinic_nic_dev *nic_dev =
432                                 HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
433         u16 q_id;
434
435         for (q_id = 0; q_id < nic_dev->num_rq; q_id++)
436                 hinic_free_all_rx_mbufs(nic_dev->rxqs[q_id]);
437 }
438
439 static void hinic_recv_jumbo_pkt(struct hinic_rxq *rxq,
440                                  struct rte_mbuf *head_mbuf,
441                                  u32 remain_pkt_len)
442 {
443         struct hinic_nic_dev *nic_dev = rxq->nic_dev;
444         struct rte_mbuf *cur_mbuf, *rxm = NULL;
445         struct hinic_rx_info *rx_info;
446         u16 sw_ci, rx_buf_len = rxq->buf_len;
447         u32 pkt_len;
448
449         while (remain_pkt_len > 0) {
450                 sw_ci = hinic_get_rq_local_ci(nic_dev->hwdev, rxq->q_id);
451                 rx_info = &rxq->rx_info[sw_ci];
452
453                 hinic_update_rq_local_ci(nic_dev->hwdev, rxq->q_id, 1);
454
455                 pkt_len = remain_pkt_len > rx_buf_len ?
456                         rx_buf_len : remain_pkt_len;
457                 remain_pkt_len -= pkt_len;
458
459                 cur_mbuf = rx_info->mbuf;
460                 cur_mbuf->data_len = (u16)pkt_len;
461                 cur_mbuf->next = NULL;
462
463                 head_mbuf->pkt_len += cur_mbuf->data_len;
464                 head_mbuf->nb_segs++;
465
466                 if (!rxm)
467                         head_mbuf->next = cur_mbuf;
468                 else
469                         rxm->next = cur_mbuf;
470
471                 rxm = cur_mbuf;
472         }
473 }
474
475 static void hinic_rss_deinit(struct hinic_nic_dev *nic_dev)
476 {
477         u8 prio_tc[HINIC_DCB_UP_MAX] = {0};
478         (void)hinic_rss_cfg(nic_dev->hwdev, 0,
479                             nic_dev->rss_tmpl_idx, 0, prio_tc);
480 }
481
482 static int hinic_rss_key_init(struct hinic_nic_dev *nic_dev,
483                               struct rte_eth_rss_conf *rss_conf)
484 {
485         u8 default_rss_key[HINIC_RSS_KEY_SIZE] = {
486                          0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2,
487                          0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0,
488                          0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4,
489                          0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c,
490                          0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa};
491         u8 hashkey[HINIC_RSS_KEY_SIZE] = {0};
492         u8 tmpl_idx = nic_dev->rss_tmpl_idx;
493
494         if (rss_conf->rss_key == NULL)
495                 memcpy(hashkey, default_rss_key, HINIC_RSS_KEY_SIZE);
496         else
497                 memcpy(hashkey, rss_conf->rss_key, rss_conf->rss_key_len);
498
499         return hinic_rss_set_template_tbl(nic_dev->hwdev, tmpl_idx, hashkey);
500 }
501
502 static void hinic_fill_rss_type(struct nic_rss_type *rss_type,
503                                 struct rte_eth_rss_conf *rss_conf)
504 {
505         u64 rss_hf = rss_conf->rss_hf;
506
507         rss_type->ipv4 = (rss_hf & (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4)) ? 1 : 0;
508         rss_type->tcp_ipv4 = (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) ? 1 : 0;
509         rss_type->ipv6 = (rss_hf & (ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6)) ? 1 : 0;
510         rss_type->ipv6_ext = (rss_hf & ETH_RSS_IPV6_EX) ? 1 : 0;
511         rss_type->tcp_ipv6 = (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP) ? 1 : 0;
512         rss_type->tcp_ipv6_ext = (rss_hf & ETH_RSS_IPV6_TCP_EX) ? 1 : 0;
513         rss_type->udp_ipv4 = (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) ? 1 : 0;
514         rss_type->udp_ipv6 = (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP) ? 1 : 0;
515 }
516
517 static void hinic_fillout_indir_tbl(struct hinic_nic_dev *nic_dev, u32 *indir)
518 {
519         u8 rss_queue_count = nic_dev->num_rss;
520         int i = 0, j;
521
522         if (rss_queue_count == 0) {
523                 /* delete q_id from indir tbl */
524                 for (i = 0; i < HINIC_RSS_INDIR_SIZE; i++)
525                         indir[i] = 0xFF;        /* Invalid value in indir tbl */
526         } else {
527                 while (i < HINIC_RSS_INDIR_SIZE)
528                         for (j = 0; (j < rss_queue_count) &&
529                              (i < HINIC_RSS_INDIR_SIZE); j++)
530                                 indir[i++] = nic_dev->rx_queue_list[j];
531         }
532 }
533
534 static int hinic_rss_init(struct hinic_nic_dev *nic_dev,
535                           __rte_unused u8 *rq2iq_map,
536                           struct rte_eth_rss_conf *rss_conf)
537 {
538         u32 indir_tbl[HINIC_RSS_INDIR_SIZE] = {0};
539         struct nic_rss_type rss_type = {0};
540         u8 prio_tc[HINIC_DCB_UP_MAX] = {0};
541         u8 tmpl_idx = 0xFF, num_tc = 0;
542         int err;
543
544         tmpl_idx = nic_dev->rss_tmpl_idx;
545
546         err = hinic_rss_key_init(nic_dev, rss_conf);
547         if (err)
548                 return err;
549
550         if (!nic_dev->rss_indir_flag) {
551                 hinic_fillout_indir_tbl(nic_dev, indir_tbl);
552                 err = hinic_rss_set_indir_tbl(nic_dev->hwdev, tmpl_idx,
553                                               indir_tbl);
554                 if (err)
555                         return err;
556         }
557
558         hinic_fill_rss_type(&rss_type, rss_conf);
559         err = hinic_set_rss_type(nic_dev->hwdev, tmpl_idx, rss_type);
560         if (err)
561                 return err;
562
563         err = hinic_rss_set_hash_engine(nic_dev->hwdev, tmpl_idx,
564                                         HINIC_RSS_HASH_ENGINE_TYPE_TOEP);
565         if (err)
566                 return err;
567
568         return hinic_rss_cfg(nic_dev->hwdev, 1, tmpl_idx, num_tc, prio_tc);
569 }
570
571 static void
572 hinic_add_rq_to_rx_queue_list(struct hinic_nic_dev *nic_dev, u16 queue_id)
573 {
574         u8 rss_queue_count = nic_dev->num_rss;
575
576         RTE_ASSERT(rss_queue_count <= (RTE_DIM(nic_dev->rx_queue_list) - 1));
577
578         nic_dev->rx_queue_list[rss_queue_count] = queue_id;
579         nic_dev->num_rss++;
580 }
581
582 /**
583  * hinic_setup_num_qps - determine num_qps from rss_tmpl_id
584  * @nic_dev: pointer to the private ethernet device
585  * Return: 0 on Success, error code otherwise.
586  **/
587 static int hinic_setup_num_qps(struct hinic_nic_dev *nic_dev)
588 {
589         int err, i;
590
591         if (!(nic_dev->flags & ETH_MQ_RX_RSS_FLAG)) {
592                 nic_dev->flags &= ~ETH_MQ_RX_RSS_FLAG;
593                 nic_dev->num_rss = 0;
594                 if (nic_dev->num_rq > 1) {
595                         /* get rss template id */
596                         err = hinic_rss_template_alloc(nic_dev->hwdev,
597                                                        &nic_dev->rss_tmpl_idx);
598                         if (err) {
599                                 PMD_DRV_LOG(WARNING, "Alloc rss template failed");
600                                 return err;
601                         }
602                         nic_dev->flags |= ETH_MQ_RX_RSS_FLAG;
603                         for (i = 0; i < nic_dev->num_rq; i++)
604                                 hinic_add_rq_to_rx_queue_list(nic_dev, i);
605                 }
606         }
607
608         return 0;
609 }
610
611 static void hinic_destroy_num_qps(struct hinic_nic_dev *nic_dev)
612 {
613         if (nic_dev->flags & ETH_MQ_RX_RSS_FLAG) {
614                 if (hinic_rss_template_free(nic_dev->hwdev,
615                                             nic_dev->rss_tmpl_idx))
616                         PMD_DRV_LOG(WARNING, "Free rss template failed");
617
618                 nic_dev->flags &= ~ETH_MQ_RX_RSS_FLAG;
619         }
620 }
621
622 static int hinic_config_mq_rx_rss(struct hinic_nic_dev *nic_dev, bool on)
623 {
624         int ret = 0;
625
626         if (on) {
627                 ret = hinic_setup_num_qps(nic_dev);
628                 if (ret)
629                         PMD_DRV_LOG(ERR, "Setup num_qps failed");
630         } else {
631                 hinic_destroy_num_qps(nic_dev);
632         }
633
634         return ret;
635 }
636
637 int hinic_config_mq_mode(struct rte_eth_dev *dev, bool on)
638 {
639         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
640         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
641         int ret = 0;
642
643         switch (dev_conf->rxmode.mq_mode) {
644         case ETH_MQ_RX_RSS:
645                 ret = hinic_config_mq_rx_rss(nic_dev, on);
646                 break;
647         default:
648                 break;
649         }
650
651         return ret;
652 }
653
654 int hinic_rx_configure(struct rte_eth_dev *dev)
655 {
656         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
657         struct rte_eth_rss_conf rss_conf =
658                 dev->data->dev_conf.rx_adv_conf.rss_conf;
659         int err;
660         bool lro_en;
661         int max_lro_size;
662         int lro_wqe_num;
663         int buf_size;
664
665         if (nic_dev->flags & ETH_MQ_RX_RSS_FLAG) {
666                 if (rss_conf.rss_hf == 0) {
667                         rss_conf.rss_hf = HINIC_RSS_OFFLOAD_ALL;
668                 } else if ((rss_conf.rss_hf & HINIC_RSS_OFFLOAD_ALL) == 0) {
669                         PMD_DRV_LOG(ERR, "Do not support rss offload all");
670                         goto rss_config_err;
671                 }
672
673                 err = hinic_rss_init(nic_dev, NULL, &rss_conf);
674                 if (err) {
675                         PMD_DRV_LOG(ERR, "Init rss failed");
676                         goto rss_config_err;
677                 }
678         }
679
680         /* Enable both L3/L4 rx checksum offload */
681         if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_CHECKSUM)
682                 nic_dev->rx_csum_en = HINIC_RX_CSUM_OFFLOAD_EN;
683
684         err = hinic_set_rx_csum_offload(nic_dev->hwdev,
685                                         HINIC_RX_CSUM_OFFLOAD_EN);
686         if (err)
687                 goto rx_csum_ofl_err;
688
689         /* config lro */
690         lro_en = dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_TCP_LRO ?
691                         true : false;
692         max_lro_size = dev->data->dev_conf.rxmode.max_lro_pkt_size;
693         buf_size = nic_dev->hwdev->nic_io->rq_buf_size;
694         lro_wqe_num = max_lro_size / buf_size ? (max_lro_size / buf_size) : 1;
695
696         err = hinic_set_rx_lro(nic_dev->hwdev, lro_en, lro_en, lro_wqe_num);
697         if (err) {
698                 PMD_DRV_LOG(ERR, "%s %s lro failed, err: %d, max_lro_size: %d",
699                                 dev->data->name, lro_en ? "Enable" : "Disable",
700                                 err, max_lro_size);
701                 goto set_rx_lro_err;
702         }
703
704         return 0;
705
706 set_rx_lro_err:
707 rx_csum_ofl_err:
708 rss_config_err:
709
710         hinic_destroy_num_qps(nic_dev);
711
712         return HINIC_ERROR;
713 }
714
715 static void hinic_rx_remove_lro(struct hinic_nic_dev *nic_dev)
716 {
717         int err;
718
719         err = hinic_set_rx_lro(nic_dev->hwdev, false, false, 0);
720         if (err)
721                 PMD_DRV_LOG(ERR, "%s disable LRO failed",
722                             nic_dev->proc_dev_name);
723 }
724
725 void hinic_rx_remove_configure(struct rte_eth_dev *dev)
726 {
727         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
728
729         if (nic_dev->flags & ETH_MQ_RX_RSS_FLAG) {
730                 hinic_rss_deinit(nic_dev);
731                 hinic_destroy_num_qps(nic_dev);
732         }
733
734         hinic_rx_remove_lro(nic_dev);
735 }
736
737 void hinic_free_all_rx_mbufs(struct hinic_rxq *rxq)
738 {
739         struct hinic_nic_dev *nic_dev = rxq->nic_dev;
740         struct hinic_rx_info *rx_info;
741         int free_wqebbs =
742                 hinic_get_rq_free_wqebbs(nic_dev->hwdev, rxq->q_id) + 1;
743         volatile struct hinic_rq_cqe *rx_cqe;
744         u16 ci;
745
746         while (free_wqebbs++ < rxq->q_depth) {
747                 ci = hinic_get_rq_local_ci(nic_dev->hwdev, rxq->q_id);
748
749                 rx_cqe = &rxq->rx_cqe[ci];
750
751                 /* clear done bit */
752                 rx_cqe->status = 0;
753
754                 rx_info = &rxq->rx_info[ci];
755                 rte_pktmbuf_free(rx_info->mbuf);
756                 rx_info->mbuf = NULL;
757
758                 hinic_update_rq_local_ci(nic_dev->hwdev, rxq->q_id, 1);
759         }
760 }
761
762 static inline void hinic_rq_cqe_be_to_cpu32(void *dst_le32,
763                                             volatile void *src_be32)
764 {
765 #if defined(__X86_64_SSE__)
766         volatile __m128i *wqe_be = (volatile __m128i *)src_be32;
767         __m128i *wqe_le = (__m128i *)dst_le32;
768         __m128i shuf_mask =  _mm_set_epi8(12, 13, 14, 15, 8, 9, 10,
769                                         11, 4, 5, 6, 7, 0, 1, 2, 3);
770
771         /* l2nic just use first 128 bits */
772         wqe_le[0] = _mm_shuffle_epi8(wqe_be[0], shuf_mask);
773 #elif defined(__ARM64_NEON__)
774         volatile uint8x16_t *wqe_be = (volatile uint8x16_t *)src_be32;
775         uint8x16_t *wqe_le = (uint8x16_t *)dst_le32;
776         const uint8x16_t shuf_mask = {3, 2, 1, 0, 7, 6, 5, 4, 11, 10,
777                                         9, 8, 15, 14, 13, 12};
778
779         /* l2nic just use first 128 bits */
780         wqe_le[0] = vqtbl1q_u8(wqe_be[0], shuf_mask);
781 #else
782         u32 i;
783         volatile u32 *wqe_be = (volatile u32 *)src_be32;
784         u32 *wqe_le = (u32 *)dst_le32;
785
786 #define HINIC_L2NIC_RQ_CQE_USED         4 /* 4Bytes unit */
787
788         for (i = 0; i < HINIC_L2NIC_RQ_CQE_USED; i++) {
789                 *wqe_le = rte_be_to_cpu_32(*wqe_be);
790                 wqe_be++;
791                 wqe_le++;
792         }
793 #endif
794 }
795
796 static inline uint64_t hinic_rx_rss_hash(uint32_t offload_type,
797                                          uint32_t cqe_hass_val,
798                                          uint32_t *rss_hash)
799 {
800         uint32_t rss_type;
801
802         rss_type = HINIC_GET_RSS_TYPES(offload_type);
803         if (likely(rss_type != 0)) {
804                 *rss_hash = cqe_hass_val;
805                 return PKT_RX_RSS_HASH;
806         }
807
808         return 0;
809 }
810
811 static inline uint64_t hinic_rx_csum(uint32_t status, struct hinic_rxq *rxq)
812 {
813         uint32_t checksum_err;
814         uint64_t flags;
815         struct hinic_nic_dev *nic_dev = rxq->nic_dev;
816
817         if (unlikely(!(nic_dev->rx_csum_en & HINIC_RX_CSUM_OFFLOAD_EN)))
818                 return PKT_RX_IP_CKSUM_UNKNOWN;
819
820         /* most case checksum is ok */
821         checksum_err = HINIC_GET_RX_CSUM_ERR(status);
822         if (likely(checksum_err == 0))
823                 return (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD);
824
825         /* If BYPASS bit set, all other status indications should be ignored */
826         if (unlikely(HINIC_CSUM_ERR_BYPASSED(checksum_err)))
827                 return PKT_RX_IP_CKSUM_UNKNOWN;
828
829         flags = 0;
830
831         /* IP checksum error */
832         if (HINIC_CSUM_ERR_IP(checksum_err))
833                 flags |= PKT_RX_IP_CKSUM_BAD;
834         else
835                 flags |= PKT_RX_IP_CKSUM_GOOD;
836
837         /* L4 checksum error */
838         if (HINIC_CSUM_ERR_L4(checksum_err))
839                 flags |= PKT_RX_L4_CKSUM_BAD;
840         else
841                 flags |= PKT_RX_L4_CKSUM_GOOD;
842
843         if (unlikely(HINIC_CSUM_ERR_OTHER(checksum_err)))
844                 flags = PKT_RX_L4_CKSUM_NONE;
845
846         rxq->rxq_stats.errors++;
847
848         return flags;
849 }
850
851 static inline uint64_t hinic_rx_vlan(uint32_t offload_type, uint32_t vlan_len,
852                                      uint16_t *vlan_tci)
853 {
854         uint16_t vlan_tag;
855
856         vlan_tag = HINIC_GET_RX_VLAN_TAG(vlan_len);
857         if (!HINIC_GET_RX_VLAN_OFFLOAD_EN(offload_type) || 0 == vlan_tag) {
858                 *vlan_tci = 0;
859                 return 0;
860         }
861
862         *vlan_tci = vlan_tag;
863
864         return PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED;
865 }
866
867 static inline u32 hinic_rx_alloc_mbuf_bulk(struct hinic_rxq *rxq,
868                                            struct rte_mbuf **mbufs,
869                                            u32 exp_mbuf_cnt)
870 {
871         int rc;
872         u32 avail_cnt;
873
874         rc = rte_pktmbuf_alloc_bulk(rxq->mb_pool, mbufs, exp_mbuf_cnt);
875         if (likely(rc == HINIC_OK)) {
876                 avail_cnt = exp_mbuf_cnt;
877         } else {
878                 avail_cnt = 0;
879                 rxq->rxq_stats.rx_nombuf += exp_mbuf_cnt;
880         }
881
882         return avail_cnt;
883 }
884
885 static struct rte_mbuf *hinic_rx_alloc_mbuf(struct hinic_rxq *rxq,
886                                         dma_addr_t *dma_addr)
887 {
888         struct rte_mbuf *mbuf = NULL;
889         int rc;
890
891         rc = rte_pktmbuf_alloc_bulk(rxq->mb_pool, &mbuf, 1);
892         if (unlikely(rc != HINIC_OK))
893                 return NULL;
894
895         *dma_addr = rte_mbuf_data_iova_default(mbuf);
896
897         return mbuf;
898 }
899
900 static inline void hinic_rearm_rxq_mbuf(struct hinic_rxq *rxq)
901 {
902         u16 pi;
903         u32 i, free_wqebbs, rearm_wqebbs, exp_wqebbs;
904         dma_addr_t dma_addr;
905         struct hinic_rq_wqe *rq_wqe;
906         struct rte_mbuf **rearm_mbufs;
907
908         /* check free wqebb fo rearm */
909         free_wqebbs = HINIC_GET_RQ_FREE_WQEBBS(rxq);
910         if (unlikely(free_wqebbs < rxq->rx_free_thresh))
911                 return;
912
913         /* get rearm mbuf array */
914         pi = HINIC_GET_RQ_LOCAL_PI(rxq);
915         rearm_mbufs = (struct rte_mbuf **)(&rxq->rx_info[pi]);
916
917         /* check rxq free wqebbs turn around */
918         exp_wqebbs = rxq->q_depth - pi;
919         if (free_wqebbs < exp_wqebbs)
920                 exp_wqebbs = free_wqebbs;
921
922         /* alloc mbuf in bulk */
923         rearm_wqebbs = hinic_rx_alloc_mbuf_bulk(rxq, rearm_mbufs, exp_wqebbs);
924         if (unlikely(rearm_wqebbs == 0))
925                 return;
926
927         /* rearm rx mbuf */
928         rq_wqe = WQ_WQE_ADDR(rxq->wq, (u32)pi);
929         for (i = 0; i < rearm_wqebbs; i++) {
930                 dma_addr = rte_mbuf_data_iova_default(rearm_mbufs[i]);
931                 rq_wqe->buf_desc.addr_high =
932                                         cpu_to_be32(upper_32_bits(dma_addr));
933                 rq_wqe->buf_desc.addr_low =
934                                         cpu_to_be32(lower_32_bits(dma_addr));
935                 rq_wqe++;
936         }
937         rxq->wq->prod_idx += rearm_wqebbs;
938         rxq->wq->delta -= rearm_wqebbs;
939
940         /* update rq hw_pi */
941         rte_wmb();
942         HINIC_UPDATE_RQ_HW_PI(rxq, pi + rearm_wqebbs);
943 }
944
945 void hinic_rx_alloc_pkts(struct hinic_rxq *rxq)
946 {
947         struct hinic_nic_dev *nic_dev = rxq->nic_dev;
948         struct hinic_rq_wqe *rq_wqe;
949         struct hinic_rx_info *rx_info;
950         struct rte_mbuf *mb;
951         dma_addr_t dma_addr;
952         u16 pi = 0;
953         int i, free_wqebbs;
954
955         free_wqebbs = HINIC_GET_RQ_FREE_WQEBBS(rxq);
956         for (i = 0; i < free_wqebbs; i++) {
957                 mb = hinic_rx_alloc_mbuf(rxq, &dma_addr);
958                 if (unlikely(!mb)) {
959                         rxq->rxq_stats.rx_nombuf++;
960                         break;
961                 }
962
963                 rq_wqe = hinic_get_rq_wqe(nic_dev->hwdev, rxq->q_id, &pi);
964                 if (unlikely(!rq_wqe)) {
965                         rte_pktmbuf_free(mb);
966                         break;
967                 }
968
969                 /* fill buffer address only */
970                 rq_wqe->buf_desc.addr_high =
971                                 cpu_to_be32(upper_32_bits(dma_addr));
972                 rq_wqe->buf_desc.addr_low =
973                                 cpu_to_be32(lower_32_bits(dma_addr));
974
975                 rx_info = &rxq->rx_info[pi];
976                 rx_info->mbuf = mb;
977         }
978
979         if (likely(i > 0)) {
980                 rte_wmb();
981                 HINIC_UPDATE_RQ_HW_PI(rxq, pi + 1);
982         }
983 }
984
985 u16 hinic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, u16 nb_pkts)
986 {
987         struct rte_mbuf *rxm;
988         struct hinic_rxq *rxq = rx_queue;
989         struct hinic_rx_info *rx_info;
990         volatile struct hinic_rq_cqe *rx_cqe;
991         u16 rx_buf_len, pkts = 0;
992         u16 sw_ci, ci_mask, wqebb_cnt = 0;
993         u32 pkt_len, status, vlan_len, lro_num;
994         u64 rx_bytes = 0;
995         struct hinic_rq_cqe cqe;
996         u32 offload_type, rss_hash;
997
998         rx_buf_len = rxq->buf_len;
999
1000         /* 1. get polling start ci */
1001         ci_mask = HINIC_GET_RQ_WQE_MASK(rxq);
1002         sw_ci = HINIC_GET_RQ_LOCAL_CI(rxq);
1003
1004         while (pkts < nb_pkts) {
1005                  /* 2. current ci is done */
1006                 rx_cqe = &rxq->rx_cqe[sw_ci];
1007                 status = __atomic_load_n(&rx_cqe->status, __ATOMIC_ACQUIRE);
1008                 if (!HINIC_GET_RX_DONE_BE(status))
1009                         break;
1010
1011                 /* convert cqe and get packet length */
1012                 hinic_rq_cqe_be_to_cpu32(&cqe, (volatile void *)rx_cqe);
1013                 vlan_len = cqe.vlan_len;
1014
1015                 rx_info = &rxq->rx_info[sw_ci];
1016                 rxm = rx_info->mbuf;
1017
1018                 /* 3. next ci point and prefetch */
1019                 sw_ci++;
1020                 sw_ci &= ci_mask;
1021
1022                 /* prefetch next mbuf first 64B */
1023                 rte_prefetch0(rxq->rx_info[sw_ci].mbuf);
1024
1025                 /* 4. jumbo frame process */
1026                 pkt_len = HINIC_GET_RX_PKT_LEN(vlan_len);
1027                 if (likely(pkt_len <= rx_buf_len)) {
1028                         rxm->data_len = pkt_len;
1029                         rxm->pkt_len = pkt_len;
1030                         wqebb_cnt++;
1031                 } else {
1032                         rxm->data_len = rx_buf_len;
1033                         rxm->pkt_len = rx_buf_len;
1034
1035                         /* if receive jumbo, updating ci will be done by
1036                          * hinic_recv_jumbo_pkt function.
1037                          */
1038                         HINIC_UPDATE_RQ_LOCAL_CI(rxq, wqebb_cnt + 1);
1039                         wqebb_cnt = 0;
1040                         hinic_recv_jumbo_pkt(rxq, rxm, pkt_len - rx_buf_len);
1041                         sw_ci = HINIC_GET_RQ_LOCAL_CI(rxq);
1042                 }
1043
1044                 /* 5. vlan/checksum/rss/pkt_type/gro offload */
1045                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1046                 rxm->port = rxq->port_id;
1047                 offload_type = cqe.offload_type;
1048
1049                 /* vlan offload */
1050                 rxm->ol_flags |= hinic_rx_vlan(offload_type, vlan_len,
1051                                                &rxm->vlan_tci);
1052
1053                 /* checksum offload */
1054                 rxm->ol_flags |= hinic_rx_csum(cqe.status, rxq);
1055
1056                 /* rss hash offload */
1057                 rss_hash = cqe.rss_hash;
1058                 rxm->ol_flags |= hinic_rx_rss_hash(offload_type, rss_hash,
1059                                                    &rxm->hash.rss);
1060
1061                 /* lro offload */
1062                 lro_num = HINIC_GET_RX_NUM_LRO(cqe.status);
1063                 if (unlikely(lro_num != 0)) {
1064                         rxm->ol_flags |= PKT_RX_LRO;
1065                         rxm->tso_segsz = pkt_len / lro_num;
1066                 }
1067
1068                 /* 6. clear done bit */
1069                 rx_cqe->status = 0;
1070
1071                 rx_bytes += pkt_len;
1072                 rx_pkts[pkts++] = rxm;
1073         }
1074
1075         if (pkts) {
1076                 /* 7. update ci */
1077                 HINIC_UPDATE_RQ_LOCAL_CI(rxq, wqebb_cnt);
1078
1079                 /* do packet stats */
1080                 rxq->rxq_stats.packets += pkts;
1081                 rxq->rxq_stats.bytes += rx_bytes;
1082         }
1083         rxq->rxq_stats.burst_pkts = pkts;
1084
1085         /* 8. rearm mbuf to rxq */
1086         hinic_rearm_rxq_mbuf(rxq);
1087
1088         return pkts;
1089 }