1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Huawei Technologies Co., Ltd
11 #include "base/hinic_compat.h"
12 #include "base/hinic_pmd_hwdev.h"
13 #include "base/hinic_pmd_wq.h"
14 #include "base/hinic_pmd_niccfg.h"
15 #include "base/hinic_pmd_nicio.h"
16 #include "hinic_pmd_ethdev.h"
17 #include "hinic_pmd_rx.h"
19 /* rxq wq operations */
20 #define HINIC_GET_RQ_WQE_MASK(rxq) \
23 #define HINIC_GET_RQ_LOCAL_CI(rxq) \
24 (((rxq)->wq->cons_idx) & HINIC_GET_RQ_WQE_MASK(rxq))
26 #define HINIC_GET_RQ_LOCAL_PI(rxq) \
27 (((rxq)->wq->prod_idx) & HINIC_GET_RQ_WQE_MASK(rxq))
29 #define HINIC_UPDATE_RQ_LOCAL_CI(rxq, wqebb_cnt) \
31 (rxq)->wq->cons_idx += (wqebb_cnt); \
32 (rxq)->wq->delta += (wqebb_cnt); \
35 #define HINIC_UPDATE_RQ_HW_PI(rxq, pi) \
36 (*((rxq)->pi_virt_addr) = \
37 cpu_to_be16((pi) & HINIC_GET_RQ_WQE_MASK(rxq)))
39 #define HINIC_GET_RQ_FREE_WQEBBS(rxq) ((rxq)->wq->delta - 1)
41 /* rxq cqe done and status bit */
42 #define HINIC_GET_RX_DONE_BE(status) \
45 #define HINIC_RX_CSUM_OFFLOAD_EN 0xFFF
47 #define RQ_CQE_SGE_VLAN_SHIFT 0
48 #define RQ_CQE_SGE_LEN_SHIFT 16
50 #define RQ_CQE_SGE_VLAN_MASK 0xFFFFU
51 #define RQ_CQE_SGE_LEN_MASK 0xFFFFU
53 #define RQ_CQE_SGE_GET(val, member) \
54 (((val) >> RQ_CQE_SGE_##member##_SHIFT) & RQ_CQE_SGE_##member##_MASK)
56 #define HINIC_GET_RX_VLAN_TAG(vlan_len) \
57 RQ_CQE_SGE_GET(vlan_len, VLAN)
59 #define HINIC_GET_RX_PKT_LEN(vlan_len) \
60 RQ_CQE_SGE_GET(vlan_len, LEN)
62 #define RQ_CQE_STATUS_CSUM_ERR_SHIFT 0
63 #define RQ_CQE_STATUS_NUM_LRO_SHIFT 16
64 #define RQ_CQE_STATUS_LRO_PUSH_SHIFT 25
65 #define RQ_CQE_STATUS_LRO_ENTER_SHIFT 26
66 #define RQ_CQE_STATUS_LRO_INTR_SHIFT 27
68 #define RQ_CQE_STATUS_BP_EN_SHIFT 30
69 #define RQ_CQE_STATUS_RXDONE_SHIFT 31
70 #define RQ_CQE_STATUS_FLUSH_SHIFT 28
72 #define RQ_CQE_STATUS_CSUM_ERR_MASK 0xFFFFU
73 #define RQ_CQE_STATUS_NUM_LRO_MASK 0xFFU
74 #define RQ_CQE_STATUS_LRO_PUSH_MASK 0X1U
75 #define RQ_CQE_STATUS_LRO_ENTER_MASK 0X1U
76 #define RQ_CQE_STATUS_LRO_INTR_MASK 0X1U
77 #define RQ_CQE_STATUS_BP_EN_MASK 0X1U
78 #define RQ_CQE_STATUS_RXDONE_MASK 0x1U
79 #define RQ_CQE_STATUS_FLUSH_MASK 0x1U
81 #define RQ_CQE_STATUS_GET(val, member) \
82 (((val) >> RQ_CQE_STATUS_##member##_SHIFT) & \
83 RQ_CQE_STATUS_##member##_MASK)
85 #define RQ_CQE_STATUS_CLEAR(val, member) \
86 ((val) & (~(RQ_CQE_STATUS_##member##_MASK << \
87 RQ_CQE_STATUS_##member##_SHIFT)))
89 #define HINIC_GET_RX_CSUM_ERR(status) \
90 RQ_CQE_STATUS_GET(status, CSUM_ERR)
92 #define HINIC_GET_RX_DONE(status) \
93 RQ_CQE_STATUS_GET(status, RXDONE)
95 #define HINIC_GET_RX_FLUSH(status) \
96 RQ_CQE_STATUS_GET(status, FLUSH)
98 #define HINIC_GET_RX_BP_EN(status) \
99 RQ_CQE_STATUS_GET(status, BP_EN)
101 #define HINIC_GET_RX_NUM_LRO(status) \
102 RQ_CQE_STATUS_GET(status, NUM_LRO)
105 #define RQ_CTRL_BUFDESC_SECT_LEN_SHIFT 0
106 #define RQ_CTRL_COMPLETE_FORMAT_SHIFT 15
107 #define RQ_CTRL_COMPLETE_LEN_SHIFT 27
108 #define RQ_CTRL_LEN_SHIFT 29
110 #define RQ_CTRL_BUFDESC_SECT_LEN_MASK 0xFFU
111 #define RQ_CTRL_COMPLETE_FORMAT_MASK 0x1U
112 #define RQ_CTRL_COMPLETE_LEN_MASK 0x3U
113 #define RQ_CTRL_LEN_MASK 0x3U
115 #define RQ_CTRL_SET(val, member) \
116 (((val) & RQ_CTRL_##member##_MASK) << RQ_CTRL_##member##_SHIFT)
118 #define RQ_CTRL_GET(val, member) \
119 (((val) >> RQ_CTRL_##member##_SHIFT) & RQ_CTRL_##member##_MASK)
121 #define RQ_CTRL_CLEAR(val, member) \
122 ((val) & (~(RQ_CTRL_##member##_MASK << RQ_CTRL_##member##_SHIFT)))
124 #define RQ_CQE_PKT_NUM_SHIFT 1
125 #define RQ_CQE_PKT_FIRST_LEN_SHIFT 19
126 #define RQ_CQE_PKT_LAST_LEN_SHIFT 6
127 #define RQ_CQE_SUPER_CQE_EN_SHIFT 0
129 #define RQ_CQE_PKT_FIRST_LEN_MASK 0x1FFFU
130 #define RQ_CQE_PKT_LAST_LEN_MASK 0x1FFFU
131 #define RQ_CQE_PKT_NUM_MASK 0x1FU
132 #define RQ_CQE_SUPER_CQE_EN_MASK 0x1
134 #define RQ_CQE_PKT_NUM_GET(val, member) \
135 (((val) >> RQ_CQE_PKT_##member##_SHIFT) & RQ_CQE_PKT_##member##_MASK)
137 #define HINIC_GET_RQ_CQE_PKT_NUM(pkt_info) RQ_CQE_PKT_NUM_GET(pkt_info, NUM)
139 #define RQ_CQE_SUPER_CQE_EN_GET(val, member) \
140 (((val) >> RQ_CQE_##member##_SHIFT) & RQ_CQE_##member##_MASK)
142 #define HINIC_GET_SUPER_CQE_EN(pkt_info) \
143 RQ_CQE_SUPER_CQE_EN_GET(pkt_info, SUPER_CQE_EN)
145 #define RQ_CQE_OFFOLAD_TYPE_VLAN_EN_SHIFT 21
146 #define RQ_CQE_OFFOLAD_TYPE_VLAN_EN_MASK 0x1U
148 #define RQ_CQE_OFFOLAD_TYPE_PKT_TYPE_SHIFT 0
149 #define RQ_CQE_OFFOLAD_TYPE_PKT_TYPE_MASK 0xFFFU
151 #define RQ_CQE_OFFOLAD_TYPE_PKT_UMBCAST_SHIFT 19
152 #define RQ_CQE_OFFOLAD_TYPE_PKT_UMBCAST_MASK 0x3U
154 #define RQ_CQE_OFFOLAD_TYPE_RSS_TYPE_SHIFT 24
155 #define RQ_CQE_OFFOLAD_TYPE_RSS_TYPE_MASK 0xFFU
157 #define RQ_CQE_OFFOLAD_TYPE_GET(val, member) (((val) >> \
158 RQ_CQE_OFFOLAD_TYPE_##member##_SHIFT) & \
159 RQ_CQE_OFFOLAD_TYPE_##member##_MASK)
161 #define HINIC_GET_RX_VLAN_OFFLOAD_EN(offload_type) \
162 RQ_CQE_OFFOLAD_TYPE_GET(offload_type, VLAN_EN)
164 #define HINIC_GET_RSS_TYPES(offload_type) \
165 RQ_CQE_OFFOLAD_TYPE_GET(offload_type, RSS_TYPE)
167 #define HINIC_GET_RX_PKT_TYPE(offload_type) \
168 RQ_CQE_OFFOLAD_TYPE_GET(offload_type, PKT_TYPE)
170 #define HINIC_GET_RX_PKT_UMBCAST(offload_type) \
171 RQ_CQE_OFFOLAD_TYPE_GET(offload_type, PKT_UMBCAST)
173 #define RQ_CQE_STATUS_CSUM_BYPASS_VAL 0x80U
174 #define RQ_CQE_STATUS_CSUM_ERR_IP_MASK 0x39U
175 #define RQ_CQE_STATUS_CSUM_ERR_L4_MASK 0x46U
176 #define RQ_CQE_STATUS_CSUM_ERR_OTHER 0x100U
178 #define HINIC_CSUM_ERR_BYPASSED(csum_err) \
179 ((csum_err) == RQ_CQE_STATUS_CSUM_BYPASS_VAL)
181 #define HINIC_CSUM_ERR_IP(csum_err) \
182 ((csum_err) & RQ_CQE_STATUS_CSUM_ERR_IP_MASK)
184 #define HINIC_CSUM_ERR_L4(csum_err) \
185 ((csum_err) & RQ_CQE_STATUS_CSUM_ERR_L4_MASK)
187 #define HINIC_CSUM_ERR_OTHER(csum_err) \
188 ((csum_err) == RQ_CQE_STATUS_CSUM_ERR_OTHER)
191 void hinic_get_func_rx_buf_size(struct hinic_nic_dev *nic_dev)
193 struct hinic_rxq *rxq;
197 for (q_id = 0; q_id < nic_dev->num_rq; q_id++) {
198 rxq = nic_dev->rxqs[q_id];
204 buf_size = rxq->buf_len;
206 buf_size = buf_size > rxq->buf_len ? rxq->buf_len : buf_size;
209 nic_dev->hwdev->nic_io->rq_buf_size = buf_size;
212 int hinic_create_rq(struct hinic_hwdev *hwdev, u16 q_id, u16 rq_depth)
215 struct hinic_nic_io *nic_io = hwdev->nic_io;
216 struct hinic_qp *qp = &nic_io->qps[q_id];
217 struct hinic_rq *rq = &qp->rq;
219 /* in case of hardware still generate interrupt, do not use msix 0 */
220 rq->msix_entry_idx = 1;
222 rq->rq_depth = rq_depth;
223 nic_io->rq_depth = rq_depth;
225 err = hinic_wq_allocate(hwdev, &nic_io->rq_wq[q_id],
226 HINIC_RQ_WQEBB_SHIFT, nic_io->rq_depth);
228 PMD_DRV_LOG(ERR, "Failed to allocate WQ for RQ");
231 rq->wq = &nic_io->rq_wq[q_id];
234 (volatile u16 *)dma_zalloc_coherent(hwdev, HINIC_PAGE_SIZE,
237 if (!rq->pi_virt_addr) {
238 PMD_DRV_LOG(ERR, "Failed to allocate rq pi virt addr");
240 goto rq_pi_alloc_err;
246 hinic_wq_free(hwdev, &nic_io->rq_wq[q_id]);
251 void hinic_destroy_rq(struct hinic_hwdev *hwdev, u16 q_id)
253 struct hinic_nic_io *nic_io = hwdev->nic_io;
254 struct hinic_qp *qp = &nic_io->qps[q_id];
255 struct hinic_rq *rq = &qp->rq;
257 if (qp->rq.wq == NULL)
260 dma_free_coherent_volatile(hwdev, HINIC_PAGE_SIZE,
261 (volatile void *)rq->pi_virt_addr,
263 hinic_wq_free(nic_io->hwdev, qp->rq.wq);
268 hinic_prepare_rq_wqe(void *wqe, __rte_unused u16 pi, dma_addr_t buf_addr,
271 struct hinic_rq_wqe *rq_wqe = wqe;
272 struct hinic_rq_ctrl *ctrl = &rq_wqe->ctrl;
273 struct hinic_rq_cqe_sect *cqe_sect = &rq_wqe->cqe_sect;
274 struct hinic_rq_bufdesc *buf_desc = &rq_wqe->buf_desc;
275 u32 rq_ceq_len = sizeof(struct hinic_rq_cqe);
278 RQ_CTRL_SET(SIZE_8BYTES(sizeof(*ctrl)), LEN) |
279 RQ_CTRL_SET(SIZE_8BYTES(sizeof(*cqe_sect)), COMPLETE_LEN) |
280 RQ_CTRL_SET(SIZE_8BYTES(sizeof(*buf_desc)), BUFDESC_SECT_LEN) |
281 RQ_CTRL_SET(RQ_COMPLETE_SGE, COMPLETE_FORMAT);
283 hinic_set_sge(&cqe_sect->sge, cqe_dma, rq_ceq_len);
285 buf_desc->addr_high = upper_32_bits(buf_addr);
286 buf_desc->addr_low = lower_32_bits(buf_addr);
289 void hinic_rxq_get_stats(struct hinic_rxq *rxq, struct hinic_rxq_stats *stats)
294 memcpy(stats, &rxq->rxq_stats, sizeof(rxq->rxq_stats));
297 void hinic_rxq_stats_reset(struct hinic_rxq *rxq)
299 struct hinic_rxq_stats *rxq_stats;
304 rxq_stats = &rxq->rxq_stats;
305 memset(rxq_stats, 0, sizeof(*rxq_stats));
308 static int hinic_rx_alloc_cqe(struct hinic_rxq *rxq)
312 cqe_mem_size = sizeof(struct hinic_rq_cqe) * rxq->q_depth;
313 rxq->cqe_start_vaddr =
314 dma_zalloc_coherent(rxq->nic_dev->hwdev,
315 cqe_mem_size, &rxq->cqe_start_paddr,
317 if (!rxq->cqe_start_vaddr) {
318 PMD_DRV_LOG(ERR, "Allocate cqe dma memory failed");
322 rxq->rx_cqe = (struct hinic_rq_cqe *)rxq->cqe_start_vaddr;
327 static void hinic_rx_free_cqe(struct hinic_rxq *rxq)
331 cqe_mem_size = sizeof(struct hinic_rq_cqe) * rxq->q_depth;
332 dma_free_coherent(rxq->nic_dev->hwdev, cqe_mem_size,
333 rxq->cqe_start_vaddr, rxq->cqe_start_paddr);
334 rxq->cqe_start_vaddr = NULL;
337 static int hinic_rx_fill_wqe(struct hinic_rxq *rxq)
339 struct hinic_nic_dev *nic_dev = rxq->nic_dev;
340 struct hinic_rq_wqe *rq_wqe;
341 dma_addr_t buf_dma_addr, cqe_dma_addr;
346 cqe_dma_addr = rxq->cqe_start_paddr;
347 for (i = 0; i < rxq->q_depth; i++) {
348 rq_wqe = hinic_get_rq_wqe(nic_dev->hwdev, rxq->q_id, &pi);
350 PMD_DRV_LOG(ERR, "Get rq wqe failed");
354 hinic_prepare_rq_wqe(rq_wqe, pi, buf_dma_addr, cqe_dma_addr);
355 cqe_dma_addr += sizeof(struct hinic_rq_cqe);
357 hinic_cpu_to_be32(rq_wqe, sizeof(struct hinic_rq_wqe));
360 hinic_return_rq_wqe(nic_dev->hwdev, rxq->q_id, i);
365 /* alloc cqe and prepare rqe */
366 int hinic_setup_rx_resources(struct hinic_rxq *rxq)
371 rx_info_sz = rxq->q_depth * sizeof(*rxq->rx_info);
372 rxq->rx_info = kzalloc_aligned(rx_info_sz, GFP_KERNEL);
376 err = hinic_rx_alloc_cqe(rxq);
378 PMD_DRV_LOG(ERR, "Allocate rx cqe failed");
382 pkts = hinic_rx_fill_wqe(rxq);
383 if (pkts != rxq->q_depth) {
384 PMD_DRV_LOG(ERR, "Fill rx wqe failed");
392 hinic_rx_free_cqe(rxq);
401 void hinic_free_rx_resources(struct hinic_rxq *rxq)
403 if (rxq->rx_info == NULL)
406 hinic_rx_free_cqe(rxq);
411 void hinic_free_all_rx_resources(struct rte_eth_dev *eth_dev)
414 struct hinic_nic_dev *nic_dev =
415 HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
417 for (q_id = 0; q_id < nic_dev->num_rq; q_id++) {
418 eth_dev->data->rx_queues[q_id] = NULL;
420 if (nic_dev->rxqs[q_id] == NULL)
423 hinic_free_all_rx_mbufs(nic_dev->rxqs[q_id]);
424 hinic_free_rx_resources(nic_dev->rxqs[q_id]);
425 kfree(nic_dev->rxqs[q_id]);
426 nic_dev->rxqs[q_id] = NULL;
430 void hinic_free_all_rx_mbuf(struct rte_eth_dev *eth_dev)
432 struct hinic_nic_dev *nic_dev =
433 HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
436 for (q_id = 0; q_id < nic_dev->num_rq; q_id++)
437 hinic_free_all_rx_mbufs(nic_dev->rxqs[q_id]);
440 static void hinic_recv_jumbo_pkt(struct hinic_rxq *rxq,
441 struct rte_mbuf *head_mbuf,
444 struct hinic_nic_dev *nic_dev = rxq->nic_dev;
445 struct rte_mbuf *cur_mbuf, *rxm = NULL;
446 struct hinic_rx_info *rx_info;
447 u16 sw_ci, rx_buf_len = rxq->buf_len;
450 while (remain_pkt_len > 0) {
451 sw_ci = hinic_get_rq_local_ci(nic_dev->hwdev, rxq->q_id);
452 rx_info = &rxq->rx_info[sw_ci];
454 hinic_update_rq_local_ci(nic_dev->hwdev, rxq->q_id, 1);
456 pkt_len = remain_pkt_len > rx_buf_len ?
457 rx_buf_len : remain_pkt_len;
458 remain_pkt_len -= pkt_len;
460 cur_mbuf = rx_info->mbuf;
461 cur_mbuf->data_len = (u16)pkt_len;
462 cur_mbuf->next = NULL;
464 head_mbuf->pkt_len += cur_mbuf->data_len;
465 head_mbuf->nb_segs++;
468 head_mbuf->next = cur_mbuf;
470 rxm->next = cur_mbuf;
476 static void hinic_rss_deinit(struct hinic_nic_dev *nic_dev)
478 u8 prio_tc[HINIC_DCB_UP_MAX] = {0};
479 (void)hinic_rss_cfg(nic_dev->hwdev, 0,
480 nic_dev->rss_tmpl_idx, 0, prio_tc);
483 static int hinic_rss_key_init(struct hinic_nic_dev *nic_dev,
484 struct rte_eth_rss_conf *rss_conf)
486 u8 default_rss_key[HINIC_RSS_KEY_SIZE] = {
487 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2,
488 0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0,
489 0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4,
490 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c,
491 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa};
492 u8 hashkey[HINIC_RSS_KEY_SIZE] = {0};
493 u8 tmpl_idx = nic_dev->rss_tmpl_idx;
495 if (rss_conf->rss_key == NULL)
496 memcpy(hashkey, default_rss_key, HINIC_RSS_KEY_SIZE);
498 memcpy(hashkey, rss_conf->rss_key, rss_conf->rss_key_len);
500 return hinic_rss_set_template_tbl(nic_dev->hwdev, tmpl_idx, hashkey);
503 static void hinic_fill_rss_type(struct nic_rss_type *rss_type,
504 struct rte_eth_rss_conf *rss_conf)
506 u64 rss_hf = rss_conf->rss_hf;
508 rss_type->ipv4 = (rss_hf & (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4)) ? 1 : 0;
509 rss_type->tcp_ipv4 = (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) ? 1 : 0;
510 rss_type->ipv6 = (rss_hf & (ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6)) ? 1 : 0;
511 rss_type->ipv6_ext = (rss_hf & ETH_RSS_IPV6_EX) ? 1 : 0;
512 rss_type->tcp_ipv6 = (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP) ? 1 : 0;
513 rss_type->tcp_ipv6_ext = (rss_hf & ETH_RSS_IPV6_TCP_EX) ? 1 : 0;
514 rss_type->udp_ipv4 = (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) ? 1 : 0;
515 rss_type->udp_ipv6 = (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP) ? 1 : 0;
518 static void hinic_fillout_indir_tbl(struct hinic_nic_dev *nic_dev, u32 *indir)
520 u8 rss_queue_count = nic_dev->num_rss;
523 if (rss_queue_count == 0) {
524 /* delete q_id from indir tbl */
525 for (i = 0; i < HINIC_RSS_INDIR_SIZE; i++)
526 indir[i] = 0xFF; /* Invalid value in indir tbl */
528 while (i < HINIC_RSS_INDIR_SIZE)
529 for (j = 0; (j < rss_queue_count) &&
530 (i < HINIC_RSS_INDIR_SIZE); j++)
531 indir[i++] = nic_dev->rx_queue_list[j];
535 static int hinic_rss_init(struct hinic_nic_dev *nic_dev,
536 __attribute__((unused)) u8 *rq2iq_map,
537 struct rte_eth_rss_conf *rss_conf)
539 u32 indir_tbl[HINIC_RSS_INDIR_SIZE] = {0};
540 struct nic_rss_type rss_type = {0};
541 u8 prio_tc[HINIC_DCB_UP_MAX] = {0};
542 u8 tmpl_idx = 0xFF, num_tc = 0;
545 tmpl_idx = nic_dev->rss_tmpl_idx;
547 err = hinic_rss_key_init(nic_dev, rss_conf);
551 if (!nic_dev->rss_indir_flag) {
552 hinic_fillout_indir_tbl(nic_dev, indir_tbl);
553 err = hinic_rss_set_indir_tbl(nic_dev->hwdev, tmpl_idx,
559 hinic_fill_rss_type(&rss_type, rss_conf);
560 err = hinic_set_rss_type(nic_dev->hwdev, tmpl_idx, rss_type);
564 err = hinic_rss_set_hash_engine(nic_dev->hwdev, tmpl_idx,
565 HINIC_RSS_HASH_ENGINE_TYPE_TOEP);
569 return hinic_rss_cfg(nic_dev->hwdev, 1, tmpl_idx, num_tc, prio_tc);
573 hinic_add_rq_to_rx_queue_list(struct hinic_nic_dev *nic_dev, u16 queue_id)
575 u8 rss_queue_count = nic_dev->num_rss;
577 RTE_ASSERT(rss_queue_count <= (RTE_DIM(nic_dev->rx_queue_list) - 1));
579 nic_dev->rx_queue_list[rss_queue_count] = queue_id;
584 * hinic_setup_num_qps - determine num_qps from rss_tmpl_id
585 * @nic_dev: pointer to the private ethernet device
586 * Return: 0 on Success, error code otherwise.
588 static int hinic_setup_num_qps(struct hinic_nic_dev *nic_dev)
592 if (!(nic_dev->flags & ETH_MQ_RX_RSS_FLAG)) {
593 nic_dev->flags &= ~ETH_MQ_RX_RSS_FLAG;
594 nic_dev->num_rss = 0;
595 if (nic_dev->num_rq > 1) {
596 /* get rss template id */
597 err = hinic_rss_template_alloc(nic_dev->hwdev,
598 &nic_dev->rss_tmpl_idx);
600 PMD_DRV_LOG(WARNING, "Alloc rss template failed");
603 nic_dev->flags |= ETH_MQ_RX_RSS_FLAG;
604 for (i = 0; i < nic_dev->num_rq; i++)
605 hinic_add_rq_to_rx_queue_list(nic_dev, i);
612 static void hinic_destroy_num_qps(struct hinic_nic_dev *nic_dev)
614 if (nic_dev->flags & ETH_MQ_RX_RSS_FLAG) {
615 if (hinic_rss_template_free(nic_dev->hwdev,
616 nic_dev->rss_tmpl_idx))
617 PMD_DRV_LOG(WARNING, "Free rss template failed");
619 nic_dev->flags &= ~ETH_MQ_RX_RSS_FLAG;
623 static int hinic_config_mq_rx_rss(struct hinic_nic_dev *nic_dev, bool on)
628 ret = hinic_setup_num_qps(nic_dev);
630 PMD_DRV_LOG(ERR, "Setup num_qps failed");
632 hinic_destroy_num_qps(nic_dev);
638 int hinic_config_mq_mode(struct rte_eth_dev *dev, bool on)
640 struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
641 struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
644 switch (dev_conf->rxmode.mq_mode) {
646 ret = hinic_config_mq_rx_rss(nic_dev, on);
655 int hinic_rx_configure(struct rte_eth_dev *dev)
657 struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
658 struct rte_eth_rss_conf rss_conf =
659 dev->data->dev_conf.rx_adv_conf.rss_conf;
662 if (nic_dev->flags & ETH_MQ_RX_RSS_FLAG) {
663 if (rss_conf.rss_hf == 0) {
664 rss_conf.rss_hf = HINIC_RSS_OFFLOAD_ALL;
665 } else if ((rss_conf.rss_hf & HINIC_RSS_OFFLOAD_ALL) == 0) {
666 PMD_DRV_LOG(ERR, "Do not support rss offload all");
670 err = hinic_rss_init(nic_dev, NULL, &rss_conf);
672 PMD_DRV_LOG(ERR, "Init rss failed");
677 /* Enable both L3/L4 rx checksum offload */
678 if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_CHECKSUM)
679 nic_dev->rx_csum_en = HINIC_RX_CSUM_OFFLOAD_EN;
681 err = hinic_set_rx_csum_offload(nic_dev->hwdev,
682 HINIC_RX_CSUM_OFFLOAD_EN);
684 goto rx_csum_ofl_err;
690 hinic_destroy_num_qps(nic_dev);
695 void hinic_rx_remove_configure(struct rte_eth_dev *dev)
697 struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
699 if (nic_dev->flags & ETH_MQ_RX_RSS_FLAG) {
700 hinic_rss_deinit(nic_dev);
701 hinic_destroy_num_qps(nic_dev);
705 void hinic_free_all_rx_mbufs(struct hinic_rxq *rxq)
707 struct hinic_nic_dev *nic_dev = rxq->nic_dev;
708 struct hinic_rx_info *rx_info;
710 hinic_get_rq_free_wqebbs(nic_dev->hwdev, rxq->q_id) + 1;
711 volatile struct hinic_rq_cqe *rx_cqe;
714 while (free_wqebbs++ < rxq->q_depth) {
715 ci = hinic_get_rq_local_ci(nic_dev->hwdev, rxq->q_id);
717 rx_cqe = &rxq->rx_cqe[ci];
722 rx_info = &rxq->rx_info[ci];
723 rte_pktmbuf_free(rx_info->mbuf);
724 rx_info->mbuf = NULL;
726 hinic_update_rq_local_ci(nic_dev->hwdev, rxq->q_id, 1);
730 static inline void hinic_rq_cqe_be_to_cpu32(void *dst_le32,
731 volatile void *src_be32)
733 #if defined(__X86_64_SSE__)
734 volatile __m128i *wqe_be = (volatile __m128i *)src_be32;
735 __m128i *wqe_le = (__m128i *)dst_le32;
736 __m128i shuf_mask = _mm_set_epi8(12, 13, 14, 15, 8, 9, 10,
737 11, 4, 5, 6, 7, 0, 1, 2, 3);
739 /* l2nic just use first 128 bits */
740 wqe_le[0] = _mm_shuffle_epi8(wqe_be[0], shuf_mask);
741 #elif defined(__ARM64_NEON__)
742 volatile uint8x16_t *wqe_be = (volatile uint8x16_t *)src_be32;
743 uint8x16_t *wqe_le = (uint8x16_t *)dst_le32;
744 const uint8x16_t shuf_mask = {3, 2, 1, 0, 7, 6, 5, 4, 11, 10,
745 9, 8, 15, 14, 13, 12};
747 /* l2nic just use first 128 bits */
748 wqe_le[0] = vqtbl1q_u8(wqe_be[0], shuf_mask);
751 volatile u32 *wqe_be = (volatile u32 *)src_be32;
752 u32 *wqe_le = (u32 *)dst_le32;
754 #define HINIC_L2NIC_RQ_CQE_USED 4 /* 4Bytes unit */
756 for (i = 0; i < HINIC_L2NIC_RQ_CQE_USED; i++) {
757 *wqe_le = rte_be_to_cpu_32(*wqe_be);
764 static inline uint64_t hinic_rx_rss_hash(uint32_t offload_type,
765 uint32_t cqe_hass_val,
770 rss_type = HINIC_GET_RSS_TYPES(offload_type);
771 if (likely(rss_type != 0)) {
772 *rss_hash = cqe_hass_val;
773 return PKT_RX_RSS_HASH;
779 static inline uint64_t hinic_rx_csum(uint32_t status, struct hinic_rxq *rxq)
781 uint32_t checksum_err;
783 struct hinic_nic_dev *nic_dev = rxq->nic_dev;
785 if (unlikely(!(nic_dev->rx_csum_en & HINIC_RX_CSUM_OFFLOAD_EN)))
786 return PKT_RX_IP_CKSUM_UNKNOWN;
788 /* most case checksum is ok */
789 checksum_err = HINIC_GET_RX_CSUM_ERR(status);
790 if (likely(checksum_err == 0))
791 return (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD);
793 /* If BYPASS bit set, all other status indications should be ignored */
794 if (unlikely(HINIC_CSUM_ERR_BYPASSED(checksum_err)))
795 return PKT_RX_IP_CKSUM_UNKNOWN;
799 /* IP checksum error */
800 if (HINIC_CSUM_ERR_IP(checksum_err))
801 flags |= PKT_RX_IP_CKSUM_BAD;
803 flags |= PKT_RX_IP_CKSUM_GOOD;
805 /* L4 checksum error */
806 if (HINIC_CSUM_ERR_L4(checksum_err))
807 flags |= PKT_RX_L4_CKSUM_BAD;
809 flags |= PKT_RX_L4_CKSUM_GOOD;
811 if (unlikely(HINIC_CSUM_ERR_OTHER(checksum_err)))
812 flags = PKT_RX_L4_CKSUM_NONE;
814 rxq->rxq_stats.errors++;
819 static inline uint64_t hinic_rx_vlan(uint32_t offload_type, uint32_t vlan_len,
824 vlan_tag = HINIC_GET_RX_VLAN_TAG(vlan_len);
825 if (!HINIC_GET_RX_VLAN_OFFLOAD_EN(offload_type) || 0 == vlan_tag) {
830 *vlan_tci = vlan_tag;
832 return PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED;
835 static inline u32 hinic_rx_alloc_mbuf_bulk(struct hinic_rxq *rxq,
836 struct rte_mbuf **mbufs,
842 rc = rte_pktmbuf_alloc_bulk(rxq->mb_pool, mbufs, exp_mbuf_cnt);
843 if (likely(rc == HINIC_OK)) {
844 avail_cnt = exp_mbuf_cnt;
847 rxq->rxq_stats.rx_nombuf += exp_mbuf_cnt;
853 static struct rte_mbuf *hinic_rx_alloc_mbuf(struct hinic_rxq *rxq,
854 dma_addr_t *dma_addr)
856 struct rte_mbuf *mbuf;
858 mbuf = rte_mbuf_raw_alloc(rxq->mb_pool);
862 *dma_addr = rte_mbuf_data_iova_default(mbuf);
867 static inline void hinic_rearm_rxq_mbuf(struct hinic_rxq *rxq)
870 u32 i, free_wqebbs, rearm_wqebbs, exp_wqebbs;
872 struct hinic_rq_wqe *rq_wqe;
873 struct rte_mbuf **rearm_mbufs;
875 /* check free wqebb fo rearm */
876 free_wqebbs = HINIC_GET_RQ_FREE_WQEBBS(rxq);
877 if (unlikely(free_wqebbs < rxq->rx_free_thresh))
880 /* get rearm mbuf array */
881 pi = HINIC_GET_RQ_LOCAL_PI(rxq);
882 rearm_mbufs = (struct rte_mbuf **)(&rxq->rx_info[pi]);
884 /* check rxq free wqebbs turn around */
885 exp_wqebbs = rxq->q_depth - pi;
886 if (free_wqebbs < exp_wqebbs)
887 exp_wqebbs = free_wqebbs;
889 /* alloc mbuf in bulk */
890 rearm_wqebbs = hinic_rx_alloc_mbuf_bulk(rxq, rearm_mbufs, exp_wqebbs);
891 if (unlikely(rearm_wqebbs == 0))
895 rq_wqe = WQ_WQE_ADDR(rxq->wq, (u32)pi);
896 for (i = 0; i < rearm_wqebbs; i++) {
897 dma_addr = rte_mbuf_data_iova_default(rearm_mbufs[i]);
898 rq_wqe->buf_desc.addr_high =
899 cpu_to_be32(upper_32_bits(dma_addr));
900 rq_wqe->buf_desc.addr_low =
901 cpu_to_be32(lower_32_bits(dma_addr));
904 rxq->wq->prod_idx += rearm_wqebbs;
905 rxq->wq->delta -= rearm_wqebbs;
907 /* update rq hw_pi */
909 HINIC_UPDATE_RQ_HW_PI(rxq, pi + rearm_wqebbs);
912 void hinic_rx_alloc_pkts(struct hinic_rxq *rxq)
914 struct hinic_nic_dev *nic_dev = rxq->nic_dev;
915 struct hinic_rq_wqe *rq_wqe;
916 struct hinic_rx_info *rx_info;
922 free_wqebbs = HINIC_GET_RQ_FREE_WQEBBS(rxq);
923 for (i = 0; i < free_wqebbs; i++) {
924 mb = hinic_rx_alloc_mbuf(rxq, &dma_addr);
926 rxq->rxq_stats.rx_nombuf++;
930 rq_wqe = hinic_get_rq_wqe(nic_dev->hwdev, rxq->q_id, &pi);
931 if (unlikely(!rq_wqe)) {
932 rte_pktmbuf_free(mb);
936 /* fill buffer address only */
937 rq_wqe->buf_desc.addr_high =
938 cpu_to_be32(upper_32_bits(dma_addr));
939 rq_wqe->buf_desc.addr_low =
940 cpu_to_be32(lower_32_bits(dma_addr));
942 rx_info = &rxq->rx_info[pi];
948 HINIC_UPDATE_RQ_HW_PI(rxq, pi + 1);
952 u16 hinic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, u16 nb_pkts)
954 struct rte_mbuf *rxm;
955 struct hinic_rxq *rxq = rx_queue;
956 struct hinic_rx_info *rx_info;
957 volatile struct hinic_rq_cqe *rx_cqe;
958 u16 rx_buf_len, pkts = 0;
959 u16 sw_ci, ci_mask, wqebb_cnt = 0;
960 u32 pkt_len, status, vlan_len;
962 struct hinic_rq_cqe cqe;
963 u32 offload_type, rss_hash;
965 rx_buf_len = rxq->buf_len;
967 /* 1. get polling start ci */
968 ci_mask = HINIC_GET_RQ_WQE_MASK(rxq);
969 sw_ci = HINIC_GET_RQ_LOCAL_CI(rxq);
971 while (pkts < nb_pkts) {
972 /* 2. current ci is done */
973 rx_cqe = &rxq->rx_cqe[sw_ci];
974 status = __atomic_load_n(&rx_cqe->status, __ATOMIC_ACQUIRE);
975 if (!HINIC_GET_RX_DONE_BE(status))
978 /* convert cqe and get packet length */
979 hinic_rq_cqe_be_to_cpu32(&cqe, (volatile void *)rx_cqe);
980 vlan_len = cqe.vlan_len;
982 rx_info = &rxq->rx_info[sw_ci];
985 /* 3. next ci point and prefetch */
989 /* prefetch next mbuf first 64B */
990 rte_prefetch0(rxq->rx_info[sw_ci].mbuf);
992 /* 4. jumbo frame process */
993 pkt_len = HINIC_GET_RX_PKT_LEN(vlan_len);
994 if (likely(pkt_len <= rx_buf_len)) {
995 rxm->data_len = pkt_len;
996 rxm->pkt_len = pkt_len;
999 rxm->data_len = rx_buf_len;
1000 rxm->pkt_len = rx_buf_len;
1002 /* if receive jumbo, updating ci will be done by
1003 * hinic_recv_jumbo_pkt function.
1005 HINIC_UPDATE_RQ_LOCAL_CI(rxq, wqebb_cnt + 1);
1007 hinic_recv_jumbo_pkt(rxq, rxm, pkt_len - rx_buf_len);
1008 sw_ci = HINIC_GET_RQ_LOCAL_CI(rxq);
1011 /* 5. vlan/checksum/rss/pkt_type/gro offload */
1012 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1013 rxm->port = rxq->port_id;
1014 offload_type = cqe.offload_type;
1017 rxm->ol_flags |= hinic_rx_vlan(offload_type, vlan_len,
1020 /* checksum offload */
1021 rxm->ol_flags |= hinic_rx_csum(cqe.status, rxq);
1023 /* rss hash offload */
1024 rss_hash = cqe.rss_hash;
1025 rxm->ol_flags |= hinic_rx_rss_hash(offload_type, rss_hash,
1028 /* 6. clear done bit */
1031 rx_bytes += pkt_len;
1032 rx_pkts[pkts++] = rxm;
1037 HINIC_UPDATE_RQ_LOCAL_CI(rxq, wqebb_cnt);
1039 /* do packet stats */
1040 rxq->rxq_stats.packets += pkts;
1041 rxq->rxq_stats.bytes += rx_bytes;
1043 rxq->rxq_stats.burst_pkts = pkts;
1045 /* 8. rearm mbuf to rxq */
1046 hinic_rearm_rxq_mbuf(rxq);