1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Huawei Technologies Co., Ltd
11 #include "base/hinic_compat.h"
12 #include "base/hinic_pmd_hwdev.h"
13 #include "base/hinic_pmd_wq.h"
14 #include "base/hinic_pmd_niccfg.h"
15 #include "base/hinic_pmd_nicio.h"
16 #include "hinic_pmd_ethdev.h"
17 #include "hinic_pmd_rx.h"
19 /* rxq wq operations */
20 #define HINIC_GET_RQ_WQE_MASK(rxq) \
23 #define HINIC_GET_RQ_LOCAL_CI(rxq) \
24 (((rxq)->wq->cons_idx) & HINIC_GET_RQ_WQE_MASK(rxq))
26 #define HINIC_GET_RQ_LOCAL_PI(rxq) \
27 (((rxq)->wq->prod_idx) & HINIC_GET_RQ_WQE_MASK(rxq))
29 #define HINIC_UPDATE_RQ_LOCAL_CI(rxq, wqebb_cnt) \
31 (rxq)->wq->cons_idx += (wqebb_cnt); \
32 (rxq)->wq->delta += (wqebb_cnt); \
35 #define HINIC_UPDATE_RQ_HW_PI(rxq, pi) \
36 (*((rxq)->pi_virt_addr) = \
37 cpu_to_be16((pi) & HINIC_GET_RQ_WQE_MASK(rxq)))
39 #define HINIC_GET_RQ_FREE_WQEBBS(rxq) ((rxq)->wq->delta - 1)
41 /* rxq cqe done and status bit */
42 #define HINIC_GET_RX_DONE_BE(status) \
45 #define HINIC_RX_CSUM_OFFLOAD_EN 0xFFF
47 #define RQ_CQE_SGE_VLAN_SHIFT 0
48 #define RQ_CQE_SGE_LEN_SHIFT 16
50 #define RQ_CQE_SGE_VLAN_MASK 0xFFFFU
51 #define RQ_CQE_SGE_LEN_MASK 0xFFFFU
53 #define RQ_CQE_SGE_GET(val, member) \
54 (((val) >> RQ_CQE_SGE_##member##_SHIFT) & RQ_CQE_SGE_##member##_MASK)
56 #define HINIC_GET_RX_VLAN_TAG(vlan_len) \
57 RQ_CQE_SGE_GET(vlan_len, VLAN)
59 #define HINIC_GET_RX_PKT_LEN(vlan_len) \
60 RQ_CQE_SGE_GET(vlan_len, LEN)
62 #define RQ_CQE_STATUS_CSUM_ERR_SHIFT 0
63 #define RQ_CQE_STATUS_NUM_LRO_SHIFT 16
64 #define RQ_CQE_STATUS_LRO_PUSH_SHIFT 25
65 #define RQ_CQE_STATUS_LRO_ENTER_SHIFT 26
66 #define RQ_CQE_STATUS_LRO_INTR_SHIFT 27
68 #define RQ_CQE_STATUS_BP_EN_SHIFT 30
69 #define RQ_CQE_STATUS_RXDONE_SHIFT 31
70 #define RQ_CQE_STATUS_FLUSH_SHIFT 28
72 #define RQ_CQE_STATUS_CSUM_ERR_MASK 0xFFFFU
73 #define RQ_CQE_STATUS_NUM_LRO_MASK 0xFFU
74 #define RQ_CQE_STATUS_LRO_PUSH_MASK 0X1U
75 #define RQ_CQE_STATUS_LRO_ENTER_MASK 0X1U
76 #define RQ_CQE_STATUS_LRO_INTR_MASK 0X1U
77 #define RQ_CQE_STATUS_BP_EN_MASK 0X1U
78 #define RQ_CQE_STATUS_RXDONE_MASK 0x1U
79 #define RQ_CQE_STATUS_FLUSH_MASK 0x1U
81 #define RQ_CQE_STATUS_GET(val, member) \
82 (((val) >> RQ_CQE_STATUS_##member##_SHIFT) & \
83 RQ_CQE_STATUS_##member##_MASK)
85 #define RQ_CQE_STATUS_CLEAR(val, member) \
86 ((val) & (~(RQ_CQE_STATUS_##member##_MASK << \
87 RQ_CQE_STATUS_##member##_SHIFT)))
89 #define HINIC_GET_RX_CSUM_ERR(status) \
90 RQ_CQE_STATUS_GET(status, CSUM_ERR)
92 #define HINIC_GET_RX_DONE(status) \
93 RQ_CQE_STATUS_GET(status, RXDONE)
95 #define HINIC_GET_RX_FLUSH(status) \
96 RQ_CQE_STATUS_GET(status, FLUSH)
98 #define HINIC_GET_RX_BP_EN(status) \
99 RQ_CQE_STATUS_GET(status, BP_EN)
101 #define HINIC_GET_RX_NUM_LRO(status) \
102 RQ_CQE_STATUS_GET(status, NUM_LRO)
105 #define RQ_CTRL_BUFDESC_SECT_LEN_SHIFT 0
106 #define RQ_CTRL_COMPLETE_FORMAT_SHIFT 15
107 #define RQ_CTRL_COMPLETE_LEN_SHIFT 27
108 #define RQ_CTRL_LEN_SHIFT 29
110 #define RQ_CTRL_BUFDESC_SECT_LEN_MASK 0xFFU
111 #define RQ_CTRL_COMPLETE_FORMAT_MASK 0x1U
112 #define RQ_CTRL_COMPLETE_LEN_MASK 0x3U
113 #define RQ_CTRL_LEN_MASK 0x3U
115 #define RQ_CTRL_SET(val, member) \
116 (((val) & RQ_CTRL_##member##_MASK) << RQ_CTRL_##member##_SHIFT)
118 #define RQ_CTRL_GET(val, member) \
119 (((val) >> RQ_CTRL_##member##_SHIFT) & RQ_CTRL_##member##_MASK)
121 #define RQ_CTRL_CLEAR(val, member) \
122 ((val) & (~(RQ_CTRL_##member##_MASK << RQ_CTRL_##member##_SHIFT)))
124 #define RQ_CQE_PKT_NUM_SHIFT 1
125 #define RQ_CQE_PKT_FIRST_LEN_SHIFT 19
126 #define RQ_CQE_PKT_LAST_LEN_SHIFT 6
127 #define RQ_CQE_SUPER_CQE_EN_SHIFT 0
129 #define RQ_CQE_PKT_FIRST_LEN_MASK 0x1FFFU
130 #define RQ_CQE_PKT_LAST_LEN_MASK 0x1FFFU
131 #define RQ_CQE_PKT_NUM_MASK 0x1FU
132 #define RQ_CQE_SUPER_CQE_EN_MASK 0x1
134 #define RQ_CQE_PKT_NUM_GET(val, member) \
135 (((val) >> RQ_CQE_PKT_##member##_SHIFT) & RQ_CQE_PKT_##member##_MASK)
137 #define HINIC_GET_RQ_CQE_PKT_NUM(pkt_info) RQ_CQE_PKT_NUM_GET(pkt_info, NUM)
139 #define RQ_CQE_SUPER_CQE_EN_GET(val, member) \
140 (((val) >> RQ_CQE_##member##_SHIFT) & RQ_CQE_##member##_MASK)
142 #define HINIC_GET_SUPER_CQE_EN(pkt_info) \
143 RQ_CQE_SUPER_CQE_EN_GET(pkt_info, SUPER_CQE_EN)
145 #define RQ_CQE_OFFOLAD_TYPE_VLAN_EN_SHIFT 21
146 #define RQ_CQE_OFFOLAD_TYPE_VLAN_EN_MASK 0x1U
148 #define RQ_CQE_OFFOLAD_TYPE_PKT_TYPE_SHIFT 0
149 #define RQ_CQE_OFFOLAD_TYPE_PKT_TYPE_MASK 0xFFFU
151 #define RQ_CQE_OFFOLAD_TYPE_PKT_UMBCAST_SHIFT 19
152 #define RQ_CQE_OFFOLAD_TYPE_PKT_UMBCAST_MASK 0x3U
154 #define RQ_CQE_OFFOLAD_TYPE_RSS_TYPE_SHIFT 24
155 #define RQ_CQE_OFFOLAD_TYPE_RSS_TYPE_MASK 0xFFU
157 #define RQ_CQE_OFFOLAD_TYPE_GET(val, member) (((val) >> \
158 RQ_CQE_OFFOLAD_TYPE_##member##_SHIFT) & \
159 RQ_CQE_OFFOLAD_TYPE_##member##_MASK)
161 #define HINIC_GET_RX_VLAN_OFFLOAD_EN(offload_type) \
162 RQ_CQE_OFFOLAD_TYPE_GET(offload_type, VLAN_EN)
164 #define HINIC_GET_RSS_TYPES(offload_type) \
165 RQ_CQE_OFFOLAD_TYPE_GET(offload_type, RSS_TYPE)
167 #define HINIC_GET_RX_PKT_TYPE(offload_type) \
168 RQ_CQE_OFFOLAD_TYPE_GET(offload_type, PKT_TYPE)
170 #define HINIC_GET_RX_PKT_UMBCAST(offload_type) \
171 RQ_CQE_OFFOLAD_TYPE_GET(offload_type, PKT_UMBCAST)
173 #define RQ_CQE_STATUS_CSUM_BYPASS_VAL 0x80U
174 #define RQ_CQE_STATUS_CSUM_ERR_IP_MASK 0x39U
175 #define RQ_CQE_STATUS_CSUM_ERR_L4_MASK 0x46U
176 #define RQ_CQE_STATUS_CSUM_ERR_OTHER 0x100U
178 #define HINIC_CSUM_ERR_BYPASSED(csum_err) \
179 ((csum_err) == RQ_CQE_STATUS_CSUM_BYPASS_VAL)
181 #define HINIC_CSUM_ERR_IP(csum_err) \
182 ((csum_err) & RQ_CQE_STATUS_CSUM_ERR_IP_MASK)
184 #define HINIC_CSUM_ERR_L4(csum_err) \
185 ((csum_err) & RQ_CQE_STATUS_CSUM_ERR_L4_MASK)
187 #define HINIC_CSUM_ERR_OTHER(csum_err) \
188 ((csum_err) == RQ_CQE_STATUS_CSUM_ERR_OTHER)
191 void hinic_get_func_rx_buf_size(struct hinic_nic_dev *nic_dev)
193 struct hinic_rxq *rxq;
197 for (q_id = 0; q_id < nic_dev->num_rq; q_id++) {
198 rxq = nic_dev->rxqs[q_id];
204 buf_size = rxq->buf_len;
206 buf_size = buf_size > rxq->buf_len ? rxq->buf_len : buf_size;
209 nic_dev->hwdev->nic_io->rq_buf_size = buf_size;
212 int hinic_create_rq(struct hinic_hwdev *hwdev, u16 q_id,
213 u16 rq_depth, unsigned int socket_id)
216 struct hinic_nic_io *nic_io = hwdev->nic_io;
217 struct hinic_qp *qp = &nic_io->qps[q_id];
218 struct hinic_rq *rq = &qp->rq;
220 /* in case of hardware still generate interrupt, do not use msix 0 */
221 rq->msix_entry_idx = 1;
223 rq->rq_depth = rq_depth;
224 nic_io->rq_depth = rq_depth;
226 err = hinic_wq_allocate(hwdev, &nic_io->rq_wq[q_id],
227 HINIC_RQ_WQEBB_SHIFT, nic_io->rq_depth, socket_id);
229 PMD_DRV_LOG(ERR, "Failed to allocate WQ for RQ");
232 rq->wq = &nic_io->rq_wq[q_id];
234 rq->pi_virt_addr = (volatile u16 *)dma_zalloc_coherent(hwdev,
235 HINIC_PAGE_SIZE, &rq->pi_dma_addr, socket_id);
236 if (!rq->pi_virt_addr) {
237 PMD_DRV_LOG(ERR, "Failed to allocate rq pi virt addr");
239 goto rq_pi_alloc_err;
245 hinic_wq_free(hwdev, &nic_io->rq_wq[q_id]);
250 void hinic_destroy_rq(struct hinic_hwdev *hwdev, u16 q_id)
252 struct hinic_nic_io *nic_io = hwdev->nic_io;
253 struct hinic_qp *qp = &nic_io->qps[q_id];
254 struct hinic_rq *rq = &qp->rq;
256 if (qp->rq.wq == NULL)
259 dma_free_coherent_volatile(hwdev, HINIC_PAGE_SIZE,
260 (volatile void *)rq->pi_virt_addr,
262 hinic_wq_free(nic_io->hwdev, qp->rq.wq);
267 hinic_prepare_rq_wqe(void *wqe, __rte_unused u16 pi, dma_addr_t buf_addr,
270 struct hinic_rq_wqe *rq_wqe = wqe;
271 struct hinic_rq_ctrl *ctrl = &rq_wqe->ctrl;
272 struct hinic_rq_cqe_sect *cqe_sect = &rq_wqe->cqe_sect;
273 struct hinic_rq_bufdesc *buf_desc = &rq_wqe->buf_desc;
274 u32 rq_ceq_len = sizeof(struct hinic_rq_cqe);
277 RQ_CTRL_SET(SIZE_8BYTES(sizeof(*ctrl)), LEN) |
278 RQ_CTRL_SET(SIZE_8BYTES(sizeof(*cqe_sect)), COMPLETE_LEN) |
279 RQ_CTRL_SET(SIZE_8BYTES(sizeof(*buf_desc)), BUFDESC_SECT_LEN) |
280 RQ_CTRL_SET(RQ_COMPLETE_SGE, COMPLETE_FORMAT);
282 hinic_set_sge(&cqe_sect->sge, cqe_dma, rq_ceq_len);
284 buf_desc->addr_high = upper_32_bits(buf_addr);
285 buf_desc->addr_low = lower_32_bits(buf_addr);
288 void hinic_rxq_get_stats(struct hinic_rxq *rxq, struct hinic_rxq_stats *stats)
293 memcpy(stats, &rxq->rxq_stats, sizeof(rxq->rxq_stats));
296 void hinic_rxq_stats_reset(struct hinic_rxq *rxq)
298 struct hinic_rxq_stats *rxq_stats;
303 rxq_stats = &rxq->rxq_stats;
304 memset(rxq_stats, 0, sizeof(*rxq_stats));
307 static int hinic_rx_alloc_cqe(struct hinic_rxq *rxq, unsigned int socket_id)
311 cqe_mem_size = sizeof(struct hinic_rq_cqe) * rxq->q_depth;
312 rxq->cqe_start_vaddr = dma_zalloc_coherent(rxq->nic_dev->hwdev,
313 cqe_mem_size, &rxq->cqe_start_paddr, socket_id);
314 if (!rxq->cqe_start_vaddr) {
315 PMD_DRV_LOG(ERR, "Allocate cqe dma memory failed");
319 rxq->rx_cqe = (struct hinic_rq_cqe *)rxq->cqe_start_vaddr;
324 static void hinic_rx_free_cqe(struct hinic_rxq *rxq)
328 cqe_mem_size = sizeof(struct hinic_rq_cqe) * rxq->q_depth;
329 dma_free_coherent(rxq->nic_dev->hwdev, cqe_mem_size,
330 rxq->cqe_start_vaddr, rxq->cqe_start_paddr);
331 rxq->cqe_start_vaddr = NULL;
334 static int hinic_rx_fill_wqe(struct hinic_rxq *rxq)
336 struct hinic_nic_dev *nic_dev = rxq->nic_dev;
337 struct hinic_rq_wqe *rq_wqe;
338 dma_addr_t buf_dma_addr, cqe_dma_addr;
343 cqe_dma_addr = rxq->cqe_start_paddr;
344 for (i = 0; i < rxq->q_depth; i++) {
345 rq_wqe = hinic_get_rq_wqe(nic_dev->hwdev, rxq->q_id, &pi);
347 PMD_DRV_LOG(ERR, "Get rq wqe failed");
351 hinic_prepare_rq_wqe(rq_wqe, pi, buf_dma_addr, cqe_dma_addr);
352 cqe_dma_addr += sizeof(struct hinic_rq_cqe);
354 hinic_cpu_to_be32(rq_wqe, sizeof(struct hinic_rq_wqe));
357 hinic_return_rq_wqe(nic_dev->hwdev, rxq->q_id, i);
362 /* alloc cqe and prepare rqe */
363 int hinic_setup_rx_resources(struct hinic_rxq *rxq)
368 rx_info_sz = rxq->q_depth * sizeof(*rxq->rx_info);
369 rxq->rx_info = rte_zmalloc_socket("rx_info", rx_info_sz,
370 RTE_CACHE_LINE_SIZE, rxq->socket_id);
374 err = hinic_rx_alloc_cqe(rxq, rxq->socket_id);
376 PMD_DRV_LOG(ERR, "Allocate rx cqe failed");
380 pkts = hinic_rx_fill_wqe(rxq);
381 if (pkts != rxq->q_depth) {
382 PMD_DRV_LOG(ERR, "Fill rx wqe failed");
390 hinic_rx_free_cqe(rxq);
393 rte_free(rxq->rx_info);
399 void hinic_free_rx_resources(struct hinic_rxq *rxq)
401 if (rxq->rx_info == NULL)
404 hinic_rx_free_cqe(rxq);
405 rte_free(rxq->rx_info);
409 void hinic_free_all_rx_resources(struct rte_eth_dev *eth_dev)
412 struct hinic_nic_dev *nic_dev =
413 HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
415 for (q_id = 0; q_id < nic_dev->num_rq; q_id++) {
416 eth_dev->data->rx_queues[q_id] = NULL;
418 if (nic_dev->rxqs[q_id] == NULL)
421 hinic_free_all_rx_mbufs(nic_dev->rxqs[q_id]);
422 hinic_free_rx_resources(nic_dev->rxqs[q_id]);
423 kfree(nic_dev->rxqs[q_id]);
424 nic_dev->rxqs[q_id] = NULL;
428 void hinic_free_all_rx_mbuf(struct rte_eth_dev *eth_dev)
430 struct hinic_nic_dev *nic_dev =
431 HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
434 for (q_id = 0; q_id < nic_dev->num_rq; q_id++)
435 hinic_free_all_rx_mbufs(nic_dev->rxqs[q_id]);
438 static void hinic_recv_jumbo_pkt(struct hinic_rxq *rxq,
439 struct rte_mbuf *head_mbuf,
442 struct hinic_nic_dev *nic_dev = rxq->nic_dev;
443 struct rte_mbuf *cur_mbuf, *rxm = NULL;
444 struct hinic_rx_info *rx_info;
445 u16 sw_ci, rx_buf_len = rxq->buf_len;
448 while (remain_pkt_len > 0) {
449 sw_ci = hinic_get_rq_local_ci(nic_dev->hwdev, rxq->q_id);
450 rx_info = &rxq->rx_info[sw_ci];
452 hinic_update_rq_local_ci(nic_dev->hwdev, rxq->q_id, 1);
454 pkt_len = remain_pkt_len > rx_buf_len ?
455 rx_buf_len : remain_pkt_len;
456 remain_pkt_len -= pkt_len;
458 cur_mbuf = rx_info->mbuf;
459 cur_mbuf->data_len = (u16)pkt_len;
460 cur_mbuf->next = NULL;
462 head_mbuf->pkt_len += cur_mbuf->data_len;
463 head_mbuf->nb_segs++;
466 head_mbuf->next = cur_mbuf;
468 rxm->next = cur_mbuf;
474 static void hinic_rss_deinit(struct hinic_nic_dev *nic_dev)
476 u8 prio_tc[HINIC_DCB_UP_MAX] = {0};
477 (void)hinic_rss_cfg(nic_dev->hwdev, 0,
478 nic_dev->rss_tmpl_idx, 0, prio_tc);
481 static int hinic_rss_key_init(struct hinic_nic_dev *nic_dev,
482 struct rte_eth_rss_conf *rss_conf)
484 u8 default_rss_key[HINIC_RSS_KEY_SIZE] = {
485 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2,
486 0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0,
487 0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4,
488 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c,
489 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa};
490 u8 hashkey[HINIC_RSS_KEY_SIZE] = {0};
491 u8 tmpl_idx = nic_dev->rss_tmpl_idx;
493 if (rss_conf->rss_key == NULL)
494 memcpy(hashkey, default_rss_key, HINIC_RSS_KEY_SIZE);
496 memcpy(hashkey, rss_conf->rss_key, rss_conf->rss_key_len);
498 return hinic_rss_set_template_tbl(nic_dev->hwdev, tmpl_idx, hashkey);
501 static void hinic_fill_rss_type(struct nic_rss_type *rss_type,
502 struct rte_eth_rss_conf *rss_conf)
504 u64 rss_hf = rss_conf->rss_hf;
506 rss_type->ipv4 = (rss_hf & (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4)) ? 1 : 0;
507 rss_type->tcp_ipv4 = (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) ? 1 : 0;
508 rss_type->ipv6 = (rss_hf & (ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6)) ? 1 : 0;
509 rss_type->ipv6_ext = (rss_hf & ETH_RSS_IPV6_EX) ? 1 : 0;
510 rss_type->tcp_ipv6 = (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP) ? 1 : 0;
511 rss_type->tcp_ipv6_ext = (rss_hf & ETH_RSS_IPV6_TCP_EX) ? 1 : 0;
512 rss_type->udp_ipv4 = (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) ? 1 : 0;
513 rss_type->udp_ipv6 = (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP) ? 1 : 0;
516 static void hinic_fillout_indir_tbl(struct hinic_nic_dev *nic_dev, u32 *indir)
518 u8 rss_queue_count = nic_dev->num_rss;
521 if (rss_queue_count == 0) {
522 /* delete q_id from indir tbl */
523 for (i = 0; i < HINIC_RSS_INDIR_SIZE; i++)
524 indir[i] = 0xFF; /* Invalid value in indir tbl */
526 while (i < HINIC_RSS_INDIR_SIZE)
527 for (j = 0; (j < rss_queue_count) &&
528 (i < HINIC_RSS_INDIR_SIZE); j++)
529 indir[i++] = nic_dev->rx_queue_list[j];
533 static int hinic_rss_init(struct hinic_nic_dev *nic_dev,
534 __rte_unused u8 *rq2iq_map,
535 struct rte_eth_rss_conf *rss_conf)
537 u32 indir_tbl[HINIC_RSS_INDIR_SIZE] = {0};
538 struct nic_rss_type rss_type = {0};
539 u8 prio_tc[HINIC_DCB_UP_MAX] = {0};
540 u8 tmpl_idx = 0xFF, num_tc = 0;
543 tmpl_idx = nic_dev->rss_tmpl_idx;
545 err = hinic_rss_key_init(nic_dev, rss_conf);
549 if (!nic_dev->rss_indir_flag) {
550 hinic_fillout_indir_tbl(nic_dev, indir_tbl);
551 err = hinic_rss_set_indir_tbl(nic_dev->hwdev, tmpl_idx,
557 hinic_fill_rss_type(&rss_type, rss_conf);
558 err = hinic_set_rss_type(nic_dev->hwdev, tmpl_idx, rss_type);
562 err = hinic_rss_set_hash_engine(nic_dev->hwdev, tmpl_idx,
563 HINIC_RSS_HASH_ENGINE_TYPE_TOEP);
567 return hinic_rss_cfg(nic_dev->hwdev, 1, tmpl_idx, num_tc, prio_tc);
571 hinic_add_rq_to_rx_queue_list(struct hinic_nic_dev *nic_dev, u16 queue_id)
573 u8 rss_queue_count = nic_dev->num_rss;
575 RTE_ASSERT(rss_queue_count <= (RTE_DIM(nic_dev->rx_queue_list) - 1));
577 nic_dev->rx_queue_list[rss_queue_count] = queue_id;
582 * hinic_setup_num_qps - determine num_qps from rss_tmpl_id
583 * @nic_dev: pointer to the private ethernet device
584 * Return: 0 on Success, error code otherwise.
586 static int hinic_setup_num_qps(struct hinic_nic_dev *nic_dev)
590 if (!(nic_dev->flags & ETH_MQ_RX_RSS_FLAG)) {
591 nic_dev->flags &= ~ETH_MQ_RX_RSS_FLAG;
592 nic_dev->num_rss = 0;
593 if (nic_dev->num_rq > 1) {
594 /* get rss template id */
595 err = hinic_rss_template_alloc(nic_dev->hwdev,
596 &nic_dev->rss_tmpl_idx);
598 PMD_DRV_LOG(WARNING, "Alloc rss template failed");
601 nic_dev->flags |= ETH_MQ_RX_RSS_FLAG;
602 for (i = 0; i < nic_dev->num_rq; i++)
603 hinic_add_rq_to_rx_queue_list(nic_dev, i);
610 static void hinic_destroy_num_qps(struct hinic_nic_dev *nic_dev)
612 if (nic_dev->flags & ETH_MQ_RX_RSS_FLAG) {
613 if (hinic_rss_template_free(nic_dev->hwdev,
614 nic_dev->rss_tmpl_idx))
615 PMD_DRV_LOG(WARNING, "Free rss template failed");
617 nic_dev->flags &= ~ETH_MQ_RX_RSS_FLAG;
621 static int hinic_config_mq_rx_rss(struct hinic_nic_dev *nic_dev, bool on)
626 ret = hinic_setup_num_qps(nic_dev);
628 PMD_DRV_LOG(ERR, "Setup num_qps failed");
630 hinic_destroy_num_qps(nic_dev);
636 int hinic_config_mq_mode(struct rte_eth_dev *dev, bool on)
638 struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
639 struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
642 switch (dev_conf->rxmode.mq_mode) {
644 ret = hinic_config_mq_rx_rss(nic_dev, on);
653 int hinic_rx_configure(struct rte_eth_dev *dev)
655 struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
656 struct rte_eth_rss_conf rss_conf =
657 dev->data->dev_conf.rx_adv_conf.rss_conf;
660 if (nic_dev->flags & ETH_MQ_RX_RSS_FLAG) {
661 if (rss_conf.rss_hf == 0) {
662 rss_conf.rss_hf = HINIC_RSS_OFFLOAD_ALL;
663 } else if ((rss_conf.rss_hf & HINIC_RSS_OFFLOAD_ALL) == 0) {
664 PMD_DRV_LOG(ERR, "Do not support rss offload all");
668 err = hinic_rss_init(nic_dev, NULL, &rss_conf);
670 PMD_DRV_LOG(ERR, "Init rss failed");
675 /* Enable both L3/L4 rx checksum offload */
676 if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_CHECKSUM)
677 nic_dev->rx_csum_en = HINIC_RX_CSUM_OFFLOAD_EN;
679 err = hinic_set_rx_csum_offload(nic_dev->hwdev,
680 HINIC_RX_CSUM_OFFLOAD_EN);
682 goto rx_csum_ofl_err;
688 hinic_destroy_num_qps(nic_dev);
693 void hinic_rx_remove_configure(struct rte_eth_dev *dev)
695 struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
697 if (nic_dev->flags & ETH_MQ_RX_RSS_FLAG) {
698 hinic_rss_deinit(nic_dev);
699 hinic_destroy_num_qps(nic_dev);
703 void hinic_free_all_rx_mbufs(struct hinic_rxq *rxq)
705 struct hinic_nic_dev *nic_dev = rxq->nic_dev;
706 struct hinic_rx_info *rx_info;
708 hinic_get_rq_free_wqebbs(nic_dev->hwdev, rxq->q_id) + 1;
709 volatile struct hinic_rq_cqe *rx_cqe;
712 while (free_wqebbs++ < rxq->q_depth) {
713 ci = hinic_get_rq_local_ci(nic_dev->hwdev, rxq->q_id);
715 rx_cqe = &rxq->rx_cqe[ci];
720 rx_info = &rxq->rx_info[ci];
721 rte_pktmbuf_free(rx_info->mbuf);
722 rx_info->mbuf = NULL;
724 hinic_update_rq_local_ci(nic_dev->hwdev, rxq->q_id, 1);
728 static inline void hinic_rq_cqe_be_to_cpu32(void *dst_le32,
729 volatile void *src_be32)
731 #if defined(__X86_64_SSE__)
732 volatile __m128i *wqe_be = (volatile __m128i *)src_be32;
733 __m128i *wqe_le = (__m128i *)dst_le32;
734 __m128i shuf_mask = _mm_set_epi8(12, 13, 14, 15, 8, 9, 10,
735 11, 4, 5, 6, 7, 0, 1, 2, 3);
737 /* l2nic just use first 128 bits */
738 wqe_le[0] = _mm_shuffle_epi8(wqe_be[0], shuf_mask);
739 #elif defined(__ARM64_NEON__)
740 volatile uint8x16_t *wqe_be = (volatile uint8x16_t *)src_be32;
741 uint8x16_t *wqe_le = (uint8x16_t *)dst_le32;
742 const uint8x16_t shuf_mask = {3, 2, 1, 0, 7, 6, 5, 4, 11, 10,
743 9, 8, 15, 14, 13, 12};
745 /* l2nic just use first 128 bits */
746 wqe_le[0] = vqtbl1q_u8(wqe_be[0], shuf_mask);
749 volatile u32 *wqe_be = (volatile u32 *)src_be32;
750 u32 *wqe_le = (u32 *)dst_le32;
752 #define HINIC_L2NIC_RQ_CQE_USED 4 /* 4Bytes unit */
754 for (i = 0; i < HINIC_L2NIC_RQ_CQE_USED; i++) {
755 *wqe_le = rte_be_to_cpu_32(*wqe_be);
762 static inline uint64_t hinic_rx_rss_hash(uint32_t offload_type,
763 uint32_t cqe_hass_val,
768 rss_type = HINIC_GET_RSS_TYPES(offload_type);
769 if (likely(rss_type != 0)) {
770 *rss_hash = cqe_hass_val;
771 return PKT_RX_RSS_HASH;
777 static inline uint64_t hinic_rx_csum(uint32_t status, struct hinic_rxq *rxq)
779 uint32_t checksum_err;
781 struct hinic_nic_dev *nic_dev = rxq->nic_dev;
783 if (unlikely(!(nic_dev->rx_csum_en & HINIC_RX_CSUM_OFFLOAD_EN)))
784 return PKT_RX_IP_CKSUM_UNKNOWN;
786 /* most case checksum is ok */
787 checksum_err = HINIC_GET_RX_CSUM_ERR(status);
788 if (likely(checksum_err == 0))
789 return (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD);
791 /* If BYPASS bit set, all other status indications should be ignored */
792 if (unlikely(HINIC_CSUM_ERR_BYPASSED(checksum_err)))
793 return PKT_RX_IP_CKSUM_UNKNOWN;
797 /* IP checksum error */
798 if (HINIC_CSUM_ERR_IP(checksum_err))
799 flags |= PKT_RX_IP_CKSUM_BAD;
801 flags |= PKT_RX_IP_CKSUM_GOOD;
803 /* L4 checksum error */
804 if (HINIC_CSUM_ERR_L4(checksum_err))
805 flags |= PKT_RX_L4_CKSUM_BAD;
807 flags |= PKT_RX_L4_CKSUM_GOOD;
809 if (unlikely(HINIC_CSUM_ERR_OTHER(checksum_err)))
810 flags = PKT_RX_L4_CKSUM_NONE;
812 rxq->rxq_stats.errors++;
817 static inline uint64_t hinic_rx_vlan(uint32_t offload_type, uint32_t vlan_len,
822 vlan_tag = HINIC_GET_RX_VLAN_TAG(vlan_len);
823 if (!HINIC_GET_RX_VLAN_OFFLOAD_EN(offload_type) || 0 == vlan_tag) {
828 *vlan_tci = vlan_tag;
830 return PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED;
833 static inline u32 hinic_rx_alloc_mbuf_bulk(struct hinic_rxq *rxq,
834 struct rte_mbuf **mbufs,
840 rc = rte_pktmbuf_alloc_bulk(rxq->mb_pool, mbufs, exp_mbuf_cnt);
841 if (likely(rc == HINIC_OK)) {
842 avail_cnt = exp_mbuf_cnt;
845 rxq->rxq_stats.rx_nombuf += exp_mbuf_cnt;
851 static struct rte_mbuf *hinic_rx_alloc_mbuf(struct hinic_rxq *rxq,
852 dma_addr_t *dma_addr)
854 struct rte_mbuf *mbuf = NULL;
857 rc = rte_pktmbuf_alloc_bulk(rxq->mb_pool, &mbuf, 1);
858 if (unlikely(rc != HINIC_OK))
861 *dma_addr = rte_mbuf_data_iova_default(mbuf);
866 static inline void hinic_rearm_rxq_mbuf(struct hinic_rxq *rxq)
869 u32 i, free_wqebbs, rearm_wqebbs, exp_wqebbs;
871 struct hinic_rq_wqe *rq_wqe;
872 struct rte_mbuf **rearm_mbufs;
874 /* check free wqebb fo rearm */
875 free_wqebbs = HINIC_GET_RQ_FREE_WQEBBS(rxq);
876 if (unlikely(free_wqebbs < rxq->rx_free_thresh))
879 /* get rearm mbuf array */
880 pi = HINIC_GET_RQ_LOCAL_PI(rxq);
881 rearm_mbufs = (struct rte_mbuf **)(&rxq->rx_info[pi]);
883 /* check rxq free wqebbs turn around */
884 exp_wqebbs = rxq->q_depth - pi;
885 if (free_wqebbs < exp_wqebbs)
886 exp_wqebbs = free_wqebbs;
888 /* alloc mbuf in bulk */
889 rearm_wqebbs = hinic_rx_alloc_mbuf_bulk(rxq, rearm_mbufs, exp_wqebbs);
890 if (unlikely(rearm_wqebbs == 0))
894 rq_wqe = WQ_WQE_ADDR(rxq->wq, (u32)pi);
895 for (i = 0; i < rearm_wqebbs; i++) {
896 dma_addr = rte_mbuf_data_iova_default(rearm_mbufs[i]);
897 rq_wqe->buf_desc.addr_high =
898 cpu_to_be32(upper_32_bits(dma_addr));
899 rq_wqe->buf_desc.addr_low =
900 cpu_to_be32(lower_32_bits(dma_addr));
903 rxq->wq->prod_idx += rearm_wqebbs;
904 rxq->wq->delta -= rearm_wqebbs;
906 /* update rq hw_pi */
908 HINIC_UPDATE_RQ_HW_PI(rxq, pi + rearm_wqebbs);
911 void hinic_rx_alloc_pkts(struct hinic_rxq *rxq)
913 struct hinic_nic_dev *nic_dev = rxq->nic_dev;
914 struct hinic_rq_wqe *rq_wqe;
915 struct hinic_rx_info *rx_info;
921 free_wqebbs = HINIC_GET_RQ_FREE_WQEBBS(rxq);
922 for (i = 0; i < free_wqebbs; i++) {
923 mb = hinic_rx_alloc_mbuf(rxq, &dma_addr);
925 rxq->rxq_stats.rx_nombuf++;
929 rq_wqe = hinic_get_rq_wqe(nic_dev->hwdev, rxq->q_id, &pi);
930 if (unlikely(!rq_wqe)) {
931 rte_pktmbuf_free(mb);
935 /* fill buffer address only */
936 rq_wqe->buf_desc.addr_high =
937 cpu_to_be32(upper_32_bits(dma_addr));
938 rq_wqe->buf_desc.addr_low =
939 cpu_to_be32(lower_32_bits(dma_addr));
941 rx_info = &rxq->rx_info[pi];
947 HINIC_UPDATE_RQ_HW_PI(rxq, pi + 1);
951 u16 hinic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, u16 nb_pkts)
953 struct rte_mbuf *rxm;
954 struct hinic_rxq *rxq = rx_queue;
955 struct hinic_rx_info *rx_info;
956 volatile struct hinic_rq_cqe *rx_cqe;
957 u16 rx_buf_len, pkts = 0;
958 u16 sw_ci, ci_mask, wqebb_cnt = 0;
959 u32 pkt_len, status, vlan_len;
961 struct hinic_rq_cqe cqe;
962 u32 offload_type, rss_hash;
964 rx_buf_len = rxq->buf_len;
966 /* 1. get polling start ci */
967 ci_mask = HINIC_GET_RQ_WQE_MASK(rxq);
968 sw_ci = HINIC_GET_RQ_LOCAL_CI(rxq);
970 while (pkts < nb_pkts) {
971 /* 2. current ci is done */
972 rx_cqe = &rxq->rx_cqe[sw_ci];
973 status = __atomic_load_n(&rx_cqe->status, __ATOMIC_ACQUIRE);
974 if (!HINIC_GET_RX_DONE_BE(status))
977 /* convert cqe and get packet length */
978 hinic_rq_cqe_be_to_cpu32(&cqe, (volatile void *)rx_cqe);
979 vlan_len = cqe.vlan_len;
981 rx_info = &rxq->rx_info[sw_ci];
984 /* 3. next ci point and prefetch */
988 /* prefetch next mbuf first 64B */
989 rte_prefetch0(rxq->rx_info[sw_ci].mbuf);
991 /* 4. jumbo frame process */
992 pkt_len = HINIC_GET_RX_PKT_LEN(vlan_len);
993 if (likely(pkt_len <= rx_buf_len)) {
994 rxm->data_len = pkt_len;
995 rxm->pkt_len = pkt_len;
998 rxm->data_len = rx_buf_len;
999 rxm->pkt_len = rx_buf_len;
1001 /* if receive jumbo, updating ci will be done by
1002 * hinic_recv_jumbo_pkt function.
1004 HINIC_UPDATE_RQ_LOCAL_CI(rxq, wqebb_cnt + 1);
1006 hinic_recv_jumbo_pkt(rxq, rxm, pkt_len - rx_buf_len);
1007 sw_ci = HINIC_GET_RQ_LOCAL_CI(rxq);
1010 /* 5. vlan/checksum/rss/pkt_type/gro offload */
1011 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1012 rxm->port = rxq->port_id;
1013 offload_type = cqe.offload_type;
1016 rxm->ol_flags |= hinic_rx_vlan(offload_type, vlan_len,
1019 /* checksum offload */
1020 rxm->ol_flags |= hinic_rx_csum(cqe.status, rxq);
1022 /* rss hash offload */
1023 rss_hash = cqe.rss_hash;
1024 rxm->ol_flags |= hinic_rx_rss_hash(offload_type, rss_hash,
1027 /* 6. clear done bit */
1030 rx_bytes += pkt_len;
1031 rx_pkts[pkts++] = rxm;
1036 HINIC_UPDATE_RQ_LOCAL_CI(rxq, wqebb_cnt);
1038 /* do packet stats */
1039 rxq->rxq_stats.packets += pkts;
1040 rxq->rxq_stats.bytes += rx_bytes;
1042 rxq->rxq_stats.burst_pkts = pkts;
1044 /* 8. rearm mbuf to rxq */
1045 hinic_rearm_rxq_mbuf(rxq);