1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
2 * Copyright(c) 2018-2019 Pensando Systems, Inc. All rights reserved.
15 #include <rte_byteorder.h>
16 #include <rte_common.h>
17 #include <rte_cycles.h>
19 #include <rte_debug.h>
20 #include <rte_interrupts.h>
22 #include <rte_memory.h>
23 #include <rte_memzone.h>
24 #include <rte_launch.h>
26 #include <rte_per_lcore.h>
27 #include <rte_lcore.h>
28 #include <rte_atomic.h>
29 #include <rte_branch_prediction.h>
30 #include <rte_mempool.h>
31 #include <rte_malloc.h>
33 #include <rte_ether.h>
34 #include <ethdev_driver.h>
35 #include <rte_prefetch.h>
39 #include <rte_string_fns.h>
40 #include <rte_errno.h>
44 #include "ionic_logs.h"
45 #include "ionic_mac_api.h"
46 #include "ionic_ethdev.h"
47 #include "ionic_lif.h"
48 #include "ionic_rxtx.h"
50 /*********************************************************************
54 **********************************************************************/
57 ionic_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
58 struct rte_eth_txq_info *qinfo)
60 struct ionic_tx_qcq *txq = dev->data->tx_queues[queue_id];
61 struct ionic_queue *q = &txq->qcq.q;
63 qinfo->nb_desc = q->num_descs;
64 qinfo->conf.offloads = dev->data->dev_conf.txmode.offloads;
65 qinfo->conf.tx_deferred_start = txq->flags & IONIC_QCQ_F_DEFERRED;
68 static __rte_always_inline void
69 ionic_tx_flush(struct ionic_tx_qcq *txq)
71 struct ionic_cq *cq = &txq->qcq.cq;
72 struct ionic_queue *q = &txq->qcq.q;
73 struct rte_mbuf *txm, *next;
74 struct ionic_txq_comp *cq_desc_base = cq->base;
75 struct ionic_txq_comp *cq_desc;
77 u_int32_t comp_index = (u_int32_t)-1;
79 cq_desc = &cq_desc_base[cq->tail_idx];
80 while (color_match(cq_desc->color, cq->done_color)) {
81 cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1);
83 /* Prefetch the next 4 descriptors (not really useful here) */
84 if ((cq->tail_idx & 0x3) == 0)
85 rte_prefetch0(&cq_desc_base[cq->tail_idx]);
87 if (cq->tail_idx == 0)
88 cq->done_color = !cq->done_color;
90 comp_index = cq_desc->comp_index;
92 cq_desc = &cq_desc_base[cq->tail_idx];
95 if (comp_index != (u_int32_t)-1) {
96 while (q->tail_idx != comp_index) {
97 info = IONIC_INFO_PTR(q, q->tail_idx);
99 q->tail_idx = Q_NEXT_TO_SRVC(q, 1);
101 /* Prefetch the next 4 descriptors */
102 if ((q->tail_idx & 0x3) == 0)
104 rte_prefetch0(&q->info[q->tail_idx]);
107 * Note: you can just use rte_pktmbuf_free,
108 * but this loop is faster
111 while (txm != NULL) {
113 rte_pktmbuf_free_seg(txm);
121 ionic_dev_tx_queue_release(void *tx_queue)
123 struct ionic_tx_qcq *txq = tx_queue;
124 struct ionic_tx_stats *stats = &txq->stats;
128 IONIC_PRINT(DEBUG, "TX queue %u pkts %ju tso %ju",
129 txq->qcq.q.index, stats->packets, stats->tso);
131 ionic_lif_txq_deinit(txq);
133 ionic_qcq_free(&txq->qcq);
137 ionic_dev_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id)
139 struct ionic_tx_qcq *txq;
141 IONIC_PRINT(DEBUG, "Stopping TX queue %u", tx_queue_id);
143 txq = eth_dev->data->tx_queues[tx_queue_id];
145 eth_dev->data->tx_queue_state[tx_queue_id] =
146 RTE_ETH_QUEUE_STATE_STOPPED;
149 * Note: we should better post NOP Tx desc and wait for its completion
150 * before disabling Tx queue
153 ionic_qcq_disable(&txq->qcq);
161 ionic_dev_tx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id,
162 uint16_t nb_desc, uint32_t socket_id,
163 const struct rte_eth_txconf *tx_conf)
165 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
166 struct ionic_tx_qcq *txq;
170 if (tx_queue_id >= lif->ntxqcqs) {
171 IONIC_PRINT(DEBUG, "Queue index %u not available "
173 tx_queue_id, lif->ntxqcqs);
177 offloads = tx_conf->offloads | eth_dev->data->dev_conf.txmode.offloads;
179 "Configuring skt %u TX queue %u with %u buffers, offloads %jx",
180 socket_id, tx_queue_id, nb_desc, offloads);
182 /* Validate number of receive descriptors */
183 if (!rte_is_power_of_2(nb_desc) || nb_desc < IONIC_MIN_RING_DESC)
184 return -EINVAL; /* or use IONIC_DEFAULT_RING_DESC */
186 /* Free memory prior to re-allocation if needed... */
187 if (eth_dev->data->tx_queues[tx_queue_id] != NULL) {
188 void *tx_queue = eth_dev->data->tx_queues[tx_queue_id];
189 ionic_dev_tx_queue_release(tx_queue);
190 eth_dev->data->tx_queues[tx_queue_id] = NULL;
193 eth_dev->data->tx_queue_state[tx_queue_id] =
194 RTE_ETH_QUEUE_STATE_STOPPED;
196 err = ionic_tx_qcq_alloc(lif, socket_id, tx_queue_id, nb_desc, &txq);
198 IONIC_PRINT(DEBUG, "Queue allocation failure");
202 /* Do not start queue with rte_eth_dev_start() */
203 if (tx_conf->tx_deferred_start)
204 txq->flags |= IONIC_QCQ_F_DEFERRED;
206 /* Convert the offload flags into queue flags */
207 if (offloads & DEV_TX_OFFLOAD_IPV4_CKSUM)
208 txq->flags |= IONIC_QCQ_F_CSUM_L3;
209 if (offloads & DEV_TX_OFFLOAD_TCP_CKSUM)
210 txq->flags |= IONIC_QCQ_F_CSUM_TCP;
211 if (offloads & DEV_TX_OFFLOAD_UDP_CKSUM)
212 txq->flags |= IONIC_QCQ_F_CSUM_UDP;
214 eth_dev->data->tx_queues[tx_queue_id] = txq;
220 * Start Transmit Units for specified queue.
223 ionic_dev_tx_queue_start(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id)
225 uint8_t *tx_queue_state = eth_dev->data->tx_queue_state;
226 struct ionic_tx_qcq *txq;
229 if (tx_queue_state[tx_queue_id] == RTE_ETH_QUEUE_STATE_STARTED) {
230 IONIC_PRINT(DEBUG, "TX queue %u already started",
235 txq = eth_dev->data->tx_queues[tx_queue_id];
237 IONIC_PRINT(DEBUG, "Starting TX queue %u, %u descs",
238 tx_queue_id, txq->qcq.q.num_descs);
240 if (!(txq->flags & IONIC_QCQ_F_INITED)) {
241 err = ionic_lif_txq_init(txq);
245 ionic_qcq_enable(&txq->qcq);
248 tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
254 ionic_tx_tcp_pseudo_csum(struct rte_mbuf *txm)
256 struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(txm, struct ether_hdr *);
257 char *l3_hdr = ((char *)eth_hdr) + txm->l2_len;
258 struct rte_tcp_hdr *tcp_hdr = (struct rte_tcp_hdr *)
259 (l3_hdr + txm->l3_len);
261 if (txm->ol_flags & PKT_TX_IP_CKSUM) {
262 struct rte_ipv4_hdr *ipv4_hdr = (struct rte_ipv4_hdr *)l3_hdr;
263 ipv4_hdr->hdr_checksum = 0;
265 tcp_hdr->cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, tcp_hdr);
267 struct rte_ipv6_hdr *ipv6_hdr = (struct rte_ipv6_hdr *)l3_hdr;
269 tcp_hdr->cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, tcp_hdr);
274 ionic_tx_tcp_inner_pseudo_csum(struct rte_mbuf *txm)
276 struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(txm, struct ether_hdr *);
277 char *l3_hdr = ((char *)eth_hdr) + txm->outer_l2_len +
278 txm->outer_l3_len + txm->l2_len;
279 struct rte_tcp_hdr *tcp_hdr = (struct rte_tcp_hdr *)
280 (l3_hdr + txm->l3_len);
282 if (txm->ol_flags & PKT_TX_IPV4) {
283 struct rte_ipv4_hdr *ipv4_hdr = (struct rte_ipv4_hdr *)l3_hdr;
284 ipv4_hdr->hdr_checksum = 0;
286 tcp_hdr->cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, tcp_hdr);
288 struct rte_ipv6_hdr *ipv6_hdr = (struct rte_ipv6_hdr *)l3_hdr;
290 tcp_hdr->cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, tcp_hdr);
295 ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc,
296 struct rte_mbuf *txm,
297 rte_iova_t addr, uint8_t nsge, uint16_t len,
298 uint32_t hdrlen, uint32_t mss,
300 uint16_t vlan_tci, bool has_vlan,
301 bool start, bool done)
305 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
306 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
307 flags |= start ? IONIC_TXQ_DESC_FLAG_TSO_SOT : 0;
308 flags |= done ? IONIC_TXQ_DESC_FLAG_TSO_EOT : 0;
310 desc->cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_TSO,
313 desc->vlan_tci = vlan_tci;
314 desc->hdr_len = hdrlen;
318 info = IONIC_INFO_PTR(q, q->head_idx);
322 q->head_idx = Q_NEXT_TO_POST(q, 1);
325 static struct ionic_txq_desc *
326 ionic_tx_tso_next(struct ionic_tx_qcq *txq, struct ionic_txq_sg_elem **elem)
328 struct ionic_queue *q = &txq->qcq.q;
329 struct ionic_txq_desc *desc_base = q->base;
330 struct ionic_txq_sg_desc_v1 *sg_desc_base = q->sg_base;
331 struct ionic_txq_desc *desc = &desc_base[q->head_idx];
332 struct ionic_txq_sg_desc_v1 *sg_desc = &sg_desc_base[q->head_idx];
334 *elem = sg_desc->elems;
339 ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm)
341 struct ionic_queue *q = &txq->qcq.q;
342 struct ionic_tx_stats *stats = &txq->stats;
343 struct ionic_txq_desc *desc;
344 struct ionic_txq_sg_elem *elem;
345 struct rte_mbuf *txm_seg;
346 rte_iova_t data_iova;
347 uint64_t desc_addr = 0, next_addr;
348 uint16_t desc_len = 0;
351 uint32_t mss = txm->tso_segsz;
352 uint32_t frag_left = 0;
359 bool has_vlan = !!(txm->ol_flags & PKT_TX_VLAN_PKT);
360 uint16_t vlan_tci = txm->vlan_tci;
361 uint64_t ol_flags = txm->ol_flags;
363 encap = ((ol_flags & PKT_TX_OUTER_IP_CKSUM) ||
364 (ol_flags & PKT_TX_OUTER_UDP_CKSUM)) &&
365 ((ol_flags & PKT_TX_OUTER_IPV4) ||
366 (ol_flags & PKT_TX_OUTER_IPV6));
368 /* Preload inner-most TCP csum field with IP pseudo hdr
369 * calculated with IP length set to zero. HW will later
370 * add in length to each TCP segment resulting from the TSO.
374 ionic_tx_tcp_inner_pseudo_csum(txm);
375 hdrlen = txm->outer_l2_len + txm->outer_l3_len +
376 txm->l2_len + txm->l3_len + txm->l4_len;
378 ionic_tx_tcp_pseudo_csum(txm);
379 hdrlen = txm->l2_len + txm->l3_len + txm->l4_len;
382 seglen = hdrlen + mss;
383 left = txm->data_len;
384 data_iova = rte_mbuf_data_iova(txm);
386 desc = ionic_tx_tso_next(txq, &elem);
389 /* Chop data up into desc segments */
392 len = RTE_MIN(seglen, left);
393 frag_left = seglen - len;
394 desc_addr = rte_cpu_to_le_64(data_iova + offset);
399 if (txm->nb_segs > 1 && frag_left > 0)
401 done = (txm->nb_segs == 1 && left == 0);
402 ionic_tx_tso_post(q, desc, txm,
403 desc_addr, desc_nsge, desc_len,
408 desc = ionic_tx_tso_next(txq, &elem);
413 /* Chop frags into desc segments */
416 while (txm_seg != NULL) {
418 data_iova = rte_mbuf_data_iova(txm_seg);
419 left = txm_seg->data_len;
422 next_addr = rte_cpu_to_le_64(data_iova + offset);
424 len = RTE_MIN(frag_left, left);
426 elem->addr = next_addr;
431 len = RTE_MIN(mss, left);
432 frag_left = mss - len;
433 desc_addr = next_addr;
439 if (txm_seg->next != NULL && frag_left > 0)
442 done = (txm_seg->next == NULL && left == 0);
443 ionic_tx_tso_post(q, desc, txm_seg,
444 desc_addr, desc_nsge, desc_len,
449 desc = ionic_tx_tso_next(txq, &elem);
453 txm_seg = txm_seg->next;
461 static __rte_always_inline int
462 ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm)
464 struct ionic_queue *q = &txq->qcq.q;
465 struct ionic_txq_desc *desc, *desc_base = q->base;
466 struct ionic_txq_sg_desc_v1 *sg_desc_base = q->sg_base;
467 struct ionic_txq_sg_elem *elem;
468 struct ionic_tx_stats *stats = &txq->stats;
469 struct rte_mbuf *txm_seg;
473 uint64_t ol_flags = txm->ol_flags;
475 uint8_t opcode = IONIC_TXQ_DESC_OPCODE_CSUM_NONE;
478 desc = &desc_base[q->head_idx];
479 info = IONIC_INFO_PTR(q, q->head_idx);
481 if ((ol_flags & PKT_TX_IP_CKSUM) &&
482 (txq->flags & IONIC_QCQ_F_CSUM_L3)) {
483 opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW;
484 flags |= IONIC_TXQ_DESC_FLAG_CSUM_L3;
487 if (((ol_flags & PKT_TX_TCP_CKSUM) &&
488 (txq->flags & IONIC_QCQ_F_CSUM_TCP)) ||
489 ((ol_flags & PKT_TX_UDP_CKSUM) &&
490 (txq->flags & IONIC_QCQ_F_CSUM_UDP))) {
491 opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW;
492 flags |= IONIC_TXQ_DESC_FLAG_CSUM_L4;
495 if (opcode == IONIC_TXQ_DESC_OPCODE_CSUM_NONE)
498 has_vlan = (ol_flags & PKT_TX_VLAN_PKT);
499 encap = ((ol_flags & PKT_TX_OUTER_IP_CKSUM) ||
500 (ol_flags & PKT_TX_OUTER_UDP_CKSUM)) &&
501 ((ol_flags & PKT_TX_OUTER_IPV4) ||
502 (ol_flags & PKT_TX_OUTER_IPV6));
504 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
505 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
507 addr = rte_cpu_to_le_64(rte_mbuf_data_iova(txm));
509 desc->cmd = encode_txq_desc_cmd(opcode, flags, txm->nb_segs - 1, addr);
510 desc->len = txm->data_len;
511 desc->vlan_tci = txm->vlan_tci;
515 elem = sg_desc_base[q->head_idx].elems;
518 while (txm_seg != NULL) {
519 elem->len = txm_seg->data_len;
520 elem->addr = rte_cpu_to_le_64(rte_mbuf_data_iova(txm_seg));
522 txm_seg = txm_seg->next;
525 q->head_idx = Q_NEXT_TO_POST(q, 1);
531 ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
534 struct ionic_tx_qcq *txq = tx_queue;
535 struct ionic_queue *q = &txq->qcq.q;
536 struct ionic_tx_stats *stats = &txq->stats;
537 uint32_t next_q_head_idx;
538 uint32_t bytes_tx = 0;
542 /* Cleaning old buffers */
545 if (unlikely(ionic_q_space_avail(q) < nb_pkts)) {
546 stats->stop += nb_pkts;
550 while (nb_tx < nb_pkts) {
551 next_q_head_idx = Q_NEXT_TO_POST(q, 1);
552 if ((next_q_head_idx & 0x3) == 0) {
553 struct ionic_txq_desc *desc_base = q->base;
554 rte_prefetch0(&desc_base[next_q_head_idx]);
555 rte_prefetch0(&q->info[next_q_head_idx]);
558 if (tx_pkts[nb_tx]->ol_flags & PKT_TX_TCP_SEG)
559 err = ionic_tx_tso(txq, tx_pkts[nb_tx]);
561 err = ionic_tx(txq, tx_pkts[nb_tx]);
563 stats->drop += nb_pkts - nb_tx;
567 bytes_tx += tx_pkts[nb_tx]->pkt_len;
576 stats->packets += nb_tx;
577 stats->bytes += bytes_tx;
582 /*********************************************************************
586 **********************************************************************/
588 #define IONIC_TX_OFFLOAD_MASK ( \
596 #define IONIC_TX_OFFLOAD_NOTSUP_MASK \
597 (PKT_TX_OFFLOAD_MASK ^ IONIC_TX_OFFLOAD_MASK)
600 ionic_prep_pkts(void *tx_queue __rte_unused, struct rte_mbuf **tx_pkts,
603 struct rte_mbuf *txm;
607 for (i = 0; i < nb_pkts; i++) {
610 if (txm->nb_segs > IONIC_TX_MAX_SG_ELEMS_V1 + 1) {
615 offloads = txm->ol_flags;
617 if (offloads & IONIC_TX_OFFLOAD_NOTSUP_MASK) {
618 rte_errno = -ENOTSUP;
626 /*********************************************************************
630 **********************************************************************/
632 static void ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index,
633 struct rte_mbuf *mbuf);
636 ionic_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
637 struct rte_eth_rxq_info *qinfo)
639 struct ionic_rx_qcq *rxq = dev->data->rx_queues[queue_id];
640 struct ionic_queue *q = &rxq->qcq.q;
642 qinfo->mp = rxq->mb_pool;
643 qinfo->scattered_rx = dev->data->scattered_rx;
644 qinfo->nb_desc = q->num_descs;
645 qinfo->conf.rx_deferred_start = rxq->flags & IONIC_QCQ_F_DEFERRED;
646 qinfo->conf.offloads = dev->data->dev_conf.rxmode.offloads;
649 static void __rte_cold
650 ionic_rx_empty(struct ionic_rx_qcq *rxq)
652 struct ionic_queue *q = &rxq->qcq.q;
653 struct rte_mbuf *mbuf;
656 while (q->tail_idx != q->head_idx) {
657 info = IONIC_INFO_PTR(q, q->tail_idx);
659 rte_mempool_put(rxq->mb_pool, mbuf);
661 q->tail_idx = Q_NEXT_TO_SRVC(q, 1);
666 ionic_dev_rx_queue_release(void *rx_queue)
668 struct ionic_rx_qcq *rxq = rx_queue;
669 struct ionic_rx_stats *stats;
678 IONIC_PRINT(DEBUG, "RX queue %u pkts %ju mtod %ju",
679 rxq->qcq.q.index, stats->packets, stats->mtods);
683 ionic_lif_rxq_deinit(rxq);
685 ionic_qcq_free(&rxq->qcq);
689 ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev,
690 uint16_t rx_queue_id,
693 const struct rte_eth_rxconf *rx_conf,
694 struct rte_mempool *mp)
696 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
697 struct ionic_rx_qcq *rxq;
701 if (rx_queue_id >= lif->nrxqcqs) {
703 "Queue index %u not available (max %u queues)",
704 rx_queue_id, lif->nrxqcqs);
708 offloads = rx_conf->offloads | eth_dev->data->dev_conf.rxmode.offloads;
710 "Configuring skt %u RX queue %u with %u buffers, offloads %jx",
711 socket_id, rx_queue_id, nb_desc, offloads);
713 if (!rx_conf->rx_drop_en)
714 IONIC_PRINT(WARNING, "No-drop mode is not supported");
716 /* Validate number of receive descriptors */
717 if (!rte_is_power_of_2(nb_desc) ||
718 nb_desc < IONIC_MIN_RING_DESC ||
719 nb_desc > IONIC_MAX_RING_DESC) {
721 "Bad descriptor count (%u) for queue %u (min: %u)",
722 nb_desc, rx_queue_id, IONIC_MIN_RING_DESC);
723 return -EINVAL; /* or use IONIC_DEFAULT_RING_DESC */
726 /* Free memory prior to re-allocation if needed... */
727 if (eth_dev->data->rx_queues[rx_queue_id] != NULL) {
728 void *rx_queue = eth_dev->data->rx_queues[rx_queue_id];
729 ionic_dev_rx_queue_release(rx_queue);
730 eth_dev->data->rx_queues[rx_queue_id] = NULL;
733 eth_dev->data->rx_queue_state[rx_queue_id] =
734 RTE_ETH_QUEUE_STATE_STOPPED;
736 err = ionic_rx_qcq_alloc(lif, socket_id, rx_queue_id, nb_desc,
739 IONIC_PRINT(ERR, "Queue %d allocation failure", rx_queue_id);
746 * Note: the interface does not currently support
747 * DEV_RX_OFFLOAD_KEEP_CRC, please also consider ETHER_CRC_LEN
748 * when the adapter will be able to keep the CRC and subtract
749 * it to the length for all received packets:
750 * if (eth_dev->data->dev_conf.rxmode.offloads &
751 * DEV_RX_OFFLOAD_KEEP_CRC)
752 * rxq->crc_len = ETHER_CRC_LEN;
755 /* Do not start queue with rte_eth_dev_start() */
756 if (rx_conf->rx_deferred_start)
757 rxq->flags |= IONIC_QCQ_F_DEFERRED;
759 eth_dev->data->rx_queues[rx_queue_id] = rxq;
764 static __rte_always_inline void
765 ionic_rx_clean(struct ionic_rx_qcq *rxq,
766 uint32_t q_desc_index, uint32_t cq_desc_index,
767 void *service_cb_arg)
769 struct ionic_queue *q = &rxq->qcq.q;
770 struct ionic_cq *cq = &rxq->qcq.cq;
771 struct ionic_rxq_comp *cq_desc_base = cq->base;
772 struct ionic_rxq_comp *cq_desc = &cq_desc_base[cq_desc_index];
773 struct rte_mbuf *rxm, *rxm_seg;
774 uint32_t max_frame_size =
775 rxq->qcq.lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len;
776 uint64_t pkt_flags = 0;
778 struct ionic_rx_stats *stats = &rxq->stats;
779 struct ionic_rx_service *recv_args = (struct ionic_rx_service *)
781 uint32_t buf_size = (uint16_t)
782 (rte_pktmbuf_data_room_size(rxq->mb_pool) -
783 RTE_PKTMBUF_HEADROOM);
787 assert(q_desc_index == cq_desc->comp_index);
789 info = IONIC_INFO_PTR(q, cq_desc->comp_index);
796 rte_pktmbuf_free(rxm);
798 * Note: rte_mempool_put is faster with no segs
799 * rte_mempool_put(rxq->mb_pool, rxm);
804 if (cq_desc->status) {
805 stats->bad_cq_status++;
806 ionic_rx_recycle(q, q_desc_index, rxm);
810 if (recv_args->nb_rx >= recv_args->nb_pkts) {
812 ionic_rx_recycle(q, q_desc_index, rxm);
816 if (cq_desc->len > max_frame_size ||
819 ionic_rx_recycle(q, q_desc_index, rxm);
823 rxm->data_off = RTE_PKTMBUF_HEADROOM;
824 rte_prefetch1((char *)rxm->buf_addr + rxm->data_off);
825 rxm->nb_segs = 1; /* cq_desc->num_sg_elems */
826 rxm->pkt_len = cq_desc->len;
827 rxm->port = rxq->qcq.lif->port_id;
831 rxm->data_len = RTE_MIN(buf_size, left);
832 left -= rxm->data_len;
835 while (rxm_seg && left) {
836 rxm_seg->data_len = RTE_MIN(buf_size, left);
837 left -= rxm_seg->data_len;
839 rxm_seg = rxm_seg->next;
844 pkt_flags |= PKT_RX_RSS_HASH;
845 rxm->hash.rss = cq_desc->rss_hash;
848 if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN) {
849 pkt_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED;
850 rxm->vlan_tci = cq_desc->vlan_tci;
854 if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC) {
855 if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_OK)
856 pkt_flags |= PKT_RX_IP_CKSUM_GOOD;
857 else if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_BAD)
858 pkt_flags |= PKT_RX_IP_CKSUM_BAD;
860 if ((cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_TCP_OK) ||
861 (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_UDP_OK))
862 pkt_flags |= PKT_RX_L4_CKSUM_GOOD;
863 else if ((cq_desc->csum_flags &
864 IONIC_RXQ_COMP_CSUM_F_TCP_BAD) ||
865 (cq_desc->csum_flags &
866 IONIC_RXQ_COMP_CSUM_F_UDP_BAD))
867 pkt_flags |= PKT_RX_L4_CKSUM_BAD;
870 rxm->ol_flags = pkt_flags;
873 switch (cq_desc->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK) {
874 case IONIC_PKT_TYPE_IPV4:
875 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4;
877 case IONIC_PKT_TYPE_IPV6:
878 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6;
880 case IONIC_PKT_TYPE_IPV4_TCP:
881 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 |
884 case IONIC_PKT_TYPE_IPV6_TCP:
885 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 |
888 case IONIC_PKT_TYPE_IPV4_UDP:
889 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 |
892 case IONIC_PKT_TYPE_IPV6_UDP:
893 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 |
898 struct rte_ether_hdr *eth_h = rte_pktmbuf_mtod(rxm,
899 struct rte_ether_hdr *);
900 uint16_t ether_type = eth_h->ether_type;
901 if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP))
902 pkt_type = RTE_PTYPE_L2_ETHER_ARP;
904 pkt_type = RTE_PTYPE_UNKNOWN;
910 rxm->packet_type = pkt_type;
912 recv_args->rx_pkts[recv_args->nb_rx] = rxm;
916 stats->bytes += rxm->pkt_len;
920 ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index,
921 struct rte_mbuf *mbuf)
923 struct ionic_rxq_desc *desc_base = q->base;
924 struct ionic_rxq_desc *old = &desc_base[q_desc_index];
925 struct ionic_rxq_desc *new = &desc_base[q->head_idx];
927 new->addr = old->addr;
930 q->info[q->head_idx] = mbuf;
932 q->head_idx = Q_NEXT_TO_POST(q, 1);
937 static __rte_always_inline int
938 ionic_rx_fill(struct ionic_rx_qcq *rxq, uint32_t len)
940 struct ionic_queue *q = &rxq->qcq.q;
941 struct ionic_rxq_desc *desc, *desc_base = q->base;
942 struct ionic_rxq_sg_desc *sg_desc, *sg_desc_base = q->sg_base;
943 struct ionic_rxq_sg_elem *elem;
946 uint32_t i, j, nsegs, buf_size, size;
948 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) -
949 RTE_PKTMBUF_HEADROOM);
951 /* Initialize software ring entries */
952 for (i = ionic_q_space_avail(q); i; i--) {
953 struct rte_mbuf *rxm = rte_mbuf_raw_alloc(rxq->mb_pool);
954 struct rte_mbuf *prev_rxm_seg;
957 IONIC_PRINT(ERR, "RX mbuf alloc failed");
961 info = IONIC_INFO_PTR(q, q->head_idx);
963 nsegs = (len + buf_size - 1) / buf_size;
965 desc = &desc_base[q->head_idx];
966 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(rxm));
967 desc->addr = dma_addr;
968 desc->len = buf_size;
970 desc->opcode = (nsegs > 1) ? IONIC_RXQ_DESC_OPCODE_SG :
971 IONIC_RXQ_DESC_OPCODE_SIMPLE;
975 sg_desc = &sg_desc_base[q->head_idx];
976 elem = sg_desc->elems;
977 for (j = 0; j < nsegs - 1 && j < IONIC_RX_MAX_SG_ELEMS; j++) {
978 struct rte_mbuf *rxm_seg;
979 rte_iova_t data_iova;
981 rxm_seg = rte_mbuf_raw_alloc(rxq->mb_pool);
982 if (rxm_seg == NULL) {
983 IONIC_PRINT(ERR, "RX mbuf alloc failed");
987 data_iova = rte_mbuf_data_iova(rxm_seg);
988 dma_addr = rte_cpu_to_le_64(data_iova);
989 elem->addr = dma_addr;
990 elem->len = buf_size;
993 rxm_seg->next = NULL;
994 prev_rxm_seg->next = rxm_seg;
995 prev_rxm_seg = rxm_seg;
999 IONIC_PRINT(ERR, "Rx SG size is not sufficient (%d < %d)",
1004 q->head_idx = Q_NEXT_TO_POST(q, 1);
1013 * Start Receive Units for specified queue.
1016 ionic_dev_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id)
1018 uint32_t frame_size = eth_dev->data->dev_conf.rxmode.max_rx_pkt_len;
1019 uint8_t *rx_queue_state = eth_dev->data->rx_queue_state;
1020 struct ionic_rx_qcq *rxq;
1023 if (rx_queue_state[rx_queue_id] == RTE_ETH_QUEUE_STATE_STARTED) {
1024 IONIC_PRINT(DEBUG, "RX queue %u already started",
1029 rxq = eth_dev->data->rx_queues[rx_queue_id];
1031 IONIC_PRINT(DEBUG, "Starting RX queue %u, %u descs (size: %u)",
1032 rx_queue_id, rxq->qcq.q.num_descs, frame_size);
1034 if (!(rxq->flags & IONIC_QCQ_F_INITED)) {
1035 err = ionic_lif_rxq_init(rxq);
1039 ionic_qcq_enable(&rxq->qcq);
1042 /* Allocate buffers for descriptor rings */
1043 if (ionic_rx_fill(rxq, frame_size) != 0) {
1044 IONIC_PRINT(ERR, "Could not alloc mbuf for queue:%d",
1049 rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
1054 static __rte_always_inline void
1055 ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t work_to_do,
1056 void *service_cb_arg)
1058 struct ionic_cq *cq = &rxq->qcq.cq;
1059 struct ionic_queue *q = &rxq->qcq.q;
1060 struct ionic_rxq_comp *cq_desc, *cq_desc_base = cq->base;
1062 uint32_t curr_q_tail_idx, curr_cq_tail_idx;
1063 uint32_t work_done = 0;
1065 if (work_to_do == 0)
1068 cq_desc = &cq_desc_base[cq->tail_idx];
1069 while (color_match(cq_desc->pkt_type_color, cq->done_color)) {
1070 curr_cq_tail_idx = cq->tail_idx;
1071 cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1);
1073 if (cq->tail_idx == 0)
1074 cq->done_color = !cq->done_color;
1076 /* Prefetch the next 4 descriptors */
1077 if ((cq->tail_idx & 0x3) == 0)
1078 rte_prefetch0(&cq_desc_base[cq->tail_idx]);
1081 more = (q->tail_idx != cq_desc->comp_index);
1083 curr_q_tail_idx = q->tail_idx;
1084 q->tail_idx = Q_NEXT_TO_SRVC(q, 1);
1086 /* Prefetch the next 4 descriptors */
1087 if ((q->tail_idx & 0x3) == 0)
1089 rte_prefetch0(&q->info[q->tail_idx]);
1091 ionic_rx_clean(rxq, curr_q_tail_idx, curr_cq_tail_idx,
1096 if (++work_done == work_to_do)
1099 cq_desc = &cq_desc_base[cq->tail_idx];
1104 * Stop Receive Units for specified queue.
1107 ionic_dev_rx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id)
1109 struct ionic_rx_qcq *rxq;
1111 IONIC_PRINT(DEBUG, "Stopping RX queue %u", rx_queue_id);
1113 rxq = eth_dev->data->rx_queues[rx_queue_id];
1115 eth_dev->data->rx_queue_state[rx_queue_id] =
1116 RTE_ETH_QUEUE_STATE_STOPPED;
1118 ionic_qcq_disable(&rxq->qcq);
1121 ionic_rxq_service(rxq, -1, NULL);
1127 ionic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
1130 struct ionic_rx_qcq *rxq = rx_queue;
1131 uint32_t frame_size =
1132 rxq->qcq.lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len;
1133 struct ionic_rx_service service_cb_arg;
1135 service_cb_arg.rx_pkts = rx_pkts;
1136 service_cb_arg.nb_pkts = nb_pkts;
1137 service_cb_arg.nb_rx = 0;
1139 ionic_rxq_service(rxq, nb_pkts, &service_cb_arg);
1141 ionic_rx_fill(rxq, frame_size);
1143 return service_cb_arg.nb_rx;