1 /* SPDX-License-Identifier: BSD-3-Clause
3 * Copyright (c) 2016-2018 Solarflare Communications Inc.
6 * This software was jointly developed between OKTET Labs (under contract
7 * for Solarflare) and Solarflare Communications, Inc.
18 #include "efx_types.h"
20 #include "efx_regs_ef10.h"
22 #include "sfc_dp_tx.h"
23 #include "sfc_tweak.h"
24 #include "sfc_kvargs.h"
28 #define sfc_ef10_tx_err(dpq, ...) \
29 SFC_DP_LOG(SFC_KVARG_DATAPATH_EF10, ERR, dpq, __VA_ARGS__)
31 /** Maximum length of the DMA descriptor data */
32 #define SFC_EF10_TX_DMA_DESC_LEN_MAX \
33 ((1u << ESF_DZ_TX_KER_BYTE_CNT_WIDTH) - 1)
36 * Maximum number of descriptors/buffers in the Tx ring.
37 * It should guarantee that corresponding event queue never overfill.
38 * EF10 native datapath uses event queue of the same size as Tx queue.
39 * Maximum number of events on datapath can be estimated as number of
40 * Tx queue entries (one event per Tx buffer in the worst case) plus
41 * Tx error and flush events.
43 #define SFC_EF10_TXQ_LIMIT(_ndesc) \
44 ((_ndesc) - 1 /* head must not step on tail */ - \
45 (SFC_EF10_EV_PER_CACHE_LINE - 1) /* max unused EvQ entries */ - \
46 1 /* Rx error */ - 1 /* flush */)
48 struct sfc_ef10_tx_sw_desc {
49 struct rte_mbuf *mbuf;
54 #define SFC_EF10_TXQ_STARTED 0x1
55 #define SFC_EF10_TXQ_NOT_RUNNING 0x2
56 #define SFC_EF10_TXQ_EXCEPTION 0x4
58 unsigned int ptr_mask;
60 unsigned int completed;
61 unsigned int max_fill_level;
62 unsigned int free_thresh;
63 unsigned int evq_read_ptr;
64 struct sfc_ef10_tx_sw_desc *sw_ring;
65 efx_qword_t *txq_hw_ring;
66 volatile void *doorbell;
67 efx_qword_t *evq_hw_ring;
70 uint16_t tso_tcp_header_offset_limit;
72 /* Datapath transmit queue anchor */
76 static inline struct sfc_ef10_txq *
77 sfc_ef10_txq_by_dp_txq(struct sfc_dp_txq *dp_txq)
79 return container_of(dp_txq, struct sfc_ef10_txq, dp);
83 sfc_ef10_tx_get_event(struct sfc_ef10_txq *txq, efx_qword_t *tx_ev)
85 volatile efx_qword_t *evq_hw_ring = txq->evq_hw_ring;
88 * Exception flag is set when reap is done.
89 * It is never done twice per packet burst get and absence of
90 * the flag is checked on burst get entry.
92 SFC_ASSERT((txq->flags & SFC_EF10_TXQ_EXCEPTION) == 0);
94 *tx_ev = evq_hw_ring[txq->evq_read_ptr & txq->ptr_mask];
96 if (!sfc_ef10_ev_present(*tx_ev))
99 if (unlikely(EFX_QWORD_FIELD(*tx_ev, FSF_AZ_EV_CODE) !=
100 FSE_AZ_EV_CODE_TX_EV)) {
102 * Do not move read_ptr to keep the event for exception
103 * handling by the control path.
105 txq->flags |= SFC_EF10_TXQ_EXCEPTION;
106 sfc_ef10_tx_err(&txq->dp.dpq,
107 "TxQ exception at EvQ read ptr %#x",
117 sfc_ef10_tx_process_events(struct sfc_ef10_txq *txq)
119 const unsigned int curr_done = txq->completed - 1;
120 unsigned int anew_done = curr_done;
123 while (sfc_ef10_tx_get_event(txq, &tx_ev)) {
125 * DROP_EVENT is an internal to the NIC, software should
126 * never see it and, therefore, may ignore it.
129 /* Update the latest done descriptor */
130 anew_done = EFX_QWORD_FIELD(tx_ev, ESF_DZ_TX_DESCR_INDX);
132 return (anew_done - curr_done) & txq->ptr_mask;
136 sfc_ef10_tx_reap(struct sfc_ef10_txq *txq)
138 const unsigned int old_read_ptr = txq->evq_read_ptr;
139 const unsigned int ptr_mask = txq->ptr_mask;
140 unsigned int completed = txq->completed;
141 unsigned int pending = completed;
143 pending += sfc_ef10_tx_process_events(txq);
145 if (pending != completed) {
146 struct rte_mbuf *bulk[SFC_TX_REAP_BULK_SIZE];
150 struct sfc_ef10_tx_sw_desc *txd;
153 txd = &txq->sw_ring[completed & ptr_mask];
154 if (txd->mbuf == NULL)
157 m = rte_pktmbuf_prefree_seg(txd->mbuf);
162 if ((nb == RTE_DIM(bulk)) ||
163 ((nb != 0) && (m->pool != bulk[0]->pool))) {
164 rte_mempool_put_bulk(bulk[0]->pool,
170 } while (++completed != pending);
173 rte_mempool_put_bulk(bulk[0]->pool, (void *)bulk, nb);
175 txq->completed = completed;
178 sfc_ef10_ev_qclear(txq->evq_hw_ring, ptr_mask, old_read_ptr,
183 sfc_ef10_tx_qdesc_dma_create(rte_iova_t addr, uint16_t size, bool eop,
186 EFX_POPULATE_QWORD_4(*edp,
187 ESF_DZ_TX_KER_TYPE, 0,
188 ESF_DZ_TX_KER_CONT, !eop,
189 ESF_DZ_TX_KER_BYTE_CNT, size,
190 ESF_DZ_TX_KER_BUF_ADDR, addr);
194 sfc_ef10_tx_qdesc_tso2_create(struct sfc_ef10_txq * const txq,
195 unsigned int added, uint16_t ipv4_id,
196 uint16_t outer_ipv4_id, uint32_t tcp_seq,
199 EFX_POPULATE_QWORD_5(txq->txq_hw_ring[added & txq->ptr_mask],
200 ESF_DZ_TX_DESC_IS_OPT, 1,
201 ESF_DZ_TX_OPTION_TYPE,
202 ESE_DZ_TX_OPTION_DESC_TSO,
203 ESF_DZ_TX_TSO_OPTION_TYPE,
204 ESE_DZ_TX_TSO_OPTION_DESC_FATSO2A,
205 ESF_DZ_TX_TSO_IP_ID, ipv4_id,
206 ESF_DZ_TX_TSO_TCP_SEQNO, tcp_seq);
207 EFX_POPULATE_QWORD_5(txq->txq_hw_ring[(added + 1) & txq->ptr_mask],
208 ESF_DZ_TX_DESC_IS_OPT, 1,
209 ESF_DZ_TX_OPTION_TYPE,
210 ESE_DZ_TX_OPTION_DESC_TSO,
211 ESF_DZ_TX_TSO_OPTION_TYPE,
212 ESE_DZ_TX_TSO_OPTION_DESC_FATSO2B,
213 ESF_DZ_TX_TSO_TCP_MSS, tcp_mss,
214 ESF_DZ_TX_TSO_OUTER_IPID, outer_ipv4_id);
218 sfc_ef10_tx_qpush(struct sfc_ef10_txq *txq, unsigned int added,
225 * This improves performance by pushing a TX descriptor at the same
226 * time as the doorbell. The descriptor must be added to the TXQ,
227 * so that can be used if the hardware decides not to use the pushed
230 desc.eq_u64[0] = txq->txq_hw_ring[pushed & txq->ptr_mask].eq_u64[0];
231 EFX_POPULATE_OWORD_3(oword,
232 ERF_DZ_TX_DESC_WPTR, added & txq->ptr_mask,
233 ERF_DZ_TX_DESC_HWORD, EFX_QWORD_FIELD(desc, EFX_DWORD_1),
234 ERF_DZ_TX_DESC_LWORD, EFX_QWORD_FIELD(desc, EFX_DWORD_0));
236 /* DMA sync to device is not required */
239 * rte_io_wmb() which guarantees that the STORE operations
240 * (i.e. Tx and event descriptor updates) that precede
241 * the rte_io_wmb() call are visible to NIC before the STORE
242 * operations that follow it (i.e. doorbell write).
246 *(volatile __m128i *)txq->doorbell = oword.eo_u128[0];
250 sfc_ef10_tx_pkt_descs_max(const struct rte_mbuf *m)
252 unsigned int extra_descs_per_seg;
253 unsigned int extra_descs_per_pkt;
256 * VLAN offload is not supported yet, so no extra descriptors
257 * are required for VLAN option descriptor.
260 /** Maximum length of the mbuf segment data */
261 #define SFC_MBUF_SEG_LEN_MAX UINT16_MAX
262 RTE_BUILD_BUG_ON(sizeof(m->data_len) != 2);
265 * Each segment is already counted once below. So, calculate
266 * how many extra DMA descriptors may be required per segment in
267 * the worst case because of maximum DMA descriptor length limit.
268 * If maximum segment length is less or equal to maximum DMA
269 * descriptor length, no extra DMA descriptors are required.
271 extra_descs_per_seg =
272 (SFC_MBUF_SEG_LEN_MAX - 1) / SFC_EF10_TX_DMA_DESC_LEN_MAX;
274 /** Maximum length of the packet */
275 #define SFC_MBUF_PKT_LEN_MAX UINT32_MAX
276 RTE_BUILD_BUG_ON(sizeof(m->pkt_len) != 4);
279 * One more limitation on maximum number of extra DMA descriptors
280 * comes from slicing entire packet because of DMA descriptor length
281 * limit taking into account that there is at least one segment
282 * which is already counted below (so division of the maximum
283 * packet length minus one with round down).
284 * TSO is not supported yet, so packet length is limited by
287 extra_descs_per_pkt =
288 (RTE_MIN((unsigned int)EFX_MAC_PDU_MAX,
289 SFC_MBUF_PKT_LEN_MAX) - 1) /
290 SFC_EF10_TX_DMA_DESC_LEN_MAX;
292 return m->nb_segs + RTE_MIN(m->nb_segs * extra_descs_per_seg,
293 extra_descs_per_pkt);
297 sfc_ef10_try_reap(struct sfc_ef10_txq * const txq, unsigned int added,
298 unsigned int needed_desc, unsigned int *dma_desc_space,
304 if (added != txq->added) {
305 sfc_ef10_tx_qpush(txq, added, txq->added);
309 sfc_ef10_tx_reap(txq);
313 * Recalculate DMA descriptor space since Tx reap may change
314 * the number of completed descriptors
316 *dma_desc_space = txq->max_fill_level -
317 (added - txq->completed);
319 return (needed_desc <= *dma_desc_space);
323 sfc_ef10_prepare_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts,
328 for (i = 0; i < nb_pkts; i++) {
329 struct rte_mbuf *m = tx_pkts[i];
332 #ifdef RTE_LIBRTE_SFC_EFX_DEBUG
334 * In non-TSO case, check that a packet segments do not exceed
335 * the size limit. Perform the check in debug mode since MTU
336 * more than 9k is not supported, but the limit here is 16k-1.
338 if (!(m->ol_flags & PKT_TX_TCP_SEG)) {
339 struct rte_mbuf *m_seg;
341 for (m_seg = m; m_seg != NULL; m_seg = m_seg->next) {
342 if (m_seg->data_len >
343 SFC_EF10_TX_DMA_DESC_LEN_MAX) {
350 ret = sfc_dp_tx_prepare_pkt(m);
351 if (unlikely(ret != 0)) {
361 sfc_ef10_xmit_tso_pkt(struct sfc_ef10_txq * const txq, struct rte_mbuf *m_seg,
362 unsigned int *added, unsigned int *dma_desc_space,
365 size_t iph_off = m_seg->l2_len;
366 size_t tcph_off = m_seg->l2_len + m_seg->l3_len;
367 size_t header_len = m_seg->l2_len + m_seg->l3_len + m_seg->l4_len;
368 /* Offset of the payload in the last segment that contains the header */
370 const struct tcp_hdr *th;
371 uint16_t packet_id = 0;
375 struct rte_mbuf *first_m_seg = m_seg;
376 unsigned int pkt_start = *added;
377 unsigned int needed_desc;
378 struct rte_mbuf *m_seg_to_free_up_to = first_m_seg;
381 if (unlikely(tcph_off > txq->tso_tcp_header_offset_limit))
385 * Preliminary estimation of required DMA descriptors, including extra
386 * descriptor for TSO header that is needed when the header is
387 * separated from payload in one segment. It does not include
388 * extra descriptors that may appear when a big segment is split across
389 * several descriptors.
391 needed_desc = m_seg->nb_segs +
392 (unsigned int)SFC_EF10_TSO_OPT_DESCS_NUM +
393 (unsigned int)SFC_EF10_TSO_HDR_DESCS_NUM;
395 if (needed_desc > *dma_desc_space &&
396 !sfc_ef10_try_reap(txq, pkt_start, needed_desc,
397 dma_desc_space, reap_done)) {
399 * If a future Tx reap may increase available DMA descriptor
400 * space, do not try to send the packet.
402 if (txq->completed != pkt_start)
405 * Do not allow to send packet if the maximum DMA
406 * descriptor space is not sufficient to hold TSO
407 * descriptors, header descriptor and at least 1
408 * segment descriptor.
410 if (*dma_desc_space < SFC_EF10_TSO_OPT_DESCS_NUM +
411 SFC_EF10_TSO_HDR_DESCS_NUM + 1)
415 /* Check if the header is not fragmented */
416 if (rte_pktmbuf_data_len(m_seg) >= header_len) {
417 hdr_addr = rte_pktmbuf_mtod(m_seg, uint8_t *);
418 hdr_iova = rte_mbuf_data_iova(m_seg);
419 if (rte_pktmbuf_data_len(m_seg) == header_len) {
420 /* Cannot send a packet that consists only of header */
421 if (unlikely(m_seg->next == NULL))
424 * Associate header mbuf with header descriptor
425 * which is located after TSO descriptors.
427 txq->sw_ring[(pkt_start + SFC_EF10_TSO_OPT_DESCS_NUM) &
428 txq->ptr_mask].mbuf = m_seg;
433 * If there is no payload offset (payload starts at the
434 * beginning of a segment) then an extra descriptor for
435 * separated header is not needed.
442 unsigned int copied_segs;
443 unsigned int hdr_addr_off = (*added & txq->ptr_mask) *
447 * Discard a packet if header linearization is needed but
448 * the header is too big.
450 if (unlikely(header_len > SFC_TSOH_STD_LEN))
453 hdr_addr = txq->tsoh + hdr_addr_off;
454 hdr_iova = txq->tsoh_iova + hdr_addr_off;
455 copied_segs = sfc_tso_prepare_header(hdr_addr, header_len,
458 /* Cannot send a packet that consists only of header */
459 if (unlikely(m_seg == NULL))
462 m_seg_to_free_up_to = m_seg;
464 * Reduce the number of needed descriptors by the number of
465 * segments that entirely consist of header data.
467 needed_desc -= copied_segs;
469 /* Extra descriptor for separated header is not needed */
475 * Tx prepare has debug-only checks that offload flags are correctly
476 * filled in in TSO mbuf. Use zero IPID if there is no IPv4 flag.
477 * If the packet is still IPv4, HW will simply start from zero IPID.
479 if (first_m_seg->ol_flags & PKT_TX_IPV4) {
480 const struct ipv4_hdr *iphe4;
482 iphe4 = (const struct ipv4_hdr *)(hdr_addr + iph_off);
483 rte_memcpy(&packet_id, &iphe4->packet_id, sizeof(uint16_t));
484 packet_id = rte_be_to_cpu_16(packet_id);
487 th = (const struct tcp_hdr *)(hdr_addr + tcph_off);
488 rte_memcpy(&sent_seq, &th->sent_seq, sizeof(uint32_t));
489 sent_seq = rte_be_to_cpu_32(sent_seq);
491 sfc_ef10_tx_qdesc_tso2_create(txq, *added, packet_id, 0, sent_seq,
492 first_m_seg->tso_segsz);
493 (*added) += SFC_EF10_TSO_OPT_DESCS_NUM;
495 sfc_ef10_tx_qdesc_dma_create(hdr_iova, header_len, false,
496 &txq->txq_hw_ring[(*added) & txq->ptr_mask]);
500 rte_iova_t next_frag = rte_mbuf_data_iova(m_seg);
501 unsigned int seg_len = rte_pktmbuf_data_len(m_seg);
509 rte_iova_t frag_addr = next_frag;
512 frag_len = RTE_MIN(seg_len,
513 SFC_EF10_TX_DMA_DESC_LEN_MAX);
515 next_frag += frag_len;
518 eop = (seg_len == 0 && m_seg->next == NULL);
520 id = (*added) & txq->ptr_mask;
524 * Initially we assume that one DMA descriptor is needed
525 * for every segment. When the segment is split across
526 * several DMA descriptors, increase the estimation.
528 needed_desc += (seg_len != 0);
531 * When no more descriptors can be added, but not all
532 * segments are processed.
534 if (*added - pkt_start == *dma_desc_space &&
536 !sfc_ef10_try_reap(txq, pkt_start, needed_desc,
537 dma_desc_space, reap_done)) {
539 struct rte_mbuf *m_next;
541 if (txq->completed != pkt_start) {
545 * Reset mbuf associations with added
548 for (i = pkt_start; i != *added; i++) {
549 id = i & txq->ptr_mask;
550 txq->sw_ring[id].mbuf = NULL;
555 /* Free the segments that cannot be sent */
556 for (m = m_seg->next; m != NULL; m = m_next) {
558 rte_pktmbuf_free_seg(m);
561 /* Ignore the rest of the segment */
565 sfc_ef10_tx_qdesc_dma_create(frag_addr, frag_len,
566 eop, &txq->txq_hw_ring[id]);
568 } while (seg_len != 0);
570 txq->sw_ring[id].mbuf = m_seg;
576 * Free segments which content was entirely copied to the TSO header
577 * memory space of Tx queue
579 for (m_seg = first_m_seg; m_seg != m_seg_to_free_up_to;) {
580 struct rte_mbuf *seg_to_free = m_seg;
583 rte_pktmbuf_free_seg(seg_to_free);
590 sfc_ef10_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
592 struct sfc_ef10_txq * const txq = sfc_ef10_txq_by_dp_txq(tx_queue);
594 unsigned int dma_desc_space;
596 struct rte_mbuf **pktp;
597 struct rte_mbuf **pktp_end;
599 if (unlikely(txq->flags &
600 (SFC_EF10_TXQ_NOT_RUNNING | SFC_EF10_TXQ_EXCEPTION)))
604 dma_desc_space = txq->max_fill_level - (added - txq->completed);
606 reap_done = (dma_desc_space < txq->free_thresh);
608 sfc_ef10_tx_reap(txq);
609 dma_desc_space = txq->max_fill_level - (added - txq->completed);
612 for (pktp = &tx_pkts[0], pktp_end = &tx_pkts[nb_pkts];
615 struct rte_mbuf *m_seg = *pktp;
616 unsigned int pkt_start = added;
619 if (likely(pktp + 1 != pktp_end))
620 rte_mbuf_prefetch_part1(pktp[1]);
622 if (m_seg->ol_flags & PKT_TX_TCP_SEG) {
625 rc = sfc_ef10_xmit_tso_pkt(txq, m_seg, &added,
626 &dma_desc_space, &reap_done);
630 /* Packet can be sent in following xmit calls */
631 if (likely(rc == ENOSPC))
635 * Packet cannot be sent, tell RTE that
636 * it is sent, but actually drop it and
637 * continue with another packet
639 rte_pktmbuf_free(*pktp);
643 goto dma_desc_space_update;
646 if (sfc_ef10_tx_pkt_descs_max(m_seg) > dma_desc_space) {
650 /* Push already prepared descriptors before polling */
651 if (added != txq->added) {
652 sfc_ef10_tx_qpush(txq, added, txq->added);
656 sfc_ef10_tx_reap(txq);
658 dma_desc_space = txq->max_fill_level -
659 (added - txq->completed);
660 if (sfc_ef10_tx_pkt_descs_max(m_seg) > dma_desc_space)
664 pkt_len = m_seg->pkt_len;
666 rte_iova_t seg_addr = rte_mbuf_data_iova(m_seg);
667 unsigned int seg_len = rte_pktmbuf_data_len(m_seg);
668 unsigned int id = added & txq->ptr_mask;
670 SFC_ASSERT(seg_len <= SFC_EF10_TX_DMA_DESC_LEN_MAX);
674 sfc_ef10_tx_qdesc_dma_create(seg_addr,
675 seg_len, (pkt_len == 0),
676 &txq->txq_hw_ring[id]);
679 * rte_pktmbuf_free() is commonly used in DPDK for
680 * recycling packets - the function checks every
681 * segment's reference counter and returns the
682 * buffer to its pool whenever possible;
683 * nevertheless, freeing mbuf segments one by one
684 * may entail some performance decline;
685 * from this point, sfc_efx_tx_reap() does the same job
686 * on its own and frees buffers in bulks (all mbufs
687 * within a bulk belong to the same pool);
688 * from this perspective, individual segment pointers
689 * must be associated with the corresponding SW
690 * descriptors independently so that only one loop
691 * is sufficient on reap to inspect all the buffers
693 txq->sw_ring[id].mbuf = m_seg;
697 } while ((m_seg = m_seg->next) != 0);
699 dma_desc_space_update:
700 dma_desc_space -= (added - pkt_start);
703 if (likely(added != txq->added)) {
704 sfc_ef10_tx_qpush(txq, added, txq->added);
708 #if SFC_TX_XMIT_PKTS_REAP_AT_LEAST_ONCE
710 sfc_ef10_tx_reap(txq);
713 return pktp - &tx_pkts[0];
717 sfc_ef10_simple_tx_reap(struct sfc_ef10_txq *txq)
719 const unsigned int old_read_ptr = txq->evq_read_ptr;
720 const unsigned int ptr_mask = txq->ptr_mask;
721 unsigned int completed = txq->completed;
722 unsigned int pending = completed;
724 pending += sfc_ef10_tx_process_events(txq);
726 if (pending != completed) {
727 struct rte_mbuf *bulk[SFC_TX_REAP_BULK_SIZE];
731 struct sfc_ef10_tx_sw_desc *txd;
733 txd = &txq->sw_ring[completed & ptr_mask];
735 if (nb == RTE_DIM(bulk)) {
736 rte_mempool_put_bulk(bulk[0]->pool,
741 bulk[nb++] = txd->mbuf;
742 } while (++completed != pending);
744 rte_mempool_put_bulk(bulk[0]->pool, (void *)bulk, nb);
746 txq->completed = completed;
749 sfc_ef10_ev_qclear(txq->evq_hw_ring, ptr_mask, old_read_ptr,
755 sfc_ef10_simple_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
758 struct sfc_ef10_txq * const txq = sfc_ef10_txq_by_dp_txq(tx_queue);
759 unsigned int ptr_mask;
761 unsigned int dma_desc_space;
763 struct rte_mbuf **pktp;
764 struct rte_mbuf **pktp_end;
766 if (unlikely(txq->flags &
767 (SFC_EF10_TXQ_NOT_RUNNING | SFC_EF10_TXQ_EXCEPTION)))
770 ptr_mask = txq->ptr_mask;
772 dma_desc_space = txq->max_fill_level - (added - txq->completed);
774 reap_done = (dma_desc_space < RTE_MAX(txq->free_thresh, nb_pkts));
776 sfc_ef10_simple_tx_reap(txq);
777 dma_desc_space = txq->max_fill_level - (added - txq->completed);
780 pktp_end = &tx_pkts[MIN(nb_pkts, dma_desc_space)];
781 for (pktp = &tx_pkts[0]; pktp != pktp_end; ++pktp) {
782 struct rte_mbuf *pkt = *pktp;
783 unsigned int id = added & ptr_mask;
785 SFC_ASSERT(rte_pktmbuf_data_len(pkt) <=
786 SFC_EF10_TX_DMA_DESC_LEN_MAX);
788 sfc_ef10_tx_qdesc_dma_create(rte_mbuf_data_iova(pkt),
789 rte_pktmbuf_data_len(pkt),
790 true, &txq->txq_hw_ring[id]);
792 txq->sw_ring[id].mbuf = pkt;
797 if (likely(added != txq->added)) {
798 sfc_ef10_tx_qpush(txq, added, txq->added);
802 #if SFC_TX_XMIT_PKTS_REAP_AT_LEAST_ONCE
804 sfc_ef10_simple_tx_reap(txq);
807 return pktp - &tx_pkts[0];
810 static sfc_dp_tx_get_dev_info_t sfc_ef10_get_dev_info;
812 sfc_ef10_get_dev_info(struct rte_eth_dev_info *dev_info)
815 * Number of descriptors just defines maximum number of pushed
816 * descriptors (fill level).
818 dev_info->tx_desc_lim.nb_min = 1;
819 dev_info->tx_desc_lim.nb_align = 1;
822 static sfc_dp_tx_qsize_up_rings_t sfc_ef10_tx_qsize_up_rings;
824 sfc_ef10_tx_qsize_up_rings(uint16_t nb_tx_desc,
825 struct sfc_dp_tx_hw_limits *limits,
826 unsigned int *txq_entries,
827 unsigned int *evq_entries,
828 unsigned int *txq_max_fill_level)
831 * rte_ethdev API guarantees that the number meets min, max and
832 * alignment requirements.
834 if (nb_tx_desc <= limits->txq_min_entries)
835 *txq_entries = limits->txq_min_entries;
837 *txq_entries = rte_align32pow2(nb_tx_desc);
839 *evq_entries = *txq_entries;
841 *txq_max_fill_level = RTE_MIN(nb_tx_desc,
842 SFC_EF10_TXQ_LIMIT(*evq_entries));
846 static sfc_dp_tx_qcreate_t sfc_ef10_tx_qcreate;
848 sfc_ef10_tx_qcreate(uint16_t port_id, uint16_t queue_id,
849 const struct rte_pci_addr *pci_addr, int socket_id,
850 const struct sfc_dp_tx_qcreate_info *info,
851 struct sfc_dp_txq **dp_txqp)
853 struct sfc_ef10_txq *txq;
857 if (info->txq_entries != info->evq_entries)
861 txq = rte_zmalloc_socket("sfc-ef10-txq", sizeof(*txq),
862 RTE_CACHE_LINE_SIZE, socket_id);
866 sfc_dp_queue_init(&txq->dp.dpq, port_id, queue_id, pci_addr);
869 txq->sw_ring = rte_calloc_socket("sfc-ef10-txq-sw_ring",
871 sizeof(*txq->sw_ring),
872 RTE_CACHE_LINE_SIZE, socket_id);
873 if (txq->sw_ring == NULL)
874 goto fail_sw_ring_alloc;
876 if (info->offloads & DEV_TX_OFFLOAD_TCP_TSO) {
877 txq->tsoh = rte_calloc_socket("sfc-ef10-txq-tsoh",
882 if (txq->tsoh == NULL)
883 goto fail_tsoh_alloc;
885 txq->tsoh_iova = rte_malloc_virt2iova(txq->tsoh);
888 txq->flags = SFC_EF10_TXQ_NOT_RUNNING;
889 txq->ptr_mask = info->txq_entries - 1;
890 txq->max_fill_level = info->max_fill_level;
891 txq->free_thresh = info->free_thresh;
892 txq->txq_hw_ring = info->txq_hw_ring;
893 txq->doorbell = (volatile uint8_t *)info->mem_bar +
894 ER_DZ_TX_DESC_UPD_REG_OFST +
895 (info->hw_index << info->vi_window_shift);
896 txq->evq_hw_ring = info->evq_hw_ring;
897 txq->tso_tcp_header_offset_limit = info->tso_tcp_header_offset_limit;
903 rte_free(txq->sw_ring);
913 static sfc_dp_tx_qdestroy_t sfc_ef10_tx_qdestroy;
915 sfc_ef10_tx_qdestroy(struct sfc_dp_txq *dp_txq)
917 struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq);
920 rte_free(txq->sw_ring);
924 static sfc_dp_tx_qstart_t sfc_ef10_tx_qstart;
926 sfc_ef10_tx_qstart(struct sfc_dp_txq *dp_txq, unsigned int evq_read_ptr,
927 unsigned int txq_desc_index)
929 struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq);
931 txq->evq_read_ptr = evq_read_ptr;
932 txq->added = txq->completed = txq_desc_index;
934 txq->flags |= SFC_EF10_TXQ_STARTED;
935 txq->flags &= ~(SFC_EF10_TXQ_NOT_RUNNING | SFC_EF10_TXQ_EXCEPTION);
940 static sfc_dp_tx_qstop_t sfc_ef10_tx_qstop;
942 sfc_ef10_tx_qstop(struct sfc_dp_txq *dp_txq, unsigned int *evq_read_ptr)
944 struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq);
946 txq->flags |= SFC_EF10_TXQ_NOT_RUNNING;
948 *evq_read_ptr = txq->evq_read_ptr;
951 static sfc_dp_tx_qtx_ev_t sfc_ef10_tx_qtx_ev;
953 sfc_ef10_tx_qtx_ev(struct sfc_dp_txq *dp_txq, __rte_unused unsigned int id)
955 __rte_unused struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq);
957 SFC_ASSERT(txq->flags & SFC_EF10_TXQ_NOT_RUNNING);
960 * It is safe to ignore Tx event since we reap all mbufs on
961 * queue purge anyway.
967 static sfc_dp_tx_qreap_t sfc_ef10_tx_qreap;
969 sfc_ef10_tx_qreap(struct sfc_dp_txq *dp_txq)
971 struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq);
972 unsigned int completed;
974 for (completed = txq->completed; completed != txq->added; ++completed) {
975 struct sfc_ef10_tx_sw_desc *txd;
977 txd = &txq->sw_ring[completed & txq->ptr_mask];
978 if (txd->mbuf != NULL) {
979 rte_pktmbuf_free_seg(txd->mbuf);
984 txq->flags &= ~SFC_EF10_TXQ_STARTED;
988 sfc_ef10_tx_qdesc_npending(struct sfc_ef10_txq *txq)
990 const unsigned int curr_done = txq->completed - 1;
991 unsigned int anew_done = curr_done;
993 const unsigned int evq_old_read_ptr = txq->evq_read_ptr;
995 if (unlikely(txq->flags &
996 (SFC_EF10_TXQ_NOT_RUNNING | SFC_EF10_TXQ_EXCEPTION)))
999 while (sfc_ef10_tx_get_event(txq, &tx_ev))
1000 anew_done = EFX_QWORD_FIELD(tx_ev, ESF_DZ_TX_DESCR_INDX);
1003 * The function does not process events, so return event queue read
1004 * pointer to the original position to allow the events that were
1005 * read to be processed later
1007 txq->evq_read_ptr = evq_old_read_ptr;
1009 return (anew_done - curr_done) & txq->ptr_mask;
1012 static sfc_dp_tx_qdesc_status_t sfc_ef10_tx_qdesc_status;
1014 sfc_ef10_tx_qdesc_status(struct sfc_dp_txq *dp_txq,
1017 struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq);
1018 unsigned int npending = sfc_ef10_tx_qdesc_npending(txq);
1020 if (unlikely(offset > txq->ptr_mask))
1023 if (unlikely(offset >= txq->max_fill_level))
1024 return RTE_ETH_TX_DESC_UNAVAIL;
1026 if (unlikely(offset < npending))
1027 return RTE_ETH_TX_DESC_FULL;
1029 return RTE_ETH_TX_DESC_DONE;
1032 struct sfc_dp_tx sfc_ef10_tx = {
1034 .name = SFC_KVARG_DATAPATH_EF10,
1036 .hw_fw_caps = SFC_DP_HW_FW_CAP_EF10,
1038 .features = SFC_DP_TX_FEAT_TSO |
1039 SFC_DP_TX_FEAT_MULTI_SEG |
1040 SFC_DP_TX_FEAT_MULTI_POOL |
1041 SFC_DP_TX_FEAT_REFCNT |
1042 SFC_DP_TX_FEAT_MULTI_PROCESS,
1043 .get_dev_info = sfc_ef10_get_dev_info,
1044 .qsize_up_rings = sfc_ef10_tx_qsize_up_rings,
1045 .qcreate = sfc_ef10_tx_qcreate,
1046 .qdestroy = sfc_ef10_tx_qdestroy,
1047 .qstart = sfc_ef10_tx_qstart,
1048 .qtx_ev = sfc_ef10_tx_qtx_ev,
1049 .qstop = sfc_ef10_tx_qstop,
1050 .qreap = sfc_ef10_tx_qreap,
1051 .qdesc_status = sfc_ef10_tx_qdesc_status,
1052 .pkt_prepare = sfc_ef10_prepare_pkts,
1053 .pkt_burst = sfc_ef10_xmit_pkts,
1056 struct sfc_dp_tx sfc_ef10_simple_tx = {
1058 .name = SFC_KVARG_DATAPATH_EF10_SIMPLE,
1061 .features = SFC_DP_TX_FEAT_MULTI_PROCESS,
1062 .get_dev_info = sfc_ef10_get_dev_info,
1063 .qsize_up_rings = sfc_ef10_tx_qsize_up_rings,
1064 .qcreate = sfc_ef10_tx_qcreate,
1065 .qdestroy = sfc_ef10_tx_qdestroy,
1066 .qstart = sfc_ef10_tx_qstart,
1067 .qtx_ev = sfc_ef10_tx_qtx_ev,
1068 .qstop = sfc_ef10_tx_qstop,
1069 .qreap = sfc_ef10_tx_qreap,
1070 .qdesc_status = sfc_ef10_tx_qdesc_status,
1071 .pkt_burst = sfc_ef10_simple_xmit_pkts,