1 /* SPDX-License-Identifier: BSD-3-Clause
3 * Copyright(c) 2019-2021 Xilinx, Inc.
4 * Copyright(c) 2018-2019 Solarflare Communications Inc.
6 * This software was jointly developed between OKTET Labs (under contract
7 * for Solarflare) and Solarflare Communications, Inc.
13 #include <rte_mbuf_dyn.h>
18 #include "efx_types.h"
20 #include "efx_regs_ef100.h"
22 #include "sfc_debug.h"
23 #include "sfc_dp_tx.h"
24 #include "sfc_tweak.h"
25 #include "sfc_kvargs.h"
26 #include "sfc_ef100.h"
29 #define sfc_ef100_tx_err(_txq, ...) \
30 SFC_DP_LOG(SFC_KVARG_DATAPATH_EF100, ERR, &(_txq)->dp.dpq, __VA_ARGS__)
32 #define sfc_ef100_tx_debug(_txq, ...) \
33 SFC_DP_LOG(SFC_KVARG_DATAPATH_EF100, DEBUG, &(_txq)->dp.dpq, \
37 /** Maximum length of the send descriptor data */
38 #define SFC_EF100_TX_SEND_DESC_LEN_MAX \
39 ((1u << ESF_GZ_TX_SEND_LEN_WIDTH) - 1)
41 /** Maximum length of the segment descriptor data */
42 #define SFC_EF100_TX_SEG_DESC_LEN_MAX \
43 ((1u << ESF_GZ_TX_SEG_LEN_WIDTH) - 1)
46 * Maximum number of descriptors/buffers in the Tx ring.
47 * It should guarantee that corresponding event queue never overfill.
48 * EF100 native datapath uses event queue of the same size as Tx queue.
49 * Maximum number of events on datapath can be estimated as number of
50 * Tx queue entries (one event per Tx buffer in the worst case) plus
51 * Tx error and flush events.
53 #define SFC_EF100_TXQ_LIMIT(_ndesc) \
54 ((_ndesc) - 1 /* head must not step on tail */ - \
55 1 /* Rx error */ - 1 /* flush */)
57 struct sfc_ef100_tx_sw_desc {
58 struct rte_mbuf *mbuf;
61 struct sfc_ef100_txq {
63 #define SFC_EF100_TXQ_STARTED 0x1
64 #define SFC_EF100_TXQ_NOT_RUNNING 0x2
65 #define SFC_EF100_TXQ_EXCEPTION 0x4
67 unsigned int ptr_mask;
69 unsigned int completed;
70 unsigned int max_fill_level;
71 unsigned int free_thresh;
72 struct sfc_ef100_tx_sw_desc *sw_ring;
73 efx_oword_t *txq_hw_ring;
74 volatile void *doorbell;
77 unsigned int evq_read_ptr;
78 unsigned int evq_phase_bit_shift;
79 volatile efx_qword_t *evq_hw_ring;
81 uint16_t tso_tcp_header_offset_limit;
82 uint16_t tso_max_nb_header_descs;
83 uint16_t tso_max_header_len;
84 uint16_t tso_max_nb_payload_descs;
85 uint32_t tso_max_payload_len;
86 uint32_t tso_max_nb_outgoing_frames;
88 /* Datapath transmit queue anchor */
92 static inline struct sfc_ef100_txq *
93 sfc_ef100_txq_by_dp_txq(struct sfc_dp_txq *dp_txq)
95 return container_of(dp_txq, struct sfc_ef100_txq, dp);
99 sfc_ef100_tx_prepare_pkt_tso(struct sfc_ef100_txq * const txq,
102 size_t header_len = ((m->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) ?
103 m->outer_l2_len + m->outer_l3_len : 0) +
104 m->l2_len + m->l3_len + m->l4_len;
105 size_t payload_len = m->pkt_len - header_len;
106 unsigned long mss_conformant_max_payload_len;
107 unsigned int nb_payload_descs;
109 #ifdef RTE_LIBRTE_SFC_EFX_DEBUG
110 switch (m->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) {
113 case RTE_MBUF_F_TX_TUNNEL_VXLAN:
115 case RTE_MBUF_F_TX_TUNNEL_GENEVE:
122 mss_conformant_max_payload_len =
123 m->tso_segsz * txq->tso_max_nb_outgoing_frames;
126 * Don't really want to know exact number of payload segments.
127 * Just use total number of segments as upper limit. Practically
128 * maximum number of payload segments is significantly bigger
129 * than maximum number header segments, so we can neglect header
130 * segments excluded total number of segments to estimate number
131 * of payload segments required.
133 nb_payload_descs = m->nb_segs;
136 * Carry out multiple independent checks using bitwise OR
137 * to avoid unnecessary conditional branching.
139 if (unlikely((header_len > txq->tso_max_header_len) |
140 (nb_payload_descs > txq->tso_max_nb_payload_descs) |
141 (payload_len > txq->tso_max_payload_len) |
142 (payload_len > mss_conformant_max_payload_len) |
143 (m->pkt_len == header_len)))
150 sfc_ef100_tx_prepare_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
153 struct sfc_ef100_txq * const txq = sfc_ef100_txq_by_dp_txq(tx_queue);
156 for (i = 0; i < nb_pkts; i++) {
157 struct rte_mbuf *m = tx_pkts[i];
158 unsigned int max_nb_header_segs = 0;
159 bool calc_phdr_cksum = false;
163 * Partial checksum offload is used in the case of
164 * inner TCP/UDP checksum offload. It requires
165 * pseudo-header checksum which is calculated below,
166 * but requires contiguous packet headers.
168 if ((m->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) &&
169 (m->ol_flags & RTE_MBUF_F_TX_L4_MASK)) {
170 calc_phdr_cksum = true;
171 max_nb_header_segs = 1;
172 } else if (m->ol_flags & RTE_MBUF_F_TX_TCP_SEG) {
173 max_nb_header_segs = txq->tso_max_nb_header_descs;
176 ret = sfc_dp_tx_prepare_pkt(m, max_nb_header_segs, 0,
177 txq->tso_tcp_header_offset_limit,
178 txq->max_fill_level, 1, 0);
179 if (unlikely(ret != 0)) {
184 if (m->ol_flags & RTE_MBUF_F_TX_TCP_SEG) {
185 ret = sfc_ef100_tx_prepare_pkt_tso(txq, m);
186 if (unlikely(ret != 0)) {
190 } else if (m->nb_segs > EFX_MASK32(ESF_GZ_TX_SEND_NUM_SEGS)) {
195 if (calc_phdr_cksum) {
197 * Full checksum offload does IPv4 header checksum
198 * and does not require any assistance.
200 ret = rte_net_intel_cksum_flags_prepare(m,
201 m->ol_flags & ~RTE_MBUF_F_TX_IP_CKSUM);
202 if (unlikely(ret != 0)) {
213 sfc_ef100_tx_get_event(struct sfc_ef100_txq *txq, efx_qword_t *ev)
215 volatile efx_qword_t *evq_hw_ring = txq->evq_hw_ring;
218 * Exception flag is set when reap is done.
219 * It is never done twice per packet burst get, and absence of
220 * the flag is checked on burst get entry.
222 SFC_ASSERT((txq->flags & SFC_EF100_TXQ_EXCEPTION) == 0);
224 *ev = evq_hw_ring[txq->evq_read_ptr & txq->ptr_mask];
226 if (!sfc_ef100_ev_present(ev,
227 (txq->evq_read_ptr >> txq->evq_phase_bit_shift) & 1))
230 if (unlikely(!sfc_ef100_ev_type_is(ev,
231 ESE_GZ_EF100_EV_TX_COMPLETION))) {
233 * Do not move read_ptr to keep the event for exception
234 * handling by the control path.
236 txq->flags |= SFC_EF100_TXQ_EXCEPTION;
237 sfc_ef100_tx_err(txq,
238 "TxQ exception at EvQ ptr %u(%#x), event %08x:%08x",
239 txq->evq_read_ptr, txq->evq_read_ptr & txq->ptr_mask,
240 EFX_QWORD_FIELD(*ev, EFX_DWORD_1),
241 EFX_QWORD_FIELD(*ev, EFX_DWORD_0));
245 sfc_ef100_tx_debug(txq, "TxQ got event %08x:%08x at %u (%#x)",
246 EFX_QWORD_FIELD(*ev, EFX_DWORD_1),
247 EFX_QWORD_FIELD(*ev, EFX_DWORD_0),
249 txq->evq_read_ptr & txq->ptr_mask);
256 sfc_ef100_tx_process_events(struct sfc_ef100_txq *txq)
258 unsigned int num_descs = 0;
261 while (sfc_ef100_tx_get_event(txq, &tx_ev))
262 num_descs += EFX_QWORD_FIELD(tx_ev, ESF_GZ_EV_TXCMPL_NUM_DESC);
268 sfc_ef100_tx_reap_num_descs(struct sfc_ef100_txq *txq, unsigned int num_descs)
271 unsigned int completed = txq->completed;
272 unsigned int pending = completed + num_descs;
273 struct rte_mbuf *bulk[SFC_TX_REAP_BULK_SIZE];
277 struct sfc_ef100_tx_sw_desc *txd;
280 txd = &txq->sw_ring[completed & txq->ptr_mask];
281 if (txd->mbuf == NULL)
284 m = rte_pktmbuf_prefree_seg(txd->mbuf);
290 if (nb == RTE_DIM(bulk) ||
291 (nb != 0 && m->pool != bulk[0]->pool)) {
292 rte_mempool_put_bulk(bulk[0]->pool,
298 } while (++completed != pending);
301 rte_mempool_put_bulk(bulk[0]->pool, (void *)bulk, nb);
303 txq->completed = completed;
308 sfc_ef100_tx_reap(struct sfc_ef100_txq *txq)
310 sfc_ef100_tx_reap_num_descs(txq, sfc_ef100_tx_process_events(txq));
314 sfc_ef100_tx_qdesc_prefix_create(const struct rte_mbuf *m, efx_oword_t *tx_desc)
316 efx_mport_id_t *mport_id =
317 RTE_MBUF_DYNFIELD(m, sfc_dp_mport_offset, efx_mport_id_t *);
319 EFX_POPULATE_OWORD_3(*tx_desc,
320 ESF_GZ_TX_PREFIX_EGRESS_MPORT,
322 ESF_GZ_TX_PREFIX_EGRESS_MPORT_EN, 1,
323 ESF_GZ_TX_DESC_TYPE, ESE_GZ_TX_DESC_TYPE_PREFIX);
327 sfc_ef100_tx_qdesc_cso_inner_l3(uint64_t tx_tunnel)
332 case RTE_MBUF_F_TX_TUNNEL_VXLAN:
333 inner_l3 = ESE_GZ_TX_DESC_CS_INNER_L3_VXLAN;
335 case RTE_MBUF_F_TX_TUNNEL_GENEVE:
336 inner_l3 = ESE_GZ_TX_DESC_CS_INNER_L3_GENEVE;
339 inner_l3 = ESE_GZ_TX_DESC_CS_INNER_L3_OFF;
346 sfc_ef100_tx_qdesc_send_create(const struct rte_mbuf *m, efx_oword_t *tx_desc)
352 uint16_t part_cksum_w;
353 uint16_t l4_offset_w;
355 if ((m->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) == 0) {
356 outer_l3 = (m->ol_flags & RTE_MBUF_F_TX_IP_CKSUM);
357 outer_l4 = (m->ol_flags & RTE_MBUF_F_TX_L4_MASK);
358 inner_l3 = ESE_GZ_TX_DESC_CS_INNER_L3_OFF;
359 partial_en = ESE_GZ_TX_DESC_CSO_PARTIAL_EN_OFF;
363 outer_l3 = (m->ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM);
364 outer_l4 = (m->ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM);
365 inner_l3 = sfc_ef100_tx_qdesc_cso_inner_l3(m->ol_flags &
366 RTE_MBUF_F_TX_TUNNEL_MASK);
368 switch (m->ol_flags & RTE_MBUF_F_TX_L4_MASK) {
369 case RTE_MBUF_F_TX_TCP_CKSUM:
370 partial_en = ESE_GZ_TX_DESC_CSO_PARTIAL_EN_TCP;
371 part_cksum_w = offsetof(struct rte_tcp_hdr, cksum) >> 1;
373 case RTE_MBUF_F_TX_UDP_CKSUM:
374 partial_en = ESE_GZ_TX_DESC_CSO_PARTIAL_EN_UDP;
375 part_cksum_w = offsetof(struct rte_udp_hdr,
379 partial_en = ESE_GZ_TX_DESC_CSO_PARTIAL_EN_OFF;
383 l4_offset_w = (m->outer_l2_len + m->outer_l3_len +
384 m->l2_len + m->l3_len) >> 1;
387 EFX_POPULATE_OWORD_10(*tx_desc,
388 ESF_GZ_TX_SEND_ADDR, rte_mbuf_data_iova(m),
389 ESF_GZ_TX_SEND_LEN, rte_pktmbuf_data_len(m),
390 ESF_GZ_TX_SEND_NUM_SEGS, m->nb_segs,
391 ESF_GZ_TX_SEND_CSO_PARTIAL_START_W, l4_offset_w,
392 ESF_GZ_TX_SEND_CSO_PARTIAL_CSUM_W, part_cksum_w,
393 ESF_GZ_TX_SEND_CSO_PARTIAL_EN, partial_en,
394 ESF_GZ_TX_SEND_CSO_INNER_L3, inner_l3,
395 ESF_GZ_TX_SEND_CSO_OUTER_L3, outer_l3,
396 ESF_GZ_TX_SEND_CSO_OUTER_L4, outer_l4,
397 ESF_GZ_TX_DESC_TYPE, ESE_GZ_TX_DESC_TYPE_SEND);
399 if (m->ol_flags & RTE_MBUF_F_TX_VLAN) {
400 efx_oword_t tx_desc_extra_fields;
402 EFX_POPULATE_OWORD_2(tx_desc_extra_fields,
403 ESF_GZ_TX_SEND_VLAN_INSERT_EN, 1,
404 ESF_GZ_TX_SEND_VLAN_INSERT_TCI, m->vlan_tci);
406 EFX_OR_OWORD(*tx_desc, tx_desc_extra_fields);
411 sfc_ef100_tx_qdesc_seg_create(rte_iova_t addr, uint16_t len,
412 efx_oword_t *tx_desc)
414 EFX_POPULATE_OWORD_3(*tx_desc,
415 ESF_GZ_TX_SEG_ADDR, addr,
416 ESF_GZ_TX_SEG_LEN, len,
417 ESF_GZ_TX_DESC_TYPE, ESE_GZ_TX_DESC_TYPE_SEG);
421 sfc_ef100_tx_qdesc_tso_create(const struct rte_mbuf *m,
422 uint16_t nb_header_descs,
423 uint16_t nb_payload_descs,
424 size_t header_len, size_t payload_len,
425 size_t outer_iph_off, size_t outer_udph_off,
426 size_t iph_off, size_t tcph_off,
427 efx_oword_t *tx_desc)
429 efx_oword_t tx_desc_extra_fields;
430 int ed_outer_udp_len = (outer_udph_off != 0) ? 1 : 0;
431 int ed_outer_ip_len = (outer_iph_off != 0) ? 1 : 0;
432 int ed_outer_ip_id = (outer_iph_off != 0) ?
433 ESE_GZ_TX_DESC_IP4_ID_INC_MOD16 : 0;
435 * If no tunnel encapsulation is present, then the ED_INNER
436 * fields should be used.
438 int ed_inner_ip_id = ESE_GZ_TX_DESC_IP4_ID_INC_MOD16;
439 uint8_t inner_l3 = sfc_ef100_tx_qdesc_cso_inner_l3(
440 m->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK);
442 EFX_POPULATE_OWORD_10(*tx_desc,
443 ESF_GZ_TX_TSO_MSS, m->tso_segsz,
444 ESF_GZ_TX_TSO_HDR_NUM_SEGS, nb_header_descs,
445 ESF_GZ_TX_TSO_PAYLOAD_NUM_SEGS, nb_payload_descs,
446 ESF_GZ_TX_TSO_ED_OUTER_IP4_ID, ed_outer_ip_id,
447 ESF_GZ_TX_TSO_ED_INNER_IP4_ID, ed_inner_ip_id,
448 ESF_GZ_TX_TSO_ED_OUTER_IP_LEN, ed_outer_ip_len,
449 ESF_GZ_TX_TSO_ED_INNER_IP_LEN, 1,
450 ESF_GZ_TX_TSO_ED_OUTER_UDP_LEN, ed_outer_udp_len,
451 ESF_GZ_TX_TSO_HDR_LEN_W, header_len >> 1,
452 ESF_GZ_TX_TSO_PAYLOAD_LEN, payload_len);
454 EFX_POPULATE_OWORD_9(tx_desc_extra_fields,
456 * Outer offsets are required for outer IPv4 ID
457 * and length edits in the case of tunnel TSO.
459 ESF_GZ_TX_TSO_OUTER_L3_OFF_W, outer_iph_off >> 1,
460 ESF_GZ_TX_TSO_OUTER_L4_OFF_W, outer_udph_off >> 1,
462 * Inner offsets are required for inner IPv4 ID
463 * and IP length edits and partial checksum
464 * offload in the case of tunnel TSO.
466 ESF_GZ_TX_TSO_INNER_L3_OFF_W, iph_off >> 1,
467 ESF_GZ_TX_TSO_INNER_L4_OFF_W, tcph_off >> 1,
468 ESF_GZ_TX_TSO_CSO_INNER_L4,
469 inner_l3 != ESE_GZ_TX_DESC_CS_INNER_L3_OFF,
470 ESF_GZ_TX_TSO_CSO_INNER_L3, inner_l3,
472 * Use outer full checksum offloads which do
473 * not require any extra information.
475 ESF_GZ_TX_TSO_CSO_OUTER_L3, 1,
476 ESF_GZ_TX_TSO_CSO_OUTER_L4, 1,
477 ESF_GZ_TX_DESC_TYPE, ESE_GZ_TX_DESC_TYPE_TSO);
479 EFX_OR_OWORD(*tx_desc, tx_desc_extra_fields);
481 if (m->ol_flags & RTE_MBUF_F_TX_VLAN) {
482 EFX_POPULATE_OWORD_2(tx_desc_extra_fields,
483 ESF_GZ_TX_TSO_VLAN_INSERT_EN, 1,
484 ESF_GZ_TX_TSO_VLAN_INSERT_TCI, m->vlan_tci);
486 EFX_OR_OWORD(*tx_desc, tx_desc_extra_fields);
491 sfc_ef100_tx_qpush(struct sfc_ef100_txq *txq, unsigned int added)
495 EFX_POPULATE_DWORD_1(dword, ERF_GZ_TX_RING_PIDX, added & txq->ptr_mask);
497 /* DMA sync to device is not required */
500 * rte_write32() has rte_io_wmb() which guarantees that the STORE
501 * operations (i.e. Rx and event descriptor updates) that precede
502 * the rte_io_wmb() call are visible to NIC before the STORE
503 * operations that follow it (i.e. doorbell write).
505 rte_write32(dword.ed_u32[0], txq->doorbell);
506 txq->dp.dpq.dbells++;
508 sfc_ef100_tx_debug(txq, "TxQ pushed doorbell at pidx %u (added=%u)",
509 EFX_DWORD_FIELD(dword, ERF_GZ_TX_RING_PIDX),
514 sfc_ef100_tx_pkt_descs_max(const struct rte_mbuf *m)
516 unsigned int extra_descs = 0;
518 /** Maximum length of an mbuf segment data */
519 #define SFC_MBUF_SEG_LEN_MAX UINT16_MAX
520 RTE_BUILD_BUG_ON(sizeof(m->data_len) != 2);
522 if (m->ol_flags & RTE_MBUF_F_TX_TCP_SEG) {
523 /* Tx TSO descriptor */
526 * Extra Tx segment descriptor may be required if header
527 * ends in the middle of segment.
532 * mbuf segment cannot be bigger than maximum segment length
533 * and maximum packet length since TSO is not supported yet.
534 * Make sure that the first segment does not need fragmentation
535 * (split into many Tx descriptors).
537 RTE_BUILD_BUG_ON(SFC_EF100_TX_SEND_DESC_LEN_MAX <
538 RTE_MIN((unsigned int)EFX_MAC_PDU_MAX,
539 SFC_MBUF_SEG_LEN_MAX));
542 if (m->ol_flags & sfc_dp_mport_override) {
543 /* Tx override prefix descriptor will be used */
548 * Any segment of scattered packet cannot be bigger than maximum
549 * segment length. Make sure that subsequent segments do not need
550 * fragmentation (split into many Tx descriptors).
552 RTE_BUILD_BUG_ON(SFC_EF100_TX_SEG_DESC_LEN_MAX < SFC_MBUF_SEG_LEN_MAX);
554 return m->nb_segs + extra_descs;
557 static struct rte_mbuf *
558 sfc_ef100_xmit_tso_pkt(struct sfc_ef100_txq * const txq,
559 struct rte_mbuf *m, unsigned int *added)
561 struct rte_mbuf *m_seg = m;
562 unsigned int nb_hdr_descs;
563 unsigned int nb_pld_descs;
564 unsigned int seg_split = 0;
565 unsigned int tso_desc_id;
567 size_t outer_iph_off;
568 size_t outer_udph_off;
572 size_t remaining_hdr_len;
574 if (m->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) {
575 outer_iph_off = m->outer_l2_len;
576 outer_udph_off = outer_iph_off + m->outer_l3_len;
581 iph_off = outer_udph_off + m->l2_len;
582 tcph_off = iph_off + m->l3_len;
583 header_len = tcph_off + m->l4_len;
586 * Remember ID of the TX_TSO descriptor to be filled in.
587 * We can't fill it in right now since we need to calculate
588 * number of header and payload segments first and don't want
589 * to traverse it twice here.
591 tso_desc_id = (*added)++ & txq->ptr_mask;
593 remaining_hdr_len = header_len;
595 id = (*added)++ & txq->ptr_mask;
596 if (rte_pktmbuf_data_len(m_seg) <= remaining_hdr_len) {
597 /* The segment is fully header segment */
598 sfc_ef100_tx_qdesc_seg_create(
599 rte_mbuf_data_iova(m_seg),
600 rte_pktmbuf_data_len(m_seg),
601 &txq->txq_hw_ring[id]);
602 remaining_hdr_len -= rte_pktmbuf_data_len(m_seg);
605 * The segment must be split into header and
608 sfc_ef100_tx_qdesc_seg_create(
609 rte_mbuf_data_iova(m_seg),
611 &txq->txq_hw_ring[id]);
612 SFC_ASSERT(txq->sw_ring[id].mbuf == NULL);
614 id = (*added)++ & txq->ptr_mask;
615 sfc_ef100_tx_qdesc_seg_create(
616 rte_mbuf_data_iova(m_seg) + remaining_hdr_len,
617 rte_pktmbuf_data_len(m_seg) - remaining_hdr_len,
618 &txq->txq_hw_ring[id]);
619 remaining_hdr_len = 0;
622 txq->sw_ring[id].mbuf = m_seg;
624 } while (remaining_hdr_len > 0);
627 * If a segment is split into header and payload segments, added
628 * pointer counts it twice and we should correct it.
630 nb_hdr_descs = ((id - tso_desc_id) & txq->ptr_mask) - seg_split;
631 nb_pld_descs = m->nb_segs - nb_hdr_descs + seg_split;
633 sfc_ef100_tx_qdesc_tso_create(m, nb_hdr_descs, nb_pld_descs, header_len,
634 rte_pktmbuf_pkt_len(m) - header_len,
635 outer_iph_off, outer_udph_off,
637 &txq->txq_hw_ring[tso_desc_id]);
643 sfc_ef100_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
645 struct sfc_ef100_txq * const txq = sfc_ef100_txq_by_dp_txq(tx_queue);
647 unsigned int dma_desc_space;
649 struct rte_mbuf **pktp;
650 struct rte_mbuf **pktp_end;
652 if (unlikely(txq->flags &
653 (SFC_EF100_TXQ_NOT_RUNNING | SFC_EF100_TXQ_EXCEPTION)))
657 dma_desc_space = txq->max_fill_level - (added - txq->completed);
659 reap_done = (dma_desc_space < txq->free_thresh);
661 sfc_ef100_tx_reap(txq);
662 dma_desc_space = txq->max_fill_level - (added - txq->completed);
665 for (pktp = &tx_pkts[0], pktp_end = &tx_pkts[nb_pkts];
668 struct rte_mbuf *m_seg = *pktp;
669 unsigned int pkt_start = added;
672 if (likely(pktp + 1 != pktp_end))
673 rte_mbuf_prefetch_part1(pktp[1]);
675 if (sfc_ef100_tx_pkt_descs_max(m_seg) > dma_desc_space) {
679 /* Push already prepared descriptors before polling */
680 if (added != txq->added) {
681 sfc_ef100_tx_qpush(txq, added);
685 sfc_ef100_tx_reap(txq);
687 dma_desc_space = txq->max_fill_level -
688 (added - txq->completed);
689 if (sfc_ef100_tx_pkt_descs_max(m_seg) > dma_desc_space)
693 if (m_seg->ol_flags & sfc_dp_mport_override) {
694 id = added++ & txq->ptr_mask;
695 sfc_ef100_tx_qdesc_prefix_create(m_seg,
696 &txq->txq_hw_ring[id]);
699 if (m_seg->ol_flags & RTE_MBUF_F_TX_TCP_SEG) {
700 m_seg = sfc_ef100_xmit_tso_pkt(txq, m_seg, &added);
702 id = added++ & txq->ptr_mask;
703 sfc_ef100_tx_qdesc_send_create(m_seg,
704 &txq->txq_hw_ring[id]);
707 * rte_pktmbuf_free() is commonly used in DPDK for
708 * recycling packets - the function checks every
709 * segment's reference counter and returns the
710 * buffer to its pool whenever possible;
711 * nevertheless, freeing mbuf segments one by one
712 * may entail some performance decline;
713 * from this point, sfc_efx_tx_reap() does the same job
714 * on its own and frees buffers in bulks (all mbufs
715 * within a bulk belong to the same pool);
716 * from this perspective, individual segment pointers
717 * must be associated with the corresponding SW
718 * descriptors independently so that only one loop
719 * is sufficient on reap to inspect all the buffers
721 txq->sw_ring[id].mbuf = m_seg;
725 while (m_seg != NULL) {
726 RTE_BUILD_BUG_ON(SFC_MBUF_SEG_LEN_MAX >
727 SFC_EF100_TX_SEG_DESC_LEN_MAX);
729 id = added++ & txq->ptr_mask;
730 sfc_ef100_tx_qdesc_seg_create(rte_mbuf_data_iova(m_seg),
731 rte_pktmbuf_data_len(m_seg),
732 &txq->txq_hw_ring[id]);
733 txq->sw_ring[id].mbuf = m_seg;
737 dma_desc_space -= (added - pkt_start);
739 sfc_pkts_bytes_add(&txq->dp.dpq.stats, 1,
740 rte_pktmbuf_pkt_len(*pktp));
743 if (likely(added != txq->added)) {
744 sfc_ef100_tx_qpush(txq, added);
748 #if SFC_TX_XMIT_PKTS_REAP_AT_LEAST_ONCE
750 sfc_ef100_tx_reap(txq);
753 return pktp - &tx_pkts[0];
756 static sfc_dp_tx_get_dev_info_t sfc_ef100_get_dev_info;
758 sfc_ef100_get_dev_info(struct rte_eth_dev_info *dev_info)
761 * Number of descriptors just defines maximum number of pushed
762 * descriptors (fill level).
764 dev_info->tx_desc_lim.nb_min = 1;
765 dev_info->tx_desc_lim.nb_align = 1;
768 static sfc_dp_tx_qsize_up_rings_t sfc_ef100_tx_qsize_up_rings;
770 sfc_ef100_tx_qsize_up_rings(uint16_t nb_tx_desc,
771 struct sfc_dp_tx_hw_limits *limits,
772 unsigned int *txq_entries,
773 unsigned int *evq_entries,
774 unsigned int *txq_max_fill_level)
777 * rte_ethdev API guarantees that the number meets min, max and
778 * alignment requirements.
780 if (nb_tx_desc <= limits->txq_min_entries)
781 *txq_entries = limits->txq_min_entries;
783 *txq_entries = rte_align32pow2(nb_tx_desc);
785 *evq_entries = *txq_entries;
787 *txq_max_fill_level = RTE_MIN(nb_tx_desc,
788 SFC_EF100_TXQ_LIMIT(*evq_entries));
792 static sfc_dp_tx_qcreate_t sfc_ef100_tx_qcreate;
794 sfc_ef100_tx_qcreate(uint16_t port_id, uint16_t queue_id,
795 const struct rte_pci_addr *pci_addr, int socket_id,
796 const struct sfc_dp_tx_qcreate_info *info,
797 struct sfc_dp_txq **dp_txqp)
799 struct sfc_ef100_txq *txq;
803 if (info->txq_entries != info->evq_entries)
807 txq = rte_zmalloc_socket("sfc-ef100-txq", sizeof(*txq),
808 RTE_CACHE_LINE_SIZE, socket_id);
812 sfc_dp_queue_init(&txq->dp.dpq, port_id, queue_id, pci_addr);
815 txq->sw_ring = rte_calloc_socket("sfc-ef100-txq-sw_ring",
817 sizeof(*txq->sw_ring),
818 RTE_CACHE_LINE_SIZE, socket_id);
819 if (txq->sw_ring == NULL)
820 goto fail_sw_ring_alloc;
822 txq->flags = SFC_EF100_TXQ_NOT_RUNNING;
823 txq->ptr_mask = info->txq_entries - 1;
824 txq->max_fill_level = info->max_fill_level;
825 txq->free_thresh = info->free_thresh;
826 txq->evq_phase_bit_shift = rte_bsf32(info->evq_entries);
827 txq->txq_hw_ring = info->txq_hw_ring;
828 txq->doorbell = (volatile uint8_t *)info->mem_bar +
829 ER_GZ_TX_RING_DOORBELL_OFST +
830 (info->hw_index << info->vi_window_shift);
831 txq->evq_hw_ring = info->evq_hw_ring;
833 txq->tso_tcp_header_offset_limit = info->tso_tcp_header_offset_limit;
834 txq->tso_max_nb_header_descs = info->tso_max_nb_header_descs;
835 txq->tso_max_header_len = info->tso_max_header_len;
836 txq->tso_max_nb_payload_descs = info->tso_max_nb_payload_descs;
837 txq->tso_max_payload_len = info->tso_max_payload_len;
838 txq->tso_max_nb_outgoing_frames = info->tso_max_nb_outgoing_frames;
840 sfc_ef100_tx_debug(txq, "TxQ doorbell is %p", txq->doorbell);
853 static sfc_dp_tx_qdestroy_t sfc_ef100_tx_qdestroy;
855 sfc_ef100_tx_qdestroy(struct sfc_dp_txq *dp_txq)
857 struct sfc_ef100_txq *txq = sfc_ef100_txq_by_dp_txq(dp_txq);
859 rte_free(txq->sw_ring);
863 static sfc_dp_tx_qstart_t sfc_ef100_tx_qstart;
865 sfc_ef100_tx_qstart(struct sfc_dp_txq *dp_txq, unsigned int evq_read_ptr,
866 unsigned int txq_desc_index)
868 struct sfc_ef100_txq *txq = sfc_ef100_txq_by_dp_txq(dp_txq);
870 txq->evq_read_ptr = evq_read_ptr;
871 txq->added = txq->completed = txq_desc_index;
873 txq->flags |= SFC_EF100_TXQ_STARTED;
874 txq->flags &= ~(SFC_EF100_TXQ_NOT_RUNNING | SFC_EF100_TXQ_EXCEPTION);
879 static sfc_dp_tx_qstop_t sfc_ef100_tx_qstop;
881 sfc_ef100_tx_qstop(struct sfc_dp_txq *dp_txq, unsigned int *evq_read_ptr)
883 struct sfc_ef100_txq *txq = sfc_ef100_txq_by_dp_txq(dp_txq);
885 txq->flags |= SFC_EF100_TXQ_NOT_RUNNING;
887 *evq_read_ptr = txq->evq_read_ptr;
890 static sfc_dp_tx_qtx_ev_t sfc_ef100_tx_qtx_ev;
892 sfc_ef100_tx_qtx_ev(struct sfc_dp_txq *dp_txq, unsigned int num_descs)
894 struct sfc_ef100_txq *txq = sfc_ef100_txq_by_dp_txq(dp_txq);
896 SFC_ASSERT(txq->flags & SFC_EF100_TXQ_NOT_RUNNING);
898 sfc_ef100_tx_reap_num_descs(txq, num_descs);
903 static sfc_dp_tx_qreap_t sfc_ef100_tx_qreap;
905 sfc_ef100_tx_qreap(struct sfc_dp_txq *dp_txq)
907 struct sfc_ef100_txq *txq = sfc_ef100_txq_by_dp_txq(dp_txq);
908 unsigned int completed;
910 for (completed = txq->completed; completed != txq->added; ++completed) {
911 struct sfc_ef100_tx_sw_desc *txd;
913 txd = &txq->sw_ring[completed & txq->ptr_mask];
914 if (txd->mbuf != NULL) {
915 rte_pktmbuf_free_seg(txd->mbuf);
920 txq->flags &= ~SFC_EF100_TXQ_STARTED;
924 sfc_ef100_tx_qdesc_npending(struct sfc_ef100_txq *txq)
926 const unsigned int evq_old_read_ptr = txq->evq_read_ptr;
927 unsigned int npending = 0;
930 if (unlikely(txq->flags &
931 (SFC_EF100_TXQ_NOT_RUNNING | SFC_EF100_TXQ_EXCEPTION)))
934 while (sfc_ef100_tx_get_event(txq, &tx_ev))
935 npending += EFX_QWORD_FIELD(tx_ev, ESF_GZ_EV_TXCMPL_NUM_DESC);
938 * The function does not process events, so return event queue read
939 * pointer to the original position to allow the events that were
940 * read to be processed later
942 txq->evq_read_ptr = evq_old_read_ptr;
947 static sfc_dp_tx_qdesc_status_t sfc_ef100_tx_qdesc_status;
949 sfc_ef100_tx_qdesc_status(struct sfc_dp_txq *dp_txq, uint16_t offset)
951 struct sfc_ef100_txq *txq = sfc_ef100_txq_by_dp_txq(dp_txq);
952 unsigned int pushed = txq->added - txq->completed;
954 if (unlikely(offset > txq->ptr_mask))
957 if (unlikely(offset >= txq->max_fill_level))
958 return RTE_ETH_TX_DESC_UNAVAIL;
960 return (offset >= pushed ||
961 offset < sfc_ef100_tx_qdesc_npending(txq)) ?
962 RTE_ETH_TX_DESC_DONE : RTE_ETH_TX_DESC_FULL;
965 struct sfc_dp_tx sfc_ef100_tx = {
967 .name = SFC_KVARG_DATAPATH_EF100,
969 .hw_fw_caps = SFC_DP_HW_FW_CAP_EF100,
971 .features = SFC_DP_TX_FEAT_MULTI_PROCESS |
972 SFC_DP_TX_FEAT_STATS,
973 .dev_offload_capa = 0,
974 .queue_offload_capa = RTE_ETH_TX_OFFLOAD_VLAN_INSERT |
975 RTE_ETH_TX_OFFLOAD_IPV4_CKSUM |
976 RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM |
977 RTE_ETH_TX_OFFLOAD_OUTER_UDP_CKSUM |
978 RTE_ETH_TX_OFFLOAD_UDP_CKSUM |
979 RTE_ETH_TX_OFFLOAD_TCP_CKSUM |
980 RTE_ETH_TX_OFFLOAD_MULTI_SEGS |
981 RTE_ETH_TX_OFFLOAD_TCP_TSO |
982 RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO |
983 RTE_ETH_TX_OFFLOAD_GENEVE_TNL_TSO,
984 .get_dev_info = sfc_ef100_get_dev_info,
985 .qsize_up_rings = sfc_ef100_tx_qsize_up_rings,
986 .qcreate = sfc_ef100_tx_qcreate,
987 .qdestroy = sfc_ef100_tx_qdestroy,
988 .qstart = sfc_ef100_tx_qstart,
989 .qtx_ev = sfc_ef100_tx_qtx_ev,
990 .qstop = sfc_ef100_tx_qstop,
991 .qreap = sfc_ef100_tx_qreap,
992 .qdesc_status = sfc_ef100_tx_qdesc_status,
993 .pkt_prepare = sfc_ef100_tx_prepare_pkts,
994 .pkt_burst = sfc_ef100_xmit_pkts,