4 * Copyright (c) 2016 Solarflare Communications Inc.
7 * This software was jointly developed between OKTET Labs (under contract
8 * for Solarflare) and Solarflare Communications, Inc.
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions are met:
13 * 1. Redistributions of source code must retain the above copyright notice,
14 * this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright notice,
16 * this list of conditions and the following disclaimer in the documentation
17 * and/or other materials provided with the distribution.
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
21 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
23 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
24 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
25 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
26 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
28 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
29 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38 #include "efx_types.h"
40 #include "efx_regs_ef10.h"
42 #include "sfc_dp_tx.h"
43 #include "sfc_tweak.h"
44 #include "sfc_kvargs.h"
47 #define sfc_ef10_tx_err(dpq, ...) \
48 SFC_DP_LOG(SFC_KVARG_DATAPATH_EF10, ERR, dpq, __VA_ARGS__)
50 /** Maximum length of the DMA descriptor data */
51 #define SFC_EF10_TX_DMA_DESC_LEN_MAX \
52 ((1u << ESF_DZ_TX_KER_BYTE_CNT_WIDTH) - 1)
55 * Maximum number of descriptors/buffers in the Tx ring.
56 * It should guarantee that corresponding event queue never overfill.
57 * EF10 native datapath uses event queue of the same size as Tx queue.
58 * Maximum number of events on datapath can be estimated as number of
59 * Tx queue entries (one event per Tx buffer in the worst case) plus
60 * Tx error and flush events.
62 #define SFC_EF10_TXQ_LIMIT(_ndesc) \
63 ((_ndesc) - 1 /* head must not step on tail */ - \
64 (SFC_EF10_EV_PER_CACHE_LINE - 1) /* max unused EvQ entries */ - \
65 1 /* Rx error */ - 1 /* flush */)
67 struct sfc_ef10_tx_sw_desc {
68 struct rte_mbuf *mbuf;
73 #define SFC_EF10_TXQ_STARTED 0x1
74 #define SFC_EF10_TXQ_NOT_RUNNING 0x2
75 #define SFC_EF10_TXQ_EXCEPTION 0x4
77 unsigned int ptr_mask;
79 unsigned int completed;
80 unsigned int max_fill_level;
81 unsigned int free_thresh;
82 unsigned int evq_read_ptr;
83 struct sfc_ef10_tx_sw_desc *sw_ring;
84 efx_qword_t *txq_hw_ring;
85 volatile void *doorbell;
86 efx_qword_t *evq_hw_ring;
88 /* Datapath transmit queue anchor */
92 static inline struct sfc_ef10_txq *
93 sfc_ef10_txq_by_dp_txq(struct sfc_dp_txq *dp_txq)
95 return container_of(dp_txq, struct sfc_ef10_txq, dp);
99 sfc_ef10_tx_get_event(struct sfc_ef10_txq *txq, efx_qword_t *tx_ev)
101 volatile efx_qword_t *evq_hw_ring = txq->evq_hw_ring;
104 * Exception flag is set when reap is done.
105 * It is never done twice per packet burst get and absence of
106 * the flag is checked on burst get entry.
108 SFC_ASSERT((txq->flags & SFC_EF10_TXQ_EXCEPTION) == 0);
110 *tx_ev = evq_hw_ring[txq->evq_read_ptr & txq->ptr_mask];
112 if (!sfc_ef10_ev_present(*tx_ev))
115 if (unlikely(EFX_QWORD_FIELD(*tx_ev, FSF_AZ_EV_CODE) !=
116 FSE_AZ_EV_CODE_TX_EV)) {
118 * Do not move read_ptr to keep the event for exception
119 * handling by the control path.
121 txq->flags |= SFC_EF10_TXQ_EXCEPTION;
122 sfc_ef10_tx_err(&txq->dp.dpq,
123 "TxQ exception at EvQ read ptr %#x",
133 sfc_ef10_tx_process_events(struct sfc_ef10_txq *txq)
135 const unsigned int curr_done = txq->completed - 1;
136 unsigned int anew_done = curr_done;
139 while (sfc_ef10_tx_get_event(txq, &tx_ev)) {
141 * DROP_EVENT is an internal to the NIC, software should
142 * never see it and, therefore, may ignore it.
145 /* Update the latest done descriptor */
146 anew_done = EFX_QWORD_FIELD(tx_ev, ESF_DZ_TX_DESCR_INDX);
148 return (anew_done - curr_done) & txq->ptr_mask;
152 sfc_ef10_tx_reap(struct sfc_ef10_txq *txq)
154 const unsigned int old_read_ptr = txq->evq_read_ptr;
155 const unsigned int ptr_mask = txq->ptr_mask;
156 unsigned int completed = txq->completed;
157 unsigned int pending = completed;
159 pending += sfc_ef10_tx_process_events(txq);
161 if (pending != completed) {
162 struct rte_mbuf *bulk[SFC_TX_REAP_BULK_SIZE];
166 struct sfc_ef10_tx_sw_desc *txd;
169 txd = &txq->sw_ring[completed & ptr_mask];
170 if (txd->mbuf == NULL)
173 m = rte_pktmbuf_prefree_seg(txd->mbuf);
178 if ((nb == RTE_DIM(bulk)) ||
179 ((nb != 0) && (m->pool != bulk[0]->pool))) {
180 rte_mempool_put_bulk(bulk[0]->pool,
186 } while (++completed != pending);
189 rte_mempool_put_bulk(bulk[0]->pool, (void *)bulk, nb);
191 txq->completed = completed;
194 sfc_ef10_ev_qclear(txq->evq_hw_ring, ptr_mask, old_read_ptr,
199 sfc_ef10_tx_qdesc_dma_create(rte_iova_t addr, uint16_t size, bool eop,
202 EFX_POPULATE_QWORD_4(*edp,
203 ESF_DZ_TX_KER_TYPE, 0,
204 ESF_DZ_TX_KER_CONT, !eop,
205 ESF_DZ_TX_KER_BYTE_CNT, size,
206 ESF_DZ_TX_KER_BUF_ADDR, addr);
210 sfc_ef10_tx_qpush(struct sfc_ef10_txq *txq, unsigned int added,
217 * This improves performance by pushing a TX descriptor at the same
218 * time as the doorbell. The descriptor must be added to the TXQ,
219 * so that can be used if the hardware decides not to use the pushed
222 desc.eq_u64[0] = txq->txq_hw_ring[pushed & txq->ptr_mask].eq_u64[0];
223 EFX_POPULATE_OWORD_3(oword,
224 ERF_DZ_TX_DESC_WPTR, added & txq->ptr_mask,
225 ERF_DZ_TX_DESC_HWORD, EFX_QWORD_FIELD(desc, EFX_DWORD_1),
226 ERF_DZ_TX_DESC_LWORD, EFX_QWORD_FIELD(desc, EFX_DWORD_0));
228 /* DMA sync to device is not required */
231 * rte_io_wmb() which guarantees that the STORE operations
232 * (i.e. Tx and event descriptor updates) that precede
233 * the rte_io_wmb() call are visible to NIC before the STORE
234 * operations that follow it (i.e. doorbell write).
238 *(volatile __m128i *)txq->doorbell = oword.eo_u128[0];
242 sfc_ef10_tx_pkt_descs_max(const struct rte_mbuf *m)
244 unsigned int extra_descs_per_seg;
245 unsigned int extra_descs_per_pkt;
248 * VLAN offload is not supported yet, so no extra descriptors
249 * are required for VLAN option descriptor.
252 /** Maximum length of the mbuf segment data */
253 #define SFC_MBUF_SEG_LEN_MAX UINT16_MAX
254 RTE_BUILD_BUG_ON(sizeof(m->data_len) != 2);
257 * Each segment is already counted once below. So, calculate
258 * how many extra DMA descriptors may be required per segment in
259 * the worst case because of maximum DMA descriptor length limit.
260 * If maximum segment length is less or equal to maximum DMA
261 * descriptor length, no extra DMA descriptors are required.
263 extra_descs_per_seg =
264 (SFC_MBUF_SEG_LEN_MAX - 1) / SFC_EF10_TX_DMA_DESC_LEN_MAX;
266 /** Maximum length of the packet */
267 #define SFC_MBUF_PKT_LEN_MAX UINT32_MAX
268 RTE_BUILD_BUG_ON(sizeof(m->pkt_len) != 4);
271 * One more limitation on maximum number of extra DMA descriptors
272 * comes from slicing entire packet because of DMA descriptor length
273 * limit taking into account that there is at least one segment
274 * which is already counted below (so division of the maximum
275 * packet length minus one with round down).
276 * TSO is not supported yet, so packet length is limited by
279 extra_descs_per_pkt =
280 (RTE_MIN((unsigned int)EFX_MAC_PDU_MAX,
281 SFC_MBUF_PKT_LEN_MAX) - 1) /
282 SFC_EF10_TX_DMA_DESC_LEN_MAX;
284 return m->nb_segs + RTE_MIN(m->nb_segs * extra_descs_per_seg,
285 extra_descs_per_pkt);
289 sfc_ef10_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
291 struct sfc_ef10_txq * const txq = sfc_ef10_txq_by_dp_txq(tx_queue);
293 unsigned int dma_desc_space;
295 struct rte_mbuf **pktp;
296 struct rte_mbuf **pktp_end;
298 if (unlikely(txq->flags &
299 (SFC_EF10_TXQ_NOT_RUNNING | SFC_EF10_TXQ_EXCEPTION)))
303 dma_desc_space = txq->max_fill_level - (added - txq->completed);
305 reap_done = (dma_desc_space < txq->free_thresh);
307 sfc_ef10_tx_reap(txq);
308 dma_desc_space = txq->max_fill_level - (added - txq->completed);
311 for (pktp = &tx_pkts[0], pktp_end = &tx_pkts[nb_pkts];
314 struct rte_mbuf *m_seg = *pktp;
315 unsigned int pkt_start = added;
318 if (likely(pktp + 1 != pktp_end))
319 rte_mbuf_prefetch_part1(pktp[1]);
321 if (sfc_ef10_tx_pkt_descs_max(m_seg) > dma_desc_space) {
325 /* Push already prepared descriptors before polling */
326 if (added != txq->added) {
327 sfc_ef10_tx_qpush(txq, added, txq->added);
331 sfc_ef10_tx_reap(txq);
333 dma_desc_space = txq->max_fill_level -
334 (added - txq->completed);
335 if (sfc_ef10_tx_pkt_descs_max(m_seg) > dma_desc_space)
339 pkt_len = m_seg->pkt_len;
341 rte_iova_t seg_addr = rte_mbuf_data_iova(m_seg);
342 unsigned int seg_len = rte_pktmbuf_data_len(m_seg);
343 unsigned int id = added & txq->ptr_mask;
345 SFC_ASSERT(seg_len <= SFC_EF10_TX_DMA_DESC_LEN_MAX);
349 sfc_ef10_tx_qdesc_dma_create(seg_addr,
350 seg_len, (pkt_len == 0),
351 &txq->txq_hw_ring[id]);
354 * rte_pktmbuf_free() is commonly used in DPDK for
355 * recycling packets - the function checks every
356 * segment's reference counter and returns the
357 * buffer to its pool whenever possible;
358 * nevertheless, freeing mbuf segments one by one
359 * may entail some performance decline;
360 * from this point, sfc_efx_tx_reap() does the same job
361 * on its own and frees buffers in bulks (all mbufs
362 * within a bulk belong to the same pool);
363 * from this perspective, individual segment pointers
364 * must be associated with the corresponding SW
365 * descriptors independently so that only one loop
366 * is sufficient on reap to inspect all the buffers
368 txq->sw_ring[id].mbuf = m_seg;
372 } while ((m_seg = m_seg->next) != 0);
374 dma_desc_space -= (added - pkt_start);
377 if (likely(added != txq->added)) {
378 sfc_ef10_tx_qpush(txq, added, txq->added);
382 #if SFC_TX_XMIT_PKTS_REAP_AT_LEAST_ONCE
384 sfc_ef10_tx_reap(txq);
387 return pktp - &tx_pkts[0];
391 sfc_ef10_simple_tx_reap(struct sfc_ef10_txq *txq)
393 const unsigned int old_read_ptr = txq->evq_read_ptr;
394 const unsigned int ptr_mask = txq->ptr_mask;
395 unsigned int completed = txq->completed;
396 unsigned int pending = completed;
398 pending += sfc_ef10_tx_process_events(txq);
400 if (pending != completed) {
401 struct rte_mbuf *bulk[SFC_TX_REAP_BULK_SIZE];
405 struct sfc_ef10_tx_sw_desc *txd;
407 txd = &txq->sw_ring[completed & ptr_mask];
409 if (nb == RTE_DIM(bulk)) {
410 rte_mempool_put_bulk(bulk[0]->pool,
415 bulk[nb++] = txd->mbuf;
416 } while (++completed != pending);
418 rte_mempool_put_bulk(bulk[0]->pool, (void *)bulk, nb);
420 txq->completed = completed;
423 sfc_ef10_ev_qclear(txq->evq_hw_ring, ptr_mask, old_read_ptr,
429 sfc_ef10_simple_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
432 struct sfc_ef10_txq * const txq = sfc_ef10_txq_by_dp_txq(tx_queue);
433 unsigned int ptr_mask;
435 unsigned int dma_desc_space;
437 struct rte_mbuf **pktp;
438 struct rte_mbuf **pktp_end;
440 if (unlikely(txq->flags &
441 (SFC_EF10_TXQ_NOT_RUNNING | SFC_EF10_TXQ_EXCEPTION)))
444 ptr_mask = txq->ptr_mask;
446 dma_desc_space = txq->max_fill_level - (added - txq->completed);
448 reap_done = (dma_desc_space < RTE_MAX(txq->free_thresh, nb_pkts));
450 sfc_ef10_simple_tx_reap(txq);
451 dma_desc_space = txq->max_fill_level - (added - txq->completed);
454 pktp_end = &tx_pkts[MIN(nb_pkts, dma_desc_space)];
455 for (pktp = &tx_pkts[0]; pktp != pktp_end; ++pktp) {
456 struct rte_mbuf *pkt = *pktp;
457 unsigned int id = added & ptr_mask;
459 SFC_ASSERT(rte_pktmbuf_data_len(pkt) <=
460 SFC_EF10_TX_DMA_DESC_LEN_MAX);
462 sfc_ef10_tx_qdesc_dma_create(rte_mbuf_data_iova(pkt),
463 rte_pktmbuf_data_len(pkt),
464 true, &txq->txq_hw_ring[id]);
466 txq->sw_ring[id].mbuf = pkt;
471 if (likely(added != txq->added)) {
472 sfc_ef10_tx_qpush(txq, added, txq->added);
476 #if SFC_TX_XMIT_PKTS_REAP_AT_LEAST_ONCE
478 sfc_ef10_simple_tx_reap(txq);
481 return pktp - &tx_pkts[0];
484 static sfc_dp_tx_get_dev_info_t sfc_ef10_get_dev_info;
486 sfc_ef10_get_dev_info(struct rte_eth_dev_info *dev_info)
489 * Number of descriptors just defines maximum number of pushed
490 * descriptors (fill level).
492 dev_info->tx_desc_lim.nb_min = 1;
493 dev_info->tx_desc_lim.nb_align = 1;
496 static sfc_dp_tx_qsize_up_rings_t sfc_ef10_tx_qsize_up_rings;
498 sfc_ef10_tx_qsize_up_rings(uint16_t nb_tx_desc,
499 unsigned int *txq_entries,
500 unsigned int *evq_entries,
501 unsigned int *txq_max_fill_level)
504 * rte_ethdev API guarantees that the number meets min, max and
505 * alignment requirements.
507 if (nb_tx_desc <= EFX_TXQ_MINNDESCS)
508 *txq_entries = EFX_TXQ_MINNDESCS;
510 *txq_entries = rte_align32pow2(nb_tx_desc);
512 *evq_entries = *txq_entries;
514 *txq_max_fill_level = RTE_MIN(nb_tx_desc,
515 SFC_EF10_TXQ_LIMIT(*evq_entries));
519 static sfc_dp_tx_qcreate_t sfc_ef10_tx_qcreate;
521 sfc_ef10_tx_qcreate(uint16_t port_id, uint16_t queue_id,
522 const struct rte_pci_addr *pci_addr, int socket_id,
523 const struct sfc_dp_tx_qcreate_info *info,
524 struct sfc_dp_txq **dp_txqp)
526 struct sfc_ef10_txq *txq;
530 if (info->txq_entries != info->evq_entries)
534 txq = rte_zmalloc_socket("sfc-ef10-txq", sizeof(*txq),
535 RTE_CACHE_LINE_SIZE, socket_id);
539 sfc_dp_queue_init(&txq->dp.dpq, port_id, queue_id, pci_addr);
542 txq->sw_ring = rte_calloc_socket("sfc-ef10-txq-sw_ring",
544 sizeof(*txq->sw_ring),
545 RTE_CACHE_LINE_SIZE, socket_id);
546 if (txq->sw_ring == NULL)
547 goto fail_sw_ring_alloc;
549 txq->flags = SFC_EF10_TXQ_NOT_RUNNING;
550 txq->ptr_mask = info->txq_entries - 1;
551 txq->max_fill_level = info->max_fill_level;
552 txq->free_thresh = info->free_thresh;
553 txq->txq_hw_ring = info->txq_hw_ring;
554 txq->doorbell = (volatile uint8_t *)info->mem_bar +
555 ER_DZ_TX_DESC_UPD_REG_OFST +
556 info->hw_index * ER_DZ_TX_DESC_UPD_REG_STEP;
557 txq->evq_hw_ring = info->evq_hw_ring;
570 static sfc_dp_tx_qdestroy_t sfc_ef10_tx_qdestroy;
572 sfc_ef10_tx_qdestroy(struct sfc_dp_txq *dp_txq)
574 struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq);
576 rte_free(txq->sw_ring);
580 static sfc_dp_tx_qstart_t sfc_ef10_tx_qstart;
582 sfc_ef10_tx_qstart(struct sfc_dp_txq *dp_txq, unsigned int evq_read_ptr,
583 unsigned int txq_desc_index)
585 struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq);
587 txq->evq_read_ptr = evq_read_ptr;
588 txq->added = txq->completed = txq_desc_index;
590 txq->flags |= SFC_EF10_TXQ_STARTED;
591 txq->flags &= ~(SFC_EF10_TXQ_NOT_RUNNING | SFC_EF10_TXQ_EXCEPTION);
596 static sfc_dp_tx_qstop_t sfc_ef10_tx_qstop;
598 sfc_ef10_tx_qstop(struct sfc_dp_txq *dp_txq, unsigned int *evq_read_ptr)
600 struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq);
602 txq->flags |= SFC_EF10_TXQ_NOT_RUNNING;
604 *evq_read_ptr = txq->evq_read_ptr;
607 static sfc_dp_tx_qtx_ev_t sfc_ef10_tx_qtx_ev;
609 sfc_ef10_tx_qtx_ev(struct sfc_dp_txq *dp_txq, __rte_unused unsigned int id)
611 __rte_unused struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq);
613 SFC_ASSERT(txq->flags & SFC_EF10_TXQ_NOT_RUNNING);
616 * It is safe to ignore Tx event since we reap all mbufs on
617 * queue purge anyway.
623 static sfc_dp_tx_qreap_t sfc_ef10_tx_qreap;
625 sfc_ef10_tx_qreap(struct sfc_dp_txq *dp_txq)
627 struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq);
628 unsigned int completed;
630 for (completed = txq->completed; completed != txq->added; ++completed) {
631 struct sfc_ef10_tx_sw_desc *txd;
633 txd = &txq->sw_ring[completed & txq->ptr_mask];
634 if (txd->mbuf != NULL) {
635 rte_pktmbuf_free_seg(txd->mbuf);
640 txq->flags &= ~SFC_EF10_TXQ_STARTED;
643 static sfc_dp_tx_qdesc_status_t sfc_ef10_tx_qdesc_status;
645 sfc_ef10_tx_qdesc_status(__rte_unused struct sfc_dp_txq *dp_txq,
646 __rte_unused uint16_t offset)
651 struct sfc_dp_tx sfc_ef10_tx = {
653 .name = SFC_KVARG_DATAPATH_EF10,
655 .hw_fw_caps = SFC_DP_HW_FW_CAP_EF10,
657 .features = SFC_DP_TX_FEAT_MULTI_SEG |
658 SFC_DP_TX_FEAT_MULTI_POOL |
659 SFC_DP_TX_FEAT_REFCNT |
660 SFC_DP_TX_FEAT_MULTI_PROCESS,
661 .get_dev_info = sfc_ef10_get_dev_info,
662 .qsize_up_rings = sfc_ef10_tx_qsize_up_rings,
663 .qcreate = sfc_ef10_tx_qcreate,
664 .qdestroy = sfc_ef10_tx_qdestroy,
665 .qstart = sfc_ef10_tx_qstart,
666 .qtx_ev = sfc_ef10_tx_qtx_ev,
667 .qstop = sfc_ef10_tx_qstop,
668 .qreap = sfc_ef10_tx_qreap,
669 .qdesc_status = sfc_ef10_tx_qdesc_status,
670 .pkt_burst = sfc_ef10_xmit_pkts,
673 struct sfc_dp_tx sfc_ef10_simple_tx = {
675 .name = SFC_KVARG_DATAPATH_EF10_SIMPLE,
678 .features = SFC_DP_TX_FEAT_MULTI_PROCESS,
679 .get_dev_info = sfc_ef10_get_dev_info,
680 .qsize_up_rings = sfc_ef10_tx_qsize_up_rings,
681 .qcreate = sfc_ef10_tx_qcreate,
682 .qdestroy = sfc_ef10_tx_qdestroy,
683 .qstart = sfc_ef10_tx_qstart,
684 .qtx_ev = sfc_ef10_tx_qtx_ev,
685 .qstop = sfc_ef10_tx_qstop,
686 .qreap = sfc_ef10_tx_qreap,
687 .qdesc_status = sfc_ef10_tx_qdesc_status,
688 .pkt_burst = sfc_ef10_simple_xmit_pkts,