1 /* SPDX-License-Identifier: BSD-3-Clause
3 * Copyright (c) 2016-2018 Solarflare Communications Inc.
6 * This software was jointly developed between OKTET Labs (under contract
7 * for Solarflare) and Solarflare Communications, Inc.
11 #include "sfc_debug.h"
15 #include "sfc_tweak.h"
16 #include "sfc_kvargs.h"
19 * Maximum number of TX queue flush attempts in case of
20 * failure or flush timeout
22 #define SFC_TX_QFLUSH_ATTEMPTS (3)
25 * Time to wait between event queue polling attempts when waiting for TX
26 * queue flush done or flush failed events
28 #define SFC_TX_QFLUSH_POLL_WAIT_MS (1)
31 * Maximum number of event queue polling attempts when waiting for TX queue
32 * flush done or flush failed events; it defines TX queue flush attempt timeout
33 * together with SFC_TX_QFLUSH_POLL_WAIT_MS
35 #define SFC_TX_QFLUSH_POLL_ATTEMPTS (2000)
38 sfc_tx_get_dev_offload_caps(struct sfc_adapter *sa)
40 const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic);
43 caps |= DEV_TX_OFFLOAD_IPV4_CKSUM;
44 caps |= DEV_TX_OFFLOAD_UDP_CKSUM;
45 caps |= DEV_TX_OFFLOAD_TCP_CKSUM;
47 if (encp->enc_tunnel_encapsulations_supported)
48 caps |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM;
50 if ((sa->dp_tx->features & SFC_DP_TX_FEAT_VLAN_INSERT) &&
51 encp->enc_hw_tx_insert_vlan_enabled)
52 caps |= DEV_TX_OFFLOAD_VLAN_INSERT;
55 caps |= DEV_TX_OFFLOAD_TCP_TSO;
61 sfc_tx_qcheck_conf(struct sfc_adapter *sa, unsigned int txq_max_fill_level,
62 const struct rte_eth_txconf *tx_conf)
64 unsigned int flags = tx_conf->txq_flags;
65 const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic);
68 if (tx_conf->tx_rs_thresh != 0) {
69 sfc_err(sa, "RS bit in transmit descriptor is not supported");
73 if (tx_conf->tx_free_thresh > txq_max_fill_level) {
75 "TxQ free threshold too large: %u vs maximum %u",
76 tx_conf->tx_free_thresh, txq_max_fill_level);
80 if (tx_conf->tx_thresh.pthresh != 0 ||
81 tx_conf->tx_thresh.hthresh != 0 ||
82 tx_conf->tx_thresh.wthresh != 0) {
84 "prefetch/host/writeback thresholds are not supported");
87 if (((flags & ETH_TXQ_FLAGS_NOMULTSEGS) == 0) &&
88 (~sa->dp_tx->features & SFC_DP_TX_FEAT_MULTI_SEG)) {
89 sfc_err(sa, "Multi-segment is not supported by %s datapath",
94 if (((flags & ETH_TXQ_FLAGS_NOMULTMEMP) == 0) &&
95 (~sa->dp_tx->features & SFC_DP_TX_FEAT_MULTI_POOL)) {
96 sfc_err(sa, "multi-mempool is not supported by %s datapath",
101 if (((flags & ETH_TXQ_FLAGS_NOREFCOUNT) == 0) &&
102 (~sa->dp_tx->features & SFC_DP_TX_FEAT_REFCNT)) {
104 "mbuf reference counters are neglected by %s datapath",
109 if ((flags & ETH_TXQ_FLAGS_NOVLANOFFL) == 0) {
110 if (!encp->enc_hw_tx_insert_vlan_enabled) {
111 sfc_err(sa, "VLAN offload is not supported");
113 } else if (~sa->dp_tx->features & SFC_DP_TX_FEAT_VLAN_INSERT) {
115 "VLAN offload is not supported by %s datapath",
121 if ((flags & ETH_TXQ_FLAGS_NOXSUMSCTP) == 0) {
122 sfc_err(sa, "SCTP offload is not supported");
126 /* We either perform both TCP and UDP offload, or no offload at all */
127 if (((flags & ETH_TXQ_FLAGS_NOXSUMTCP) == 0) !=
128 ((flags & ETH_TXQ_FLAGS_NOXSUMUDP) == 0)) {
129 sfc_err(sa, "TCP and UDP offloads can't be set independently");
137 sfc_tx_qflush_done(struct sfc_txq *txq)
139 txq->state |= SFC_TXQ_FLUSHED;
140 txq->state &= ~SFC_TXQ_FLUSHING;
144 sfc_tx_qinit(struct sfc_adapter *sa, unsigned int sw_index,
145 uint16_t nb_tx_desc, unsigned int socket_id,
146 const struct rte_eth_txconf *tx_conf)
148 const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic);
149 unsigned int txq_entries;
150 unsigned int evq_entries;
151 unsigned int txq_max_fill_level;
152 struct sfc_txq_info *txq_info;
156 struct sfc_dp_tx_qcreate_info info;
158 sfc_log_init(sa, "TxQ = %u", sw_index);
160 rc = sa->dp_tx->qsize_up_rings(nb_tx_desc, &txq_entries, &evq_entries,
161 &txq_max_fill_level);
163 goto fail_size_up_rings;
164 SFC_ASSERT(txq_entries >= EFX_TXQ_MINNDESCS);
165 SFC_ASSERT(txq_entries <= sa->txq_max_entries);
166 SFC_ASSERT(txq_entries >= nb_tx_desc);
167 SFC_ASSERT(txq_max_fill_level <= nb_tx_desc);
169 rc = sfc_tx_qcheck_conf(sa, txq_max_fill_level, tx_conf);
173 SFC_ASSERT(sw_index < sa->txq_count);
174 txq_info = &sa->txq_info[sw_index];
176 txq_info->entries = txq_entries;
178 rc = sfc_ev_qinit(sa, SFC_EVQ_TYPE_TX, sw_index,
179 evq_entries, socket_id, &evq);
184 txq = rte_zmalloc_socket("sfc-txq", sizeof(*txq), 0, socket_id);
190 txq->hw_index = sw_index;
193 (tx_conf->tx_free_thresh) ? tx_conf->tx_free_thresh :
194 SFC_TX_DEFAULT_FREE_THRESH;
195 txq->flags = tx_conf->txq_flags;
197 rc = sfc_dma_alloc(sa, "txq", sw_index, EFX_TXQ_SIZE(txq_info->entries),
198 socket_id, &txq->mem);
202 memset(&info, 0, sizeof(info));
203 info.max_fill_level = txq_max_fill_level;
204 info.free_thresh = txq->free_thresh;
205 info.flags = tx_conf->txq_flags;
206 info.txq_entries = txq_info->entries;
207 info.dma_desc_size_max = encp->enc_tx_dma_desc_size_max;
208 info.txq_hw_ring = txq->mem.esm_base;
209 info.evq_entries = evq_entries;
210 info.evq_hw_ring = evq->mem.esm_base;
211 info.hw_index = txq->hw_index;
212 info.mem_bar = sa->mem_bar.esb_base;
214 rc = sa->dp_tx->qcreate(sa->eth_dev->data->port_id, sw_index,
215 &RTE_ETH_DEV_TO_PCI(sa->eth_dev)->addr,
216 socket_id, &info, &txq->dp);
218 goto fail_dp_tx_qinit;
220 evq->dp_txq = txq->dp;
222 txq->state = SFC_TXQ_INITIALIZED;
224 txq_info->deferred_start = (tx_conf->tx_deferred_start != 0);
229 sfc_dma_free(sa, &txq->mem);
232 txq_info->txq = NULL;
239 txq_info->entries = 0;
243 sfc_log_init(sa, "failed (TxQ = %u, rc = %d)", sw_index, rc);
248 sfc_tx_qfini(struct sfc_adapter *sa, unsigned int sw_index)
250 struct sfc_txq_info *txq_info;
253 sfc_log_init(sa, "TxQ = %u", sw_index);
255 SFC_ASSERT(sw_index < sa->txq_count);
256 txq_info = &sa->txq_info[sw_index];
259 SFC_ASSERT(txq != NULL);
260 SFC_ASSERT(txq->state == SFC_TXQ_INITIALIZED);
262 sa->dp_tx->qdestroy(txq->dp);
265 txq_info->txq = NULL;
266 txq_info->entries = 0;
268 sfc_dma_free(sa, &txq->mem);
270 sfc_ev_qfini(txq->evq);
277 sfc_tx_qinit_info(struct sfc_adapter *sa, unsigned int sw_index)
279 sfc_log_init(sa, "TxQ = %u", sw_index);
285 sfc_tx_check_mode(struct sfc_adapter *sa, const struct rte_eth_txmode *txmode)
289 switch (txmode->mq_mode) {
293 sfc_err(sa, "Tx multi-queue mode %u not supported",
299 * These features are claimed to be i40e-specific,
300 * but it does make sense to double-check their absence
302 if (txmode->hw_vlan_reject_tagged) {
303 sfc_err(sa, "Rejecting tagged packets not supported");
307 if (txmode->hw_vlan_reject_untagged) {
308 sfc_err(sa, "Rejecting untagged packets not supported");
312 if (txmode->hw_vlan_insert_pvid) {
313 sfc_err(sa, "Port-based VLAN insertion not supported");
321 * Destroy excess queues that are no longer needed after reconfiguration
325 sfc_tx_fini_queues(struct sfc_adapter *sa, unsigned int nb_tx_queues)
329 SFC_ASSERT(nb_tx_queues <= sa->txq_count);
331 sw_index = sa->txq_count;
332 while (--sw_index >= (int)nb_tx_queues) {
333 if (sa->txq_info[sw_index].txq != NULL)
334 sfc_tx_qfini(sa, sw_index);
337 sa->txq_count = nb_tx_queues;
341 sfc_tx_configure(struct sfc_adapter *sa)
343 const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic);
344 const struct rte_eth_conf *dev_conf = &sa->eth_dev->data->dev_conf;
345 const unsigned int nb_tx_queues = sa->eth_dev->data->nb_tx_queues;
348 sfc_log_init(sa, "nb_tx_queues=%u (old %u)",
349 nb_tx_queues, sa->txq_count);
352 * The datapath implementation assumes absence of boundary
353 * limits on Tx DMA descriptors. Addition of these checks on
354 * datapath would simply make the datapath slower.
356 if (encp->enc_tx_dma_desc_boundary != 0) {
358 goto fail_tx_dma_desc_boundary;
361 rc = sfc_tx_check_mode(sa, &dev_conf->txmode);
363 goto fail_check_mode;
365 if (nb_tx_queues == sa->txq_count)
368 if (sa->txq_info == NULL) {
369 sa->txq_info = rte_calloc_socket("sfc-txqs", nb_tx_queues,
370 sizeof(sa->txq_info[0]), 0,
372 if (sa->txq_info == NULL)
373 goto fail_txqs_alloc;
375 struct sfc_txq_info *new_txq_info;
377 if (nb_tx_queues < sa->txq_count)
378 sfc_tx_fini_queues(sa, nb_tx_queues);
381 rte_realloc(sa->txq_info,
382 nb_tx_queues * sizeof(sa->txq_info[0]), 0);
383 if (new_txq_info == NULL && nb_tx_queues > 0)
384 goto fail_txqs_realloc;
386 sa->txq_info = new_txq_info;
387 if (nb_tx_queues > sa->txq_count)
388 memset(&sa->txq_info[sa->txq_count], 0,
389 (nb_tx_queues - sa->txq_count) *
390 sizeof(sa->txq_info[0]));
393 while (sa->txq_count < nb_tx_queues) {
394 rc = sfc_tx_qinit_info(sa, sa->txq_count);
396 goto fail_tx_qinit_info;
410 fail_tx_dma_desc_boundary:
411 sfc_log_init(sa, "failed (rc = %d)", rc);
416 sfc_tx_close(struct sfc_adapter *sa)
418 sfc_tx_fini_queues(sa, 0);
420 rte_free(sa->txq_info);
425 sfc_tx_qstart(struct sfc_adapter *sa, unsigned int sw_index)
427 const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic);
428 struct rte_eth_dev_data *dev_data;
429 struct sfc_txq_info *txq_info;
433 unsigned int desc_index;
436 sfc_log_init(sa, "TxQ = %u", sw_index);
438 SFC_ASSERT(sw_index < sa->txq_count);
439 txq_info = &sa->txq_info[sw_index];
443 SFC_ASSERT(txq->state == SFC_TXQ_INITIALIZED);
447 rc = sfc_ev_qstart(evq, sfc_evq_index_by_txq_sw_index(sa, sw_index));
452 * It seems that DPDK has no controls regarding IPv4 offloads,
453 * hence, we always enable it here
455 if ((txq->flags & ETH_TXQ_FLAGS_NOXSUMTCP) ||
456 (txq->flags & ETH_TXQ_FLAGS_NOXSUMUDP)) {
457 flags = EFX_TXQ_CKSUM_IPV4;
459 if (encp->enc_tunnel_encapsulations_supported != 0)
460 flags |= EFX_TXQ_CKSUM_INNER_IPV4;
462 flags = EFX_TXQ_CKSUM_IPV4 | EFX_TXQ_CKSUM_TCPUDP;
464 if (encp->enc_tunnel_encapsulations_supported != 0)
465 flags |= EFX_TXQ_CKSUM_INNER_IPV4 |
466 EFX_TXQ_CKSUM_INNER_TCPUDP;
469 flags |= EFX_TXQ_FATSOV2;
472 rc = efx_tx_qcreate(sa->nic, sw_index, 0, &txq->mem,
473 txq_info->entries, 0 /* not used on EF10 */,
475 &txq->common, &desc_index);
477 if (sa->tso && (rc == ENOSPC))
478 sfc_err(sa, "ran out of TSO contexts");
480 goto fail_tx_qcreate;
483 efx_tx_qenable(txq->common);
485 txq->state |= SFC_TXQ_STARTED;
487 rc = sa->dp_tx->qstart(txq->dp, evq->read_ptr, desc_index);
492 * It seems to be used by DPDK for debug purposes only ('rte_ether')
494 dev_data = sa->eth_dev->data;
495 dev_data->tx_queue_state[sw_index] = RTE_ETH_QUEUE_STATE_STARTED;
500 txq->state = SFC_TXQ_INITIALIZED;
501 efx_tx_qdestroy(txq->common);
511 sfc_tx_qstop(struct sfc_adapter *sa, unsigned int sw_index)
513 struct rte_eth_dev_data *dev_data;
514 struct sfc_txq_info *txq_info;
516 unsigned int retry_count;
517 unsigned int wait_count;
520 sfc_log_init(sa, "TxQ = %u", sw_index);
522 SFC_ASSERT(sw_index < sa->txq_count);
523 txq_info = &sa->txq_info[sw_index];
527 if (txq->state == SFC_TXQ_INITIALIZED)
530 SFC_ASSERT(txq->state & SFC_TXQ_STARTED);
532 sa->dp_tx->qstop(txq->dp, &txq->evq->read_ptr);
535 * Retry TX queue flushing in case of flush failed or
536 * timeout; in the worst case it can delay for 6 seconds
538 for (retry_count = 0;
539 ((txq->state & SFC_TXQ_FLUSHED) == 0) &&
540 (retry_count < SFC_TX_QFLUSH_ATTEMPTS);
542 rc = efx_tx_qflush(txq->common);
544 txq->state |= (rc == EALREADY) ?
545 SFC_TXQ_FLUSHED : SFC_TXQ_FLUSH_FAILED;
550 * Wait for TX queue flush done or flush failed event at least
551 * SFC_TX_QFLUSH_POLL_WAIT_MS milliseconds and not more
552 * than 2 seconds (SFC_TX_QFLUSH_POLL_WAIT_MS multiplied
553 * by SFC_TX_QFLUSH_POLL_ATTEMPTS)
557 rte_delay_ms(SFC_TX_QFLUSH_POLL_WAIT_MS);
558 sfc_ev_qpoll(txq->evq);
559 } while ((txq->state & SFC_TXQ_FLUSHING) &&
560 wait_count++ < SFC_TX_QFLUSH_POLL_ATTEMPTS);
562 if (txq->state & SFC_TXQ_FLUSHING)
563 sfc_err(sa, "TxQ %u flush timed out", sw_index);
565 if (txq->state & SFC_TXQ_FLUSHED)
566 sfc_info(sa, "TxQ %u flushed", sw_index);
569 sa->dp_tx->qreap(txq->dp);
571 txq->state = SFC_TXQ_INITIALIZED;
573 efx_tx_qdestroy(txq->common);
575 sfc_ev_qstop(txq->evq);
578 * It seems to be used by DPDK for debug purposes only ('rte_ether')
580 dev_data = sa->eth_dev->data;
581 dev_data->tx_queue_state[sw_index] = RTE_ETH_QUEUE_STATE_STOPPED;
585 sfc_tx_start(struct sfc_adapter *sa)
587 unsigned int sw_index;
590 sfc_log_init(sa, "txq_count = %u", sa->txq_count);
593 if (!efx_nic_cfg_get(sa->nic)->enc_fw_assisted_tso_v2_enabled) {
594 sfc_warn(sa, "TSO support was unable to be restored");
599 rc = efx_tx_init(sa->nic);
601 goto fail_efx_tx_init;
603 for (sw_index = 0; sw_index < sa->txq_count; ++sw_index) {
604 if (!(sa->txq_info[sw_index].deferred_start) ||
605 sa->txq_info[sw_index].deferred_started) {
606 rc = sfc_tx_qstart(sa, sw_index);
615 while (sw_index-- > 0)
616 sfc_tx_qstop(sa, sw_index);
618 efx_tx_fini(sa->nic);
621 sfc_log_init(sa, "failed (rc = %d)", rc);
626 sfc_tx_stop(struct sfc_adapter *sa)
628 unsigned int sw_index;
630 sfc_log_init(sa, "txq_count = %u", sa->txq_count);
632 sw_index = sa->txq_count;
633 while (sw_index-- > 0) {
634 if (sa->txq_info[sw_index].txq != NULL)
635 sfc_tx_qstop(sa, sw_index);
638 efx_tx_fini(sa->nic);
642 sfc_efx_tx_reap(struct sfc_efx_txq *txq)
644 unsigned int completed;
646 sfc_ev_qpoll(txq->evq);
648 for (completed = txq->completed;
649 completed != txq->pending; completed++) {
650 struct sfc_efx_tx_sw_desc *txd;
652 txd = &txq->sw_ring[completed & txq->ptr_mask];
654 if (txd->mbuf != NULL) {
655 rte_pktmbuf_free(txd->mbuf);
660 txq->completed = completed;
664 * The function is used to insert or update VLAN tag;
665 * the firmware has state of the firmware tag to insert per TxQ
666 * (controlled by option descriptors), hence, if the tag of the
667 * packet to be sent is different from one remembered by the firmware,
668 * the function will update it
671 sfc_efx_tx_maybe_insert_tag(struct sfc_efx_txq *txq, struct rte_mbuf *m,
674 uint16_t this_tag = ((m->ol_flags & PKT_TX_VLAN_PKT) ?
677 if (this_tag == txq->hw_vlan_tci)
681 * The expression inside SFC_ASSERT() is not desired to be checked in
682 * a non-debug build because it might be too expensive on the data path
684 SFC_ASSERT(efx_nic_cfg_get(txq->evq->sa->nic)->enc_hw_tx_insert_vlan_enabled);
686 efx_tx_qdesc_vlantci_create(txq->common, rte_cpu_to_be_16(this_tag),
689 txq->hw_vlan_tci = this_tag;
695 sfc_efx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
697 struct sfc_dp_txq *dp_txq = (struct sfc_dp_txq *)tx_queue;
698 struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq);
699 unsigned int added = txq->added;
700 unsigned int pushed = added;
701 unsigned int pkts_sent = 0;
702 efx_desc_t *pend = &txq->pend_desc[0];
703 const unsigned int hard_max_fill = txq->max_fill_level;
704 const unsigned int soft_max_fill = hard_max_fill - txq->free_thresh;
705 unsigned int fill_level = added - txq->completed;
708 struct rte_mbuf **pktp;
710 if (unlikely((txq->flags & SFC_EFX_TXQ_FLAG_RUNNING) == 0))
714 * If insufficient space for a single packet is present,
715 * we should reap; otherwise, we shouldn't do that all the time
716 * to avoid latency increase
718 reap_done = (fill_level > soft_max_fill);
721 sfc_efx_tx_reap(txq);
723 * Recalculate fill level since 'txq->completed'
724 * might have changed on reap
726 fill_level = added - txq->completed;
729 for (pkts_sent = 0, pktp = &tx_pkts[0];
730 (pkts_sent < nb_pkts) && (fill_level <= soft_max_fill);
731 pkts_sent++, pktp++) {
732 struct rte_mbuf *m_seg = *pktp;
733 size_t pkt_len = m_seg->pkt_len;
734 unsigned int pkt_descs = 0;
738 * Here VLAN TCI is expected to be zero in case if no
739 * DEV_TX_VLAN_OFFLOAD capability is advertised;
740 * if the calling app ignores the absence of
741 * DEV_TX_VLAN_OFFLOAD and pushes VLAN TCI, then
742 * TX_ERROR will occur
744 pkt_descs += sfc_efx_tx_maybe_insert_tag(txq, m_seg, &pend);
746 if (m_seg->ol_flags & PKT_TX_TCP_SEG) {
748 * We expect correct 'pkt->l[2, 3, 4]_len' values
749 * to be set correctly by the caller
751 if (sfc_efx_tso_do(txq, added, &m_seg, &in_off, &pend,
752 &pkt_descs, &pkt_len) != 0) {
753 /* We may have reached this place for
754 * one of the following reasons:
756 * 1) Packet header length is greater
757 * than SFC_TSOH_STD_LEN
758 * 2) TCP header starts at more then
759 * 208 bytes into the frame
761 * We will deceive RTE saying that we have sent
762 * the packet, but we will actually drop it.
763 * Hence, we should revert 'pend' to the
764 * previous state (in case we have added
765 * VLAN descriptor) and start processing
766 * another one packet. But the original
767 * mbuf shouldn't be orphaned
771 rte_pktmbuf_free(*pktp);
777 * We've only added 2 FATSOv2 option descriptors
778 * and 1 descriptor for the linearized packet header.
779 * The outstanding work will be done in the same manner
780 * as for the usual non-TSO path
784 for (; m_seg != NULL; m_seg = m_seg->next) {
785 efsys_dma_addr_t next_frag;
788 seg_len = m_seg->data_len;
789 next_frag = rte_mbuf_data_iova(m_seg);
792 * If we've started TSO transaction few steps earlier,
793 * we'll skip packet header using an offset in the
794 * current segment (which has been set to the
795 * first one containing payload)
802 efsys_dma_addr_t frag_addr = next_frag;
806 * It is assumed here that there is no
807 * limitation on address boundary
808 * crossing by DMA descriptor.
810 frag_len = MIN(seg_len, txq->dma_desc_size_max);
811 next_frag += frag_len;
815 efx_tx_qdesc_dma_create(txq->common,
821 } while (seg_len != 0);
826 fill_level += pkt_descs;
827 if (unlikely(fill_level > hard_max_fill)) {
829 * Our estimation for maximum number of descriptors
830 * required to send a packet seems to be wrong.
831 * Try to reap (if we haven't yet).
834 sfc_efx_tx_reap(txq);
836 fill_level = added - txq->completed;
837 if (fill_level > hard_max_fill) {
847 /* Assign mbuf to the last used desc */
848 txq->sw_ring[(added - 1) & txq->ptr_mask].mbuf = *pktp;
851 if (likely(pkts_sent > 0)) {
852 rc = efx_tx_qdesc_post(txq->common, txq->pend_desc,
853 pend - &txq->pend_desc[0],
854 txq->completed, &txq->added);
857 if (likely(pushed != txq->added))
858 efx_tx_qpush(txq->common, txq->added, pushed);
861 #if SFC_TX_XMIT_PKTS_REAP_AT_LEAST_ONCE
863 sfc_efx_tx_reap(txq);
871 sfc_txq_by_dp_txq(const struct sfc_dp_txq *dp_txq)
873 const struct sfc_dp_queue *dpq = &dp_txq->dpq;
874 struct rte_eth_dev *eth_dev;
875 struct sfc_adapter *sa;
878 SFC_ASSERT(rte_eth_dev_is_valid_port(dpq->port_id));
879 eth_dev = &rte_eth_devices[dpq->port_id];
881 sa = eth_dev->data->dev_private;
883 SFC_ASSERT(dpq->queue_id < sa->txq_count);
884 txq = sa->txq_info[dpq->queue_id].txq;
886 SFC_ASSERT(txq != NULL);
890 static sfc_dp_tx_qsize_up_rings_t sfc_efx_tx_qsize_up_rings;
892 sfc_efx_tx_qsize_up_rings(uint16_t nb_tx_desc,
893 unsigned int *txq_entries,
894 unsigned int *evq_entries,
895 unsigned int *txq_max_fill_level)
897 *txq_entries = nb_tx_desc;
898 *evq_entries = nb_tx_desc;
899 *txq_max_fill_level = EFX_TXQ_LIMIT(*txq_entries);
903 static sfc_dp_tx_qcreate_t sfc_efx_tx_qcreate;
905 sfc_efx_tx_qcreate(uint16_t port_id, uint16_t queue_id,
906 const struct rte_pci_addr *pci_addr,
908 const struct sfc_dp_tx_qcreate_info *info,
909 struct sfc_dp_txq **dp_txqp)
911 struct sfc_efx_txq *txq;
912 struct sfc_txq *ctrl_txq;
916 txq = rte_zmalloc_socket("sfc-efx-txq", sizeof(*txq),
917 RTE_CACHE_LINE_SIZE, socket_id);
921 sfc_dp_queue_init(&txq->dp.dpq, port_id, queue_id, pci_addr);
924 txq->pend_desc = rte_calloc_socket("sfc-efx-txq-pend-desc",
925 EFX_TXQ_LIMIT(info->txq_entries),
926 sizeof(*txq->pend_desc), 0,
928 if (txq->pend_desc == NULL)
929 goto fail_pend_desc_alloc;
932 txq->sw_ring = rte_calloc_socket("sfc-efx-txq-sw_ring",
934 sizeof(*txq->sw_ring),
935 RTE_CACHE_LINE_SIZE, socket_id);
936 if (txq->sw_ring == NULL)
937 goto fail_sw_ring_alloc;
939 ctrl_txq = sfc_txq_by_dp_txq(&txq->dp);
940 if (ctrl_txq->evq->sa->tso) {
941 rc = sfc_efx_tso_alloc_tsoh_objs(txq->sw_ring,
942 info->txq_entries, socket_id);
944 goto fail_alloc_tsoh_objs;
947 txq->evq = ctrl_txq->evq;
948 txq->ptr_mask = info->txq_entries - 1;
949 txq->max_fill_level = info->max_fill_level;
950 txq->free_thresh = info->free_thresh;
951 txq->dma_desc_size_max = info->dma_desc_size_max;
956 fail_alloc_tsoh_objs:
957 rte_free(txq->sw_ring);
960 rte_free(txq->pend_desc);
962 fail_pend_desc_alloc:
969 static sfc_dp_tx_qdestroy_t sfc_efx_tx_qdestroy;
971 sfc_efx_tx_qdestroy(struct sfc_dp_txq *dp_txq)
973 struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq);
975 sfc_efx_tso_free_tsoh_objs(txq->sw_ring, txq->ptr_mask + 1);
976 rte_free(txq->sw_ring);
977 rte_free(txq->pend_desc);
981 static sfc_dp_tx_qstart_t sfc_efx_tx_qstart;
983 sfc_efx_tx_qstart(struct sfc_dp_txq *dp_txq,
984 __rte_unused unsigned int evq_read_ptr,
985 unsigned int txq_desc_index)
987 /* libefx-based datapath is specific to libefx-based PMD */
988 struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq);
989 struct sfc_txq *ctrl_txq = sfc_txq_by_dp_txq(dp_txq);
991 txq->common = ctrl_txq->common;
993 txq->pending = txq->completed = txq->added = txq_desc_index;
994 txq->hw_vlan_tci = 0;
996 txq->flags |= (SFC_EFX_TXQ_FLAG_STARTED | SFC_EFX_TXQ_FLAG_RUNNING);
1001 static sfc_dp_tx_qstop_t sfc_efx_tx_qstop;
1003 sfc_efx_tx_qstop(struct sfc_dp_txq *dp_txq,
1004 __rte_unused unsigned int *evq_read_ptr)
1006 struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq);
1008 txq->flags &= ~SFC_EFX_TXQ_FLAG_RUNNING;
1011 static sfc_dp_tx_qreap_t sfc_efx_tx_qreap;
1013 sfc_efx_tx_qreap(struct sfc_dp_txq *dp_txq)
1015 struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq);
1018 sfc_efx_tx_reap(txq);
1020 for (txds = 0; txds <= txq->ptr_mask; txds++) {
1021 if (txq->sw_ring[txds].mbuf != NULL) {
1022 rte_pktmbuf_free(txq->sw_ring[txds].mbuf);
1023 txq->sw_ring[txds].mbuf = NULL;
1027 txq->flags &= ~SFC_EFX_TXQ_FLAG_STARTED;
1030 static sfc_dp_tx_qdesc_status_t sfc_efx_tx_qdesc_status;
1032 sfc_efx_tx_qdesc_status(struct sfc_dp_txq *dp_txq, uint16_t offset)
1034 struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq);
1036 if (unlikely(offset > txq->ptr_mask))
1039 if (unlikely(offset >= txq->max_fill_level))
1040 return RTE_ETH_TX_DESC_UNAVAIL;
1043 * Poll EvQ to derive up-to-date 'txq->pending' figure;
1044 * it is required for the queue to be running, but the
1045 * check is omitted because API design assumes that it
1046 * is the duty of the caller to satisfy all conditions
1048 SFC_ASSERT((txq->flags & SFC_EFX_TXQ_FLAG_RUNNING) ==
1049 SFC_EFX_TXQ_FLAG_RUNNING);
1050 sfc_ev_qpoll(txq->evq);
1053 * Ring tail is 'txq->pending', and although descriptors
1054 * between 'txq->completed' and 'txq->pending' are still
1055 * in use by the driver, they should be reported as DONE
1057 if (unlikely(offset < (txq->added - txq->pending)))
1058 return RTE_ETH_TX_DESC_FULL;
1061 * There is no separate return value for unused descriptors;
1062 * the latter will be reported as DONE because genuine DONE
1063 * descriptors will be freed anyway in SW on the next burst
1065 return RTE_ETH_TX_DESC_DONE;
1068 struct sfc_dp_tx sfc_efx_tx = {
1070 .name = SFC_KVARG_DATAPATH_EFX,
1074 .features = SFC_DP_TX_FEAT_VLAN_INSERT |
1075 SFC_DP_TX_FEAT_TSO |
1076 SFC_DP_TX_FEAT_MULTI_POOL |
1077 SFC_DP_TX_FEAT_REFCNT |
1078 SFC_DP_TX_FEAT_MULTI_SEG,
1079 .qsize_up_rings = sfc_efx_tx_qsize_up_rings,
1080 .qcreate = sfc_efx_tx_qcreate,
1081 .qdestroy = sfc_efx_tx_qdestroy,
1082 .qstart = sfc_efx_tx_qstart,
1083 .qstop = sfc_efx_tx_qstop,
1084 .qreap = sfc_efx_tx_qreap,
1085 .qdesc_status = sfc_efx_tx_qdesc_status,
1086 .pkt_burst = sfc_efx_xmit_pkts,