From dbdc82416b723b1f089bdcea99d5441016aa724d Mon Sep 17 00:00:00 2001 From: Andrew Rybchenko Date: Mon, 20 Mar 2017 10:15:14 +0000 Subject: [PATCH] net/sfc: factor out libefx-based Tx datapath Split control and datapath to make datapath substitutable and possibly reusable with alternative control path. libefx-based Tx datapath is bound to libefx control path, but it should be possible to use other datapaths with alternative control path(s). Signed-off-by: Andrew Rybchenko --- doc/guides/nics/sfc_efx.rst | 8 + drivers/net/sfc/sfc.h | 1 + drivers/net/sfc/sfc_dp.c | 4 +- drivers/net/sfc/sfc_dp.h | 1 + drivers/net/sfc/sfc_dp_tx.h | 148 ++++++++++++++++ drivers/net/sfc/sfc_ethdev.c | 47 +++++- drivers/net/sfc/sfc_ev.c | 48 ++++-- drivers/net/sfc/sfc_ev.h | 8 +- drivers/net/sfc/sfc_kvargs.c | 1 + drivers/net/sfc/sfc_kvargs.h | 4 + drivers/net/sfc/sfc_tso.c | 22 +-- drivers/net/sfc/sfc_tx.c | 318 +++++++++++++++++++++++++---------- drivers/net/sfc/sfc_tx.h | 98 +++++++---- 13 files changed, 564 insertions(+), 144 deletions(-) create mode 100644 drivers/net/sfc/sfc_dp_tx.h diff --git a/doc/guides/nics/sfc_efx.rst b/doc/guides/nics/sfc_efx.rst index 43bd9f15f4..94c4d07d70 100644 --- a/doc/guides/nics/sfc_efx.rst +++ b/doc/guides/nics/sfc_efx.rst @@ -236,6 +236,14 @@ boolean parameters value. more efficient than libefx-based and provides richer packet type classification, but lacks Rx scatter support. +- ``tx_datapath`` [auto|efx] (default **auto**) + + Choose transmit datapath implementation. + **auto** allows the driver itself to make a choice based on firmware + features available and required by the datapath implementation. + **efx** chooses libefx-based datapath which supports VLAN insertion + (full-feature firmware variant only), TSO and multi-segment mbufs. + - ``perf_profile`` [auto|throughput|low-latency] (default **throughput**) Choose hardware tunning to be optimized for either throughput or diff --git a/drivers/net/sfc/sfc.h b/drivers/net/sfc/sfc.h index 02c97d18c5..c2961ea11a 100644 --- a/drivers/net/sfc/sfc.h +++ b/drivers/net/sfc/sfc.h @@ -228,6 +228,7 @@ struct sfc_adapter { #endif const struct sfc_dp_rx *dp_rx; + const struct sfc_dp_tx *dp_tx; }; /* diff --git a/drivers/net/sfc/sfc_dp.c b/drivers/net/sfc/sfc_dp.c index b52b2eec16..860aa9213f 100644 --- a/drivers/net/sfc/sfc_dp.c +++ b/drivers/net/sfc/sfc_dp.c @@ -87,7 +87,9 @@ sfc_dp_register(struct sfc_dp_list *head, struct sfc_dp *entry) if (sfc_dp_find_by_name(head, entry->type, entry->name) != NULL) { rte_log(RTE_LOG_ERR, RTE_LOGTYPE_PMD, "sfc %s dapapath '%s' already registered\n", - entry->type == SFC_DP_RX ? "Rx" : "unknown", + entry->type == SFC_DP_RX ? "Rx" : + entry->type == SFC_DP_TX ? "Tx" : + "unknown", entry->name); return EEXIST; } diff --git a/drivers/net/sfc/sfc_dp.h b/drivers/net/sfc/sfc_dp.h index f83ea58b92..eff0aa8795 100644 --- a/drivers/net/sfc/sfc_dp.h +++ b/drivers/net/sfc/sfc_dp.h @@ -56,6 +56,7 @@ typedef void (sfc_dp_exception_t)(void *ctrl); enum sfc_dp_type { SFC_DP_RX = 0, /**< Receive datapath */ + SFC_DP_TX, /**< Transmit datapath */ }; diff --git a/drivers/net/sfc/sfc_dp_tx.h b/drivers/net/sfc/sfc_dp_tx.h new file mode 100644 index 0000000000..1f922e5976 --- /dev/null +++ b/drivers/net/sfc/sfc_dp_tx.h @@ -0,0 +1,148 @@ +/*- + * BSD LICENSE + * + * Copyright (c) 2016 Solarflare Communications Inc. + * All rights reserved. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SFC_DP_TX_H +#define _SFC_DP_TX_H + +#include + +#include "sfc_dp.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Generic transmit queue information used on data path. + * It must be kept as small as it is possible since it is built into + * the structure used on datapath. + */ +struct sfc_dp_txq { + struct sfc_dp_queue dpq; +}; + +/** + * Datapath transmit queue creation information. + * + * The structure is used just to pass information from control path to + * datapath. It could be just function arguments, but it would be hardly + * readable. + */ +struct sfc_dp_tx_qcreate_info { + /** Minimum number of unused Tx descriptors to do reap */ + unsigned int free_thresh; + /** Transmit queue configuration flags */ + unsigned int flags; + /** Tx queue size */ + unsigned int txq_entries; + /** Maximum size of data in the DMA descriptor */ + uint16_t dma_desc_size_max; +}; + +/** + * Allocate and initialize datapath transmit queue. + * + * @param port_id The port identifier + * @param queue_id The queue identifier + * @param pci_addr PCI function address + * @param socket_id Socket identifier to allocate memory + * @param info Tx queue details wrapped in structure + * @param dp_txqp Location for generic datapath transmit queue pointer + * + * @return 0 or positive errno. + */ +typedef int (sfc_dp_tx_qcreate_t)(uint16_t port_id, uint16_t queue_id, + const struct rte_pci_addr *pci_addr, + int socket_id, + const struct sfc_dp_tx_qcreate_info *info, + struct sfc_dp_txq **dp_txqp); + +/** + * Free resources allocated for datapath transmit queue. + */ +typedef void (sfc_dp_tx_qdestroy_t)(struct sfc_dp_txq *dp_txq); + +/** + * Transmit queue start callback. + * + * It handovers EvQ to the datapath. + */ +typedef int (sfc_dp_tx_qstart_t)(struct sfc_dp_txq *dp_txq, + unsigned int evq_read_ptr, + unsigned int txq_desc_index); + +/** + * Transmit queue stop function called before the queue flush. + * + * It returns EvQ to the control path. + */ +typedef void (sfc_dp_tx_qstop_t)(struct sfc_dp_txq *dp_txq, + unsigned int *evq_read_ptr); + +/** + * Transmit queue function called after the queue flush. + */ +typedef void (sfc_dp_tx_qreap_t)(struct sfc_dp_txq *dp_txq); + +/** Transmit datapath definition */ +struct sfc_dp_tx { + struct sfc_dp dp; + + sfc_dp_tx_qcreate_t *qcreate; + sfc_dp_tx_qdestroy_t *qdestroy; + sfc_dp_tx_qstart_t *qstart; + sfc_dp_tx_qstop_t *qstop; + sfc_dp_tx_qreap_t *qreap; + eth_tx_burst_t pkt_burst; +}; + +static inline struct sfc_dp_tx * +sfc_dp_find_tx_by_name(struct sfc_dp_list *head, const char *name) +{ + struct sfc_dp *p = sfc_dp_find_by_name(head, SFC_DP_TX, name); + + return (p == NULL) ? NULL : container_of(p, struct sfc_dp_tx, dp); +} + +static inline struct sfc_dp_tx * +sfc_dp_find_tx_by_caps(struct sfc_dp_list *head, unsigned int avail_caps) +{ + struct sfc_dp *p = sfc_dp_find_by_caps(head, SFC_DP_TX, avail_caps); + + return (p == NULL) ? NULL : container_of(p, struct sfc_dp_tx, dp); +} + +extern struct sfc_dp_tx sfc_efx_tx; + +#ifdef __cplusplus +} +#endif +#endif /* _SFC_DP_TX_H */ diff --git a/drivers/net/sfc/sfc_ethdev.c b/drivers/net/sfc/sfc_ethdev.c index 010641b919..38ee2f823d 100644 --- a/drivers/net/sfc/sfc_ethdev.c +++ b/drivers/net/sfc/sfc_ethdev.c @@ -467,7 +467,7 @@ sfc_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id, if (rc != 0) goto fail_tx_qinit; - dev->data->tx_queues[tx_queue_id] = sa->txq_info[tx_queue_id].txq; + dev->data->tx_queues[tx_queue_id] = sa->txq_info[tx_queue_id].txq->dp; sfc_adapter_unlock(sa); return 0; @@ -481,13 +481,15 @@ fail_tx_qinit: static void sfc_tx_queue_release(void *queue) { - struct sfc_txq *txq = queue; + struct sfc_dp_txq *dp_txq = queue; + struct sfc_txq *txq; unsigned int sw_index; struct sfc_adapter *sa; - if (txq == NULL) + if (dp_txq == NULL) return; + txq = sfc_txq_by_dp_txq(dp_txq); sw_index = sfc_txq_sw_index(txq); SFC_ASSERT(txq->evq != NULL); @@ -1361,6 +1363,7 @@ sfc_eth_dev_set_ops(struct rte_eth_dev *dev) struct sfc_adapter *sa = dev->data->dev_private; unsigned int avail_caps = 0; const char *rx_name = NULL; + const char *tx_name = NULL; int rc; if (sa == NULL || sa->state == SFC_ADAPTER_UNINITIALIZED) @@ -1408,12 +1411,45 @@ sfc_eth_dev_set_ops(struct rte_eth_dev *dev) dev->rx_pkt_burst = sa->dp_rx->pkt_burst; - dev->tx_pkt_burst = sfc_xmit_pkts; + rc = sfc_kvargs_process(sa, SFC_KVARG_TX_DATAPATH, + sfc_kvarg_string_handler, &tx_name); + if (rc != 0) + goto fail_kvarg_tx_datapath; + + if (tx_name != NULL) { + sa->dp_tx = sfc_dp_find_tx_by_name(&sfc_dp_head, tx_name); + if (sa->dp_tx == NULL) { + sfc_err(sa, "Tx datapath %s not found", tx_name); + rc = ENOENT; + goto fail_dp_tx; + } + if (!sfc_dp_match_hw_fw_caps(&sa->dp_tx->dp, avail_caps)) { + sfc_err(sa, + "Insufficient Hw/FW capabilities to use Tx datapath %s", + tx_name); + rc = EINVAL; + goto fail_dp_tx; + } + } else { + sa->dp_tx = sfc_dp_find_tx_by_caps(&sfc_dp_head, avail_caps); + if (sa->dp_tx == NULL) { + sfc_err(sa, "Tx datapath by caps %#x not found", + avail_caps); + rc = ENOENT; + goto fail_dp_tx; + } + } + + sfc_info(sa, "use %s Tx datapath", sa->dp_tx->dp.name); + + dev->tx_pkt_burst = sa->dp_tx->pkt_burst; dev->dev_ops = &sfc_eth_dev_ops; return 0; +fail_dp_tx: +fail_kvarg_tx_datapath: fail_dp_rx: fail_kvarg_rx_datapath: return rc; @@ -1427,6 +1463,8 @@ sfc_register_dp(void) /* Prefer EF10 datapath */ sfc_dp_register(&sfc_dp_head, &sfc_ef10_rx.dp); sfc_dp_register(&sfc_dp_head, &sfc_efx_rx.dp); + + sfc_dp_register(&sfc_dp_head, &sfc_efx_tx.dp); } } @@ -1563,6 +1601,7 @@ RTE_PMD_REGISTER_PCI_TABLE(net_sfc_efx, pci_id_sfc_efx_map); RTE_PMD_REGISTER_KMOD_DEP(net_sfc_efx, "* igb_uio | uio_pci_generic | vfio"); RTE_PMD_REGISTER_PARAM_STRING(net_sfc_efx, SFC_KVARG_RX_DATAPATH "=" SFC_KVARG_VALUES_RX_DATAPATH " " + SFC_KVARG_TX_DATAPATH "=" SFC_KVARG_VALUES_TX_DATAPATH " " SFC_KVARG_PERF_PROFILE "=" SFC_KVARG_VALUES_PERF_PROFILE " " SFC_KVARG_STATS_UPDATE_PERIOD_MS "= " SFC_KVARG_MCDI_LOGGING "=" SFC_KVARG_VALUES_BOOL " " diff --git a/drivers/net/sfc/sfc_ev.c b/drivers/net/sfc/sfc_ev.c index 84b2fd12dc..2f96fb8425 100644 --- a/drivers/net/sfc/sfc_ev.c +++ b/drivers/net/sfc/sfc_ev.c @@ -183,16 +183,18 @@ static boolean_t sfc_ev_tx(void *arg, __rte_unused uint32_t label, uint32_t id) { struct sfc_evq *evq = arg; - struct sfc_txq *txq; + struct sfc_dp_txq *dp_txq; + struct sfc_efx_txq *txq; unsigned int stop; unsigned int delta; - txq = evq->txq; + dp_txq = evq->dp_txq; + SFC_ASSERT(dp_txq != NULL); - SFC_ASSERT(txq != NULL); + txq = sfc_efx_txq_by_dp_txq(dp_txq); SFC_ASSERT(txq->evq == evq); - if (unlikely((txq->state & SFC_TXQ_STARTED) == 0)) + if (unlikely((txq->flags & SFC_EFX_TXQ_FLAG_STARTED) == 0)) goto done; stop = (id + 1) & txq->ptr_mask; @@ -305,9 +307,13 @@ static boolean_t sfc_ev_txq_flush_done(void *arg, __rte_unused uint32_t txq_hw_index) { struct sfc_evq *evq = arg; + struct sfc_dp_txq *dp_txq; struct sfc_txq *txq; - txq = evq->txq; + dp_txq = evq->dp_txq; + SFC_ASSERT(dp_txq != NULL); + + txq = sfc_txq_by_dp_txq(dp_txq); SFC_ASSERT(txq != NULL); SFC_ASSERT(txq->hw_index == txq_hw_index); SFC_ASSERT(txq->evq == evq); @@ -441,7 +447,7 @@ static const efx_ev_callbacks_t sfc_ev_callbacks_dp_rx = { .eec_link_change = sfc_ev_nop_link_change, }; -static const efx_ev_callbacks_t sfc_ev_callbacks_tx = { +static const efx_ev_callbacks_t sfc_ev_callbacks_efx_tx = { .eec_initialized = sfc_ev_initialized, .eec_rx = sfc_ev_nop_rx, .eec_tx = sfc_ev_tx, @@ -456,6 +462,21 @@ static const efx_ev_callbacks_t sfc_ev_callbacks_tx = { .eec_link_change = sfc_ev_nop_link_change, }; +static const efx_ev_callbacks_t sfc_ev_callbacks_dp_tx = { + .eec_initialized = sfc_ev_initialized, + .eec_rx = sfc_ev_nop_rx, + .eec_tx = sfc_ev_nop_tx, + .eec_exception = sfc_ev_exception, + .eec_rxq_flush_done = sfc_ev_nop_rxq_flush_done, + .eec_rxq_flush_failed = sfc_ev_nop_rxq_flush_failed, + .eec_txq_flush_done = sfc_ev_txq_flush_done, + .eec_software = sfc_ev_software, + .eec_sram = sfc_ev_sram, + .eec_wake_up = sfc_ev_wake_up, + .eec_timer = sfc_ev_timer, + .eec_link_change = sfc_ev_nop_link_change, +}; + void sfc_ev_qpoll(struct sfc_evq *evq) @@ -487,8 +508,10 @@ sfc_ev_qpoll(struct sfc_evq *evq) rxq_sw_index); } - if (evq->txq != NULL) { - unsigned int txq_sw_index = sfc_txq_sw_index(evq->txq); + if (evq->dp_txq != NULL) { + unsigned int txq_sw_index; + + txq_sw_index = evq->dp_txq->dpq.queue_id; sfc_warn(sa, "restart TxQ %u because of exception on its EvQ %u", @@ -558,14 +581,17 @@ sfc_ev_qstart(struct sfc_adapter *sa, unsigned int sw_index) if (rc != 0) goto fail_ev_qcreate; - SFC_ASSERT(evq->dp_rxq == NULL || evq->txq == NULL); + SFC_ASSERT(evq->dp_rxq == NULL || evq->dp_txq == NULL); if (evq->dp_rxq != 0) { if (strcmp(sa->dp_rx->dp.name, SFC_KVARG_DATAPATH_EFX) == 0) evq->callbacks = &sfc_ev_callbacks_efx_rx; else evq->callbacks = &sfc_ev_callbacks_dp_rx; - } else if (evq->txq != 0) { - evq->callbacks = &sfc_ev_callbacks_tx; + } else if (evq->dp_txq != 0) { + if (strcmp(sa->dp_tx->dp.name, SFC_KVARG_DATAPATH_EFX) == 0) + evq->callbacks = &sfc_ev_callbacks_efx_tx; + else + evq->callbacks = &sfc_ev_callbacks_dp_tx; } else { evq->callbacks = &sfc_ev_callbacks; } diff --git a/drivers/net/sfc/sfc_ev.h b/drivers/net/sfc/sfc_ev.h index 760df98f2e..e8d3090be9 100644 --- a/drivers/net/sfc/sfc_ev.h +++ b/drivers/net/sfc/sfc_ev.h @@ -32,8 +32,12 @@ #ifndef _SFC_EV_H_ #define _SFC_EV_H_ +#include + #include "efx.h" +#include "sfc.h" + #ifdef __cplusplus extern "C" { #endif @@ -43,7 +47,7 @@ extern "C" { struct sfc_adapter; struct sfc_dp_rxq; -struct sfc_txq; +struct sfc_dp_txq; enum sfc_evq_state { SFC_EVQ_UNINITIALIZED = 0, @@ -62,7 +66,7 @@ struct sfc_evq { boolean_t exception; efsys_mem_t mem; struct sfc_dp_rxq *dp_rxq; - struct sfc_txq *txq; + struct sfc_dp_txq *dp_txq; /* Not used on datapath */ struct sfc_adapter *sa; diff --git a/drivers/net/sfc/sfc_kvargs.c b/drivers/net/sfc/sfc_kvargs.c index 01dff4c094..7bcd5951ea 100644 --- a/drivers/net/sfc/sfc_kvargs.c +++ b/drivers/net/sfc/sfc_kvargs.c @@ -49,6 +49,7 @@ sfc_kvargs_parse(struct sfc_adapter *sa) SFC_KVARG_MCDI_LOGGING, SFC_KVARG_PERF_PROFILE, SFC_KVARG_RX_DATAPATH, + SFC_KVARG_TX_DATAPATH, NULL, }; diff --git a/drivers/net/sfc/sfc_kvargs.h b/drivers/net/sfc/sfc_kvargs.h index 38d17e024c..a3a08b7ae3 100644 --- a/drivers/net/sfc/sfc_kvargs.h +++ b/drivers/net/sfc/sfc_kvargs.h @@ -64,6 +64,10 @@ extern "C" { "[" SFC_KVARG_DATAPATH_EFX "|" \ SFC_KVARG_DATAPATH_EF10 "]" +#define SFC_KVARG_TX_DATAPATH "tx_datapath" +#define SFC_KVARG_VALUES_TX_DATAPATH \ + "[" SFC_KVARG_DATAPATH_EFX "]" + struct sfc_adapter; int sfc_kvargs_parse(struct sfc_adapter *sa); diff --git a/drivers/net/sfc/sfc_tso.c b/drivers/net/sfc/sfc_tso.c index 271861fcbd..fb79d74911 100644 --- a/drivers/net/sfc/sfc_tso.c +++ b/drivers/net/sfc/sfc_tso.c @@ -44,13 +44,13 @@ #define SFC_TSO_OPDESCS_IDX_SHIFT 2 int -sfc_tso_alloc_tsoh_objs(struct sfc_tx_sw_desc *sw_ring, - unsigned int txq_entries, unsigned int socket_id) +sfc_efx_tso_alloc_tsoh_objs(struct sfc_efx_tx_sw_desc *sw_ring, + unsigned int txq_entries, unsigned int socket_id) { unsigned int i; for (i = 0; i < txq_entries; ++i) { - sw_ring[i].tsoh = rte_malloc_socket("sfc-txq-tsoh-obj", + sw_ring[i].tsoh = rte_malloc_socket("sfc-efx-txq-tsoh-obj", SFC_TSOH_STD_LEN, RTE_CACHE_LINE_SIZE, socket_id); @@ -68,7 +68,8 @@ fail_alloc_tsoh_objs: } void -sfc_tso_free_tsoh_objs(struct sfc_tx_sw_desc *sw_ring, unsigned int txq_entries) +sfc_efx_tso_free_tsoh_objs(struct sfc_efx_tx_sw_desc *sw_ring, + unsigned int txq_entries) { unsigned int i; @@ -79,8 +80,8 @@ sfc_tso_free_tsoh_objs(struct sfc_tx_sw_desc *sw_ring, unsigned int txq_entries) } static void -sfc_tso_prepare_header(struct sfc_txq *txq, struct rte_mbuf **in_seg, - size_t *in_off, unsigned int idx, size_t bytes_left) +sfc_efx_tso_prepare_header(struct sfc_efx_txq *txq, struct rte_mbuf **in_seg, + size_t *in_off, unsigned int idx, size_t bytes_left) { struct rte_mbuf *m = *in_seg; size_t bytes_to_copy = 0; @@ -111,9 +112,9 @@ sfc_tso_prepare_header(struct sfc_txq *txq, struct rte_mbuf **in_seg, } int -sfc_tso_do(struct sfc_txq *txq, unsigned int idx, struct rte_mbuf **in_seg, - size_t *in_off, efx_desc_t **pend, unsigned int *pkt_descs, - size_t *pkt_len) +sfc_efx_tso_do(struct sfc_efx_txq *txq, unsigned int idx, + struct rte_mbuf **in_seg, size_t *in_off, efx_desc_t **pend, + unsigned int *pkt_descs, size_t *pkt_len) { uint8_t *tsoh; const struct tcp_hdr *th; @@ -150,7 +151,8 @@ sfc_tso_do(struct sfc_txq *txq, unsigned int idx, struct rte_mbuf **in_seg, * limitations on address boundaries crossing by DMA descriptor data. */ if (m->data_len < header_len) { - sfc_tso_prepare_header(txq, in_seg, in_off, idx, header_len); + sfc_efx_tso_prepare_header(txq, in_seg, in_off, idx, + header_len); tsoh = txq->sw_ring[idx & txq->ptr_mask].tsoh; header_paddr = rte_malloc_virt2phy((void *)tsoh); diff --git a/drivers/net/sfc/sfc_tx.c b/drivers/net/sfc/sfc_tx.c index 6131a498c3..e716f2ff79 100644 --- a/drivers/net/sfc/sfc_tx.c +++ b/drivers/net/sfc/sfc_tx.c @@ -35,6 +35,7 @@ #include "sfc_ev.h" #include "sfc_tx.h" #include "sfc_tweak.h" +#include "sfc_kvargs.h" /* * Maximum number of TX queue flush attempts in case of @@ -111,29 +112,6 @@ sfc_tx_qflush_done(struct sfc_txq *txq) txq->state &= ~SFC_TXQ_FLUSHING; } -static void -sfc_tx_reap(struct sfc_txq *txq) -{ - unsigned int completed; - - - sfc_ev_qpoll(txq->evq); - - for (completed = txq->completed; - completed != txq->pending; completed++) { - struct sfc_tx_sw_desc *txd; - - txd = &txq->sw_ring[completed & txq->ptr_mask]; - - if (txd->mbuf != NULL) { - rte_pktmbuf_free(txd->mbuf); - txd->mbuf = NULL; - } - } - - txq->completed = completed; -} - int sfc_tx_qinit(struct sfc_adapter *sa, unsigned int sw_index, uint16_t nb_tx_desc, unsigned int socket_id, @@ -145,6 +123,7 @@ sfc_tx_qinit(struct sfc_adapter *sa, unsigned int sw_index, struct sfc_txq *txq; unsigned int evq_index = sfc_evq_index_by_txq_sw_index(sa, sw_index); int rc = 0; + struct sfc_dp_tx_qcreate_info info; sfc_log_init(sa, "TxQ = %u", sw_index); @@ -169,57 +148,45 @@ sfc_tx_qinit(struct sfc_adapter *sa, unsigned int sw_index, if (txq == NULL) goto fail_txq_alloc; + txq_info->txq = txq; + + txq->hw_index = sw_index; + txq->evq = evq; + txq->free_thresh = + (tx_conf->tx_free_thresh) ? tx_conf->tx_free_thresh : + SFC_TX_DEFAULT_FREE_THRESH; + txq->flags = tx_conf->txq_flags; + rc = sfc_dma_alloc(sa, "txq", sw_index, EFX_TXQ_SIZE(txq_info->entries), socket_id, &txq->mem); if (rc != 0) goto fail_dma_alloc; - rc = ENOMEM; - txq->pend_desc = rte_calloc_socket("sfc-txq-pend-desc", - EFX_TXQ_LIMIT(txq_info->entries), - sizeof(efx_desc_t), 0, socket_id); - if (txq->pend_desc == NULL) - goto fail_pend_desc_alloc; + memset(&info, 0, sizeof(info)); + info.free_thresh = txq->free_thresh; + info.flags = tx_conf->txq_flags; + info.txq_entries = txq_info->entries; + info.dma_desc_size_max = encp->enc_tx_dma_desc_size_max; - rc = ENOMEM; - txq->sw_ring = rte_calloc_socket("sfc-txq-desc", txq_info->entries, - sizeof(*txq->sw_ring), 0, socket_id); - if (txq->sw_ring == NULL) - goto fail_desc_alloc; + rc = sa->dp_tx->qcreate(sa->eth_dev->data->port_id, sw_index, + &SFC_DEV_TO_PCI(sa->eth_dev)->addr, + socket_id, &info, &txq->dp); + if (rc != 0) + goto fail_dp_tx_qinit; - if (sa->tso) { - rc = sfc_tso_alloc_tsoh_objs(txq->sw_ring, txq_info->entries, - socket_id); - if (rc != 0) - goto fail_alloc_tsoh_objs; - } + evq->dp_txq = txq->dp; txq->state = SFC_TXQ_INITIALIZED; - txq->ptr_mask = txq_info->entries - 1; - txq->free_thresh = (tx_conf->tx_free_thresh) ? tx_conf->tx_free_thresh : - SFC_TX_DEFAULT_FREE_THRESH; - txq->dma_desc_size_max = encp->enc_tx_dma_desc_size_max; - txq->hw_index = sw_index; - txq->flags = tx_conf->txq_flags; - txq->evq = evq; - evq->txq = txq; - - txq_info->txq = txq; txq_info->deferred_start = (tx_conf->tx_deferred_start != 0); return 0; -fail_alloc_tsoh_objs: - rte_free(txq->sw_ring); - -fail_desc_alloc: - rte_free(txq->pend_desc); - -fail_pend_desc_alloc: +fail_dp_tx_qinit: sfc_dma_free(sa, &txq->mem); fail_dma_alloc: + txq_info->txq = NULL; rte_free(txq); fail_txq_alloc: @@ -248,13 +215,12 @@ sfc_tx_qfini(struct sfc_adapter *sa, unsigned int sw_index) SFC_ASSERT(txq != NULL); SFC_ASSERT(txq->state == SFC_TXQ_INITIALIZED); - sfc_tso_free_tsoh_objs(txq->sw_ring, txq_info->entries); + sa->dp_tx->qdestroy(txq->dp); + txq->dp = NULL; txq_info->txq = NULL; txq_info->entries = 0; - rte_free(txq->sw_ring); - rte_free(txq->pend_desc); sfc_dma_free(sa, &txq->mem); rte_free(txq); } @@ -421,12 +387,13 @@ sfc_tx_qstart(struct sfc_adapter *sa, unsigned int sw_index) goto fail_tx_qcreate; } - txq->added = txq->pending = txq->completed = desc_index; - txq->hw_vlan_tci = 0; - efx_tx_qenable(txq->common); - txq->state |= (SFC_TXQ_STARTED | SFC_TXQ_RUNNING); + txq->state |= SFC_TXQ_STARTED; + + rc = sa->dp_tx->qstart(txq->dp, evq->read_ptr, desc_index); + if (rc != 0) + goto fail_dp_qstart; /* * It seems to be used by DPDK for debug purposes only ('rte_ether') @@ -436,6 +403,10 @@ sfc_tx_qstart(struct sfc_adapter *sa, unsigned int sw_index) return 0; +fail_dp_qstart: + txq->state = SFC_TXQ_INITIALIZED; + efx_tx_qdestroy(txq->common); + fail_tx_qcreate: sfc_ev_qstop(sa, evq->evq_index); @@ -451,7 +422,6 @@ sfc_tx_qstop(struct sfc_adapter *sa, unsigned int sw_index) struct sfc_txq *txq; unsigned int retry_count; unsigned int wait_count; - unsigned int txds; sfc_log_init(sa, "TxQ = %u", sw_index); @@ -465,7 +435,7 @@ sfc_tx_qstop(struct sfc_adapter *sa, unsigned int sw_index) SFC_ASSERT(txq->state & SFC_TXQ_STARTED); - txq->state &= ~SFC_TXQ_RUNNING; + sa->dp_tx->qstop(txq->dp, &txq->evq->read_ptr); /* * Retry TX queue flushing in case of flush failed or @@ -500,14 +470,7 @@ sfc_tx_qstop(struct sfc_adapter *sa, unsigned int sw_index) sfc_info(sa, "TxQ %u flushed", sw_index); } - sfc_tx_reap(txq); - - for (txds = 0; txds < txq_info->entries; txds++) { - if (txq->sw_ring[txds].mbuf != NULL) { - rte_pktmbuf_free(txq->sw_ring[txds].mbuf); - txq->sw_ring[txds].mbuf = NULL; - } - } + sa->dp_tx->qreap(txq->dp); txq->state = SFC_TXQ_INITIALIZED; @@ -579,6 +542,28 @@ sfc_tx_stop(struct sfc_adapter *sa) efx_tx_fini(sa->nic); } +static void +sfc_efx_tx_reap(struct sfc_efx_txq *txq) +{ + unsigned int completed; + + sfc_ev_qpoll(txq->evq); + + for (completed = txq->completed; + completed != txq->pending; completed++) { + struct sfc_efx_tx_sw_desc *txd; + + txd = &txq->sw_ring[completed & txq->ptr_mask]; + + if (txd->mbuf != NULL) { + rte_pktmbuf_free(txd->mbuf); + txd->mbuf = NULL; + } + } + + txq->completed = completed; +} + /* * The function is used to insert or update VLAN tag; * the firmware has state of the firmware tag to insert per TxQ @@ -587,8 +572,8 @@ sfc_tx_stop(struct sfc_adapter *sa) * the function will update it */ static unsigned int -sfc_tx_maybe_insert_tag(struct sfc_txq *txq, struct rte_mbuf *m, - efx_desc_t **pend) +sfc_efx_tx_maybe_insert_tag(struct sfc_efx_txq *txq, struct rte_mbuf *m, + efx_desc_t **pend) { uint16_t this_tag = ((m->ol_flags & PKT_TX_VLAN_PKT) ? m->vlan_tci : 0); @@ -610,10 +595,11 @@ sfc_tx_maybe_insert_tag(struct sfc_txq *txq, struct rte_mbuf *m, return 1; } -uint16_t -sfc_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) +static uint16_t +sfc_efx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { - struct sfc_txq *txq = (struct sfc_txq *)tx_queue; + struct sfc_dp_txq *dp_txq = (struct sfc_dp_txq *)tx_queue; + struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq); unsigned int added = txq->added; unsigned int pushed = added; unsigned int pkts_sent = 0; @@ -625,7 +611,7 @@ sfc_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) int rc __rte_unused; struct rte_mbuf **pktp; - if (unlikely((txq->state & SFC_TXQ_RUNNING) == 0)) + if (unlikely((txq->flags & SFC_EFX_TXQ_FLAG_RUNNING) == 0)) goto done; /* @@ -636,7 +622,7 @@ sfc_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) reap_done = (fill_level > soft_max_fill); if (reap_done) { - sfc_tx_reap(txq); + sfc_efx_tx_reap(txq); /* * Recalculate fill level since 'txq->completed' * might have changed on reap @@ -659,15 +645,15 @@ sfc_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) * DEV_TX_VLAN_OFFLOAD and pushes VLAN TCI, then * TX_ERROR will occur */ - pkt_descs += sfc_tx_maybe_insert_tag(txq, m_seg, &pend); + pkt_descs += sfc_efx_tx_maybe_insert_tag(txq, m_seg, &pend); if (m_seg->ol_flags & PKT_TX_TCP_SEG) { /* * We expect correct 'pkt->l[2, 3, 4]_len' values * to be set correctly by the caller */ - if (sfc_tso_do(txq, added, &m_seg, &in_off, &pend, - &pkt_descs, &pkt_len) != 0) { + if (sfc_efx_tso_do(txq, added, &m_seg, &in_off, &pend, + &pkt_descs, &pkt_len) != 0) { /* We may have reached this place for * one of the following reasons: * @@ -749,7 +735,7 @@ sfc_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) * Try to reap (if we haven't yet). */ if (!reap_done) { - sfc_tx_reap(txq); + sfc_efx_tx_reap(txq); reap_done = B_TRUE; fill_level = added - txq->completed; if (fill_level > hard_max_fill) { @@ -778,9 +764,169 @@ sfc_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) #if SFC_TX_XMIT_PKTS_REAP_AT_LEAST_ONCE if (!reap_done) - sfc_tx_reap(txq); + sfc_efx_tx_reap(txq); #endif done: return pkts_sent; } + +struct sfc_txq * +sfc_txq_by_dp_txq(const struct sfc_dp_txq *dp_txq) +{ + const struct sfc_dp_queue *dpq = &dp_txq->dpq; + struct rte_eth_dev *eth_dev; + struct sfc_adapter *sa; + struct sfc_txq *txq; + + SFC_ASSERT(rte_eth_dev_is_valid_port(dpq->port_id)); + eth_dev = &rte_eth_devices[dpq->port_id]; + + sa = eth_dev->data->dev_private; + + SFC_ASSERT(dpq->queue_id < sa->txq_count); + txq = sa->txq_info[dpq->queue_id].txq; + + SFC_ASSERT(txq != NULL); + return txq; +} + +static sfc_dp_tx_qcreate_t sfc_efx_tx_qcreate; +static int +sfc_efx_tx_qcreate(uint16_t port_id, uint16_t queue_id, + const struct rte_pci_addr *pci_addr, + int socket_id, + const struct sfc_dp_tx_qcreate_info *info, + struct sfc_dp_txq **dp_txqp) +{ + struct sfc_efx_txq *txq; + struct sfc_txq *ctrl_txq; + int rc; + + rc = ENOMEM; + txq = rte_zmalloc_socket("sfc-efx-txq", sizeof(*txq), + RTE_CACHE_LINE_SIZE, socket_id); + if (txq == NULL) + goto fail_txq_alloc; + + sfc_dp_queue_init(&txq->dp.dpq, port_id, queue_id, pci_addr); + + rc = ENOMEM; + txq->pend_desc = rte_calloc_socket("sfc-efx-txq-pend-desc", + EFX_TXQ_LIMIT(info->txq_entries), + sizeof(*txq->pend_desc), 0, + socket_id); + if (txq->pend_desc == NULL) + goto fail_pend_desc_alloc; + + rc = ENOMEM; + txq->sw_ring = rte_calloc_socket("sfc-efx-txq-sw_ring", + info->txq_entries, + sizeof(*txq->sw_ring), + RTE_CACHE_LINE_SIZE, socket_id); + if (txq->sw_ring == NULL) + goto fail_sw_ring_alloc; + + ctrl_txq = sfc_txq_by_dp_txq(&txq->dp); + if (ctrl_txq->evq->sa->tso) { + rc = sfc_efx_tso_alloc_tsoh_objs(txq->sw_ring, + info->txq_entries, socket_id); + if (rc != 0) + goto fail_alloc_tsoh_objs; + } + + txq->evq = ctrl_txq->evq; + txq->ptr_mask = info->txq_entries - 1; + txq->free_thresh = info->free_thresh; + txq->dma_desc_size_max = info->dma_desc_size_max; + + *dp_txqp = &txq->dp; + return 0; + +fail_alloc_tsoh_objs: + rte_free(txq->sw_ring); + +fail_sw_ring_alloc: + rte_free(txq->pend_desc); + +fail_pend_desc_alloc: + rte_free(txq); + +fail_txq_alloc: + return rc; +} + +static sfc_dp_tx_qdestroy_t sfc_efx_tx_qdestroy; +static void +sfc_efx_tx_qdestroy(struct sfc_dp_txq *dp_txq) +{ + struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq); + + sfc_efx_tso_free_tsoh_objs(txq->sw_ring, txq->ptr_mask + 1); + rte_free(txq->sw_ring); + rte_free(txq->pend_desc); + rte_free(txq); +} + +static sfc_dp_tx_qstart_t sfc_efx_tx_qstart; +static int +sfc_efx_tx_qstart(struct sfc_dp_txq *dp_txq, + __rte_unused unsigned int evq_read_ptr, + unsigned int txq_desc_index) +{ + /* libefx-based datapath is specific to libefx-based PMD */ + struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq); + struct sfc_txq *ctrl_txq = sfc_txq_by_dp_txq(dp_txq); + + txq->common = ctrl_txq->common; + + txq->pending = txq->completed = txq->added = txq_desc_index; + txq->hw_vlan_tci = 0; + + txq->flags |= (SFC_EFX_TXQ_FLAG_STARTED | SFC_EFX_TXQ_FLAG_RUNNING); + + return 0; +} + +static sfc_dp_tx_qstop_t sfc_efx_tx_qstop; +static void +sfc_efx_tx_qstop(struct sfc_dp_txq *dp_txq, + __rte_unused unsigned int *evq_read_ptr) +{ + struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq); + + txq->flags &= ~SFC_EFX_TXQ_FLAG_RUNNING; +} + +static sfc_dp_tx_qreap_t sfc_efx_tx_qreap; +static void +sfc_efx_tx_qreap(struct sfc_dp_txq *dp_txq) +{ + struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq); + unsigned int txds; + + sfc_efx_tx_reap(txq); + + for (txds = 0; txds <= txq->ptr_mask; txds++) { + if (txq->sw_ring[txds].mbuf != NULL) { + rte_pktmbuf_free(txq->sw_ring[txds].mbuf); + txq->sw_ring[txds].mbuf = NULL; + } + } + + txq->flags &= ~SFC_EFX_TXQ_FLAG_STARTED; +} + +struct sfc_dp_tx sfc_efx_tx = { + .dp = { + .name = SFC_KVARG_DATAPATH_EFX, + .type = SFC_DP_TX, + .hw_fw_caps = 0, + }, + .qcreate = sfc_efx_tx_qcreate, + .qdestroy = sfc_efx_tx_qdestroy, + .qstart = sfc_efx_tx_qstart, + .qstop = sfc_efx_tx_qstop, + .qreap = sfc_efx_tx_qreap, + .pkt_burst = sfc_efx_xmit_pkts, +}; diff --git a/drivers/net/sfc/sfc_tx.h b/drivers/net/sfc/sfc_tx.h index 35b65d3d78..94477f7d3a 100644 --- a/drivers/net/sfc/sfc_tx.h +++ b/drivers/net/sfc/sfc_tx.h @@ -37,6 +37,8 @@ #include "efx.h" +#include "sfc_dp_tx.h" + #ifdef __cplusplus extern "C" { #endif @@ -44,7 +46,11 @@ extern "C" { struct sfc_adapter; struct sfc_evq; -struct sfc_tx_sw_desc { +/** + * Software Tx descriptor information associated with hardware Tx + * descriptor. + */ +struct sfc_efx_tx_sw_desc { struct rte_mbuf *mbuf; uint8_t *tsoh; /* Buffer to store TSO header */ }; @@ -54,37 +60,71 @@ enum sfc_txq_state_bit { #define SFC_TXQ_INITIALIZED (1 << SFC_TXQ_INITIALIZED_BIT) SFC_TXQ_STARTED_BIT, #define SFC_TXQ_STARTED (1 << SFC_TXQ_STARTED_BIT) - SFC_TXQ_RUNNING_BIT, -#define SFC_TXQ_RUNNING (1 << SFC_TXQ_RUNNING_BIT) SFC_TXQ_FLUSHING_BIT, #define SFC_TXQ_FLUSHING (1 << SFC_TXQ_FLUSHING_BIT) SFC_TXQ_FLUSHED_BIT, #define SFC_TXQ_FLUSHED (1 << SFC_TXQ_FLUSHED_BIT) }; +/** + * Transmit queue control information. Not used on datapath. + * Allocated on the socket specified on the queue setup. + */ struct sfc_txq { - struct sfc_evq *evq; - struct sfc_tx_sw_desc *sw_ring; - unsigned int state; - unsigned int ptr_mask; - efx_desc_t *pend_desc; - efx_txq_t *common; - efsys_mem_t mem; - unsigned int added; - unsigned int pending; - unsigned int completed; - unsigned int free_thresh; - uint16_t hw_vlan_tci; - uint16_t dma_desc_size_max; - - unsigned int hw_index; - unsigned int flags; + unsigned int state; + unsigned int hw_index; + struct sfc_evq *evq; + efsys_mem_t mem; + struct sfc_dp_txq *dp; + efx_txq_t *common; + unsigned int free_thresh; + unsigned int flags; }; +static inline unsigned int +sfc_txq_sw_index_by_hw_index(unsigned int hw_index) +{ + return hw_index; +} + static inline unsigned int sfc_txq_sw_index(const struct sfc_txq *txq) { - return txq->hw_index; + return sfc_txq_sw_index_by_hw_index(txq->hw_index); +} + +struct sfc_txq *sfc_txq_by_dp_txq(const struct sfc_dp_txq *dp_txq); + +/** + * Transmit queue information used on libefx-based data path. + * Allocated on the socket specified on the queue setup. + */ +struct sfc_efx_txq { + struct sfc_evq *evq; + struct sfc_efx_tx_sw_desc *sw_ring; + unsigned int ptr_mask; + efx_desc_t *pend_desc; + efx_txq_t *common; + unsigned int added; + unsigned int pending; + unsigned int completed; + unsigned int free_thresh; + uint16_t hw_vlan_tci; + uint16_t dma_desc_size_max; + + unsigned int hw_index; + unsigned int flags; +#define SFC_EFX_TXQ_FLAG_STARTED 0x1 +#define SFC_EFX_TXQ_FLAG_RUNNING 0x2 + + /* Datapath transmit queue anchor */ + struct sfc_dp_txq dp; +}; + +static inline struct sfc_efx_txq * +sfc_efx_txq_by_dp_txq(struct sfc_dp_txq *dp_txq) +{ + return container_of(dp_txq, struct sfc_efx_txq, dp); } struct sfc_txq_info { @@ -108,17 +148,15 @@ void sfc_tx_qstop(struct sfc_adapter *sa, unsigned int sw_index); int sfc_tx_start(struct sfc_adapter *sa); void sfc_tx_stop(struct sfc_adapter *sa); -uint16_t sfc_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, - uint16_t nb_pkts); - /* From 'sfc_tso.c' */ -int sfc_tso_alloc_tsoh_objs(struct sfc_tx_sw_desc *sw_ring, - unsigned int txq_entries, unsigned int socket_id); -void sfc_tso_free_tsoh_objs(struct sfc_tx_sw_desc *sw_ring, - unsigned int txq_entries); -int sfc_tso_do(struct sfc_txq *txq, unsigned int idx, struct rte_mbuf **in_seg, - size_t *in_off, efx_desc_t **pend, unsigned int *pkt_descs, - size_t *pkt_len); +int sfc_efx_tso_alloc_tsoh_objs(struct sfc_efx_tx_sw_desc *sw_ring, + unsigned int txq_entries, + unsigned int socket_id); +void sfc_efx_tso_free_tsoh_objs(struct sfc_efx_tx_sw_desc *sw_ring, + unsigned int txq_entries); +int sfc_efx_tso_do(struct sfc_efx_txq *txq, unsigned int idx, + struct rte_mbuf **in_seg, size_t *in_off, efx_desc_t **pend, + unsigned int *pkt_descs, size_t *pkt_len); #ifdef __cplusplus } -- 2.20.1