more efficient than libefx-based and provides richer packet type
classification, but lacks Rx scatter support.
+- ``tx_datapath`` [auto|efx] (default **auto**)
+
+ Choose transmit datapath implementation.
+ **auto** allows the driver itself to make a choice based on firmware
+ features available and required by the datapath implementation.
+ **efx** chooses libefx-based datapath which supports VLAN insertion
+ (full-feature firmware variant only), TSO and multi-segment mbufs.
+
- ``perf_profile`` [auto|throughput|low-latency] (default **throughput**)
Choose hardware tunning to be optimized for either throughput or
#endif
const struct sfc_dp_rx *dp_rx;
+ const struct sfc_dp_tx *dp_tx;
};
/*
if (sfc_dp_find_by_name(head, entry->type, entry->name) != NULL) {
rte_log(RTE_LOG_ERR, RTE_LOGTYPE_PMD,
"sfc %s dapapath '%s' already registered\n",
- entry->type == SFC_DP_RX ? "Rx" : "unknown",
+ entry->type == SFC_DP_RX ? "Rx" :
+ entry->type == SFC_DP_TX ? "Tx" :
+ "unknown",
entry->name);
return EEXIST;
}
enum sfc_dp_type {
SFC_DP_RX = 0, /**< Receive datapath */
+ SFC_DP_TX, /**< Transmit datapath */
};
--- /dev/null
+/*-
+ * BSD LICENSE
+ *
+ * Copyright (c) 2016 Solarflare Communications Inc.
+ * All rights reserved.
+ *
+ * This software was jointly developed between OKTET Labs (under contract
+ * for Solarflare) and Solarflare Communications, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _SFC_DP_TX_H
+#define _SFC_DP_TX_H
+
+#include <rte_ethdev.h>
+
+#include "sfc_dp.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Generic transmit queue information used on data path.
+ * It must be kept as small as it is possible since it is built into
+ * the structure used on datapath.
+ */
+struct sfc_dp_txq {
+ struct sfc_dp_queue dpq;
+};
+
+/**
+ * Datapath transmit queue creation information.
+ *
+ * The structure is used just to pass information from control path to
+ * datapath. It could be just function arguments, but it would be hardly
+ * readable.
+ */
+struct sfc_dp_tx_qcreate_info {
+ /** Minimum number of unused Tx descriptors to do reap */
+ unsigned int free_thresh;
+ /** Transmit queue configuration flags */
+ unsigned int flags;
+ /** Tx queue size */
+ unsigned int txq_entries;
+ /** Maximum size of data in the DMA descriptor */
+ uint16_t dma_desc_size_max;
+};
+
+/**
+ * Allocate and initialize datapath transmit queue.
+ *
+ * @param port_id The port identifier
+ * @param queue_id The queue identifier
+ * @param pci_addr PCI function address
+ * @param socket_id Socket identifier to allocate memory
+ * @param info Tx queue details wrapped in structure
+ * @param dp_txqp Location for generic datapath transmit queue pointer
+ *
+ * @return 0 or positive errno.
+ */
+typedef int (sfc_dp_tx_qcreate_t)(uint16_t port_id, uint16_t queue_id,
+ const struct rte_pci_addr *pci_addr,
+ int socket_id,
+ const struct sfc_dp_tx_qcreate_info *info,
+ struct sfc_dp_txq **dp_txqp);
+
+/**
+ * Free resources allocated for datapath transmit queue.
+ */
+typedef void (sfc_dp_tx_qdestroy_t)(struct sfc_dp_txq *dp_txq);
+
+/**
+ * Transmit queue start callback.
+ *
+ * It handovers EvQ to the datapath.
+ */
+typedef int (sfc_dp_tx_qstart_t)(struct sfc_dp_txq *dp_txq,
+ unsigned int evq_read_ptr,
+ unsigned int txq_desc_index);
+
+/**
+ * Transmit queue stop function called before the queue flush.
+ *
+ * It returns EvQ to the control path.
+ */
+typedef void (sfc_dp_tx_qstop_t)(struct sfc_dp_txq *dp_txq,
+ unsigned int *evq_read_ptr);
+
+/**
+ * Transmit queue function called after the queue flush.
+ */
+typedef void (sfc_dp_tx_qreap_t)(struct sfc_dp_txq *dp_txq);
+
+/** Transmit datapath definition */
+struct sfc_dp_tx {
+ struct sfc_dp dp;
+
+ sfc_dp_tx_qcreate_t *qcreate;
+ sfc_dp_tx_qdestroy_t *qdestroy;
+ sfc_dp_tx_qstart_t *qstart;
+ sfc_dp_tx_qstop_t *qstop;
+ sfc_dp_tx_qreap_t *qreap;
+ eth_tx_burst_t pkt_burst;
+};
+
+static inline struct sfc_dp_tx *
+sfc_dp_find_tx_by_name(struct sfc_dp_list *head, const char *name)
+{
+ struct sfc_dp *p = sfc_dp_find_by_name(head, SFC_DP_TX, name);
+
+ return (p == NULL) ? NULL : container_of(p, struct sfc_dp_tx, dp);
+}
+
+static inline struct sfc_dp_tx *
+sfc_dp_find_tx_by_caps(struct sfc_dp_list *head, unsigned int avail_caps)
+{
+ struct sfc_dp *p = sfc_dp_find_by_caps(head, SFC_DP_TX, avail_caps);
+
+ return (p == NULL) ? NULL : container_of(p, struct sfc_dp_tx, dp);
+}
+
+extern struct sfc_dp_tx sfc_efx_tx;
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* _SFC_DP_TX_H */
if (rc != 0)
goto fail_tx_qinit;
- dev->data->tx_queues[tx_queue_id] = sa->txq_info[tx_queue_id].txq;
+ dev->data->tx_queues[tx_queue_id] = sa->txq_info[tx_queue_id].txq->dp;
sfc_adapter_unlock(sa);
return 0;
static void
sfc_tx_queue_release(void *queue)
{
- struct sfc_txq *txq = queue;
+ struct sfc_dp_txq *dp_txq = queue;
+ struct sfc_txq *txq;
unsigned int sw_index;
struct sfc_adapter *sa;
- if (txq == NULL)
+ if (dp_txq == NULL)
return;
+ txq = sfc_txq_by_dp_txq(dp_txq);
sw_index = sfc_txq_sw_index(txq);
SFC_ASSERT(txq->evq != NULL);
struct sfc_adapter *sa = dev->data->dev_private;
unsigned int avail_caps = 0;
const char *rx_name = NULL;
+ const char *tx_name = NULL;
int rc;
if (sa == NULL || sa->state == SFC_ADAPTER_UNINITIALIZED)
dev->rx_pkt_burst = sa->dp_rx->pkt_burst;
- dev->tx_pkt_burst = sfc_xmit_pkts;
+ rc = sfc_kvargs_process(sa, SFC_KVARG_TX_DATAPATH,
+ sfc_kvarg_string_handler, &tx_name);
+ if (rc != 0)
+ goto fail_kvarg_tx_datapath;
+
+ if (tx_name != NULL) {
+ sa->dp_tx = sfc_dp_find_tx_by_name(&sfc_dp_head, tx_name);
+ if (sa->dp_tx == NULL) {
+ sfc_err(sa, "Tx datapath %s not found", tx_name);
+ rc = ENOENT;
+ goto fail_dp_tx;
+ }
+ if (!sfc_dp_match_hw_fw_caps(&sa->dp_tx->dp, avail_caps)) {
+ sfc_err(sa,
+ "Insufficient Hw/FW capabilities to use Tx datapath %s",
+ tx_name);
+ rc = EINVAL;
+ goto fail_dp_tx;
+ }
+ } else {
+ sa->dp_tx = sfc_dp_find_tx_by_caps(&sfc_dp_head, avail_caps);
+ if (sa->dp_tx == NULL) {
+ sfc_err(sa, "Tx datapath by caps %#x not found",
+ avail_caps);
+ rc = ENOENT;
+ goto fail_dp_tx;
+ }
+ }
+
+ sfc_info(sa, "use %s Tx datapath", sa->dp_tx->dp.name);
+
+ dev->tx_pkt_burst = sa->dp_tx->pkt_burst;
dev->dev_ops = &sfc_eth_dev_ops;
return 0;
+fail_dp_tx:
+fail_kvarg_tx_datapath:
fail_dp_rx:
fail_kvarg_rx_datapath:
return rc;
/* Prefer EF10 datapath */
sfc_dp_register(&sfc_dp_head, &sfc_ef10_rx.dp);
sfc_dp_register(&sfc_dp_head, &sfc_efx_rx.dp);
+
+ sfc_dp_register(&sfc_dp_head, &sfc_efx_tx.dp);
}
}
RTE_PMD_REGISTER_KMOD_DEP(net_sfc_efx, "* igb_uio | uio_pci_generic | vfio");
RTE_PMD_REGISTER_PARAM_STRING(net_sfc_efx,
SFC_KVARG_RX_DATAPATH "=" SFC_KVARG_VALUES_RX_DATAPATH " "
+ SFC_KVARG_TX_DATAPATH "=" SFC_KVARG_VALUES_TX_DATAPATH " "
SFC_KVARG_PERF_PROFILE "=" SFC_KVARG_VALUES_PERF_PROFILE " "
SFC_KVARG_STATS_UPDATE_PERIOD_MS "=<long> "
SFC_KVARG_MCDI_LOGGING "=" SFC_KVARG_VALUES_BOOL " "
sfc_ev_tx(void *arg, __rte_unused uint32_t label, uint32_t id)
{
struct sfc_evq *evq = arg;
- struct sfc_txq *txq;
+ struct sfc_dp_txq *dp_txq;
+ struct sfc_efx_txq *txq;
unsigned int stop;
unsigned int delta;
- txq = evq->txq;
+ dp_txq = evq->dp_txq;
+ SFC_ASSERT(dp_txq != NULL);
- SFC_ASSERT(txq != NULL);
+ txq = sfc_efx_txq_by_dp_txq(dp_txq);
SFC_ASSERT(txq->evq == evq);
- if (unlikely((txq->state & SFC_TXQ_STARTED) == 0))
+ if (unlikely((txq->flags & SFC_EFX_TXQ_FLAG_STARTED) == 0))
goto done;
stop = (id + 1) & txq->ptr_mask;
sfc_ev_txq_flush_done(void *arg, __rte_unused uint32_t txq_hw_index)
{
struct sfc_evq *evq = arg;
+ struct sfc_dp_txq *dp_txq;
struct sfc_txq *txq;
- txq = evq->txq;
+ dp_txq = evq->dp_txq;
+ SFC_ASSERT(dp_txq != NULL);
+
+ txq = sfc_txq_by_dp_txq(dp_txq);
SFC_ASSERT(txq != NULL);
SFC_ASSERT(txq->hw_index == txq_hw_index);
SFC_ASSERT(txq->evq == evq);
.eec_link_change = sfc_ev_nop_link_change,
};
-static const efx_ev_callbacks_t sfc_ev_callbacks_tx = {
+static const efx_ev_callbacks_t sfc_ev_callbacks_efx_tx = {
.eec_initialized = sfc_ev_initialized,
.eec_rx = sfc_ev_nop_rx,
.eec_tx = sfc_ev_tx,
.eec_link_change = sfc_ev_nop_link_change,
};
+static const efx_ev_callbacks_t sfc_ev_callbacks_dp_tx = {
+ .eec_initialized = sfc_ev_initialized,
+ .eec_rx = sfc_ev_nop_rx,
+ .eec_tx = sfc_ev_nop_tx,
+ .eec_exception = sfc_ev_exception,
+ .eec_rxq_flush_done = sfc_ev_nop_rxq_flush_done,
+ .eec_rxq_flush_failed = sfc_ev_nop_rxq_flush_failed,
+ .eec_txq_flush_done = sfc_ev_txq_flush_done,
+ .eec_software = sfc_ev_software,
+ .eec_sram = sfc_ev_sram,
+ .eec_wake_up = sfc_ev_wake_up,
+ .eec_timer = sfc_ev_timer,
+ .eec_link_change = sfc_ev_nop_link_change,
+};
+
void
sfc_ev_qpoll(struct sfc_evq *evq)
rxq_sw_index);
}
- if (evq->txq != NULL) {
- unsigned int txq_sw_index = sfc_txq_sw_index(evq->txq);
+ if (evq->dp_txq != NULL) {
+ unsigned int txq_sw_index;
+
+ txq_sw_index = evq->dp_txq->dpq.queue_id;
sfc_warn(sa,
"restart TxQ %u because of exception on its EvQ %u",
if (rc != 0)
goto fail_ev_qcreate;
- SFC_ASSERT(evq->dp_rxq == NULL || evq->txq == NULL);
+ SFC_ASSERT(evq->dp_rxq == NULL || evq->dp_txq == NULL);
if (evq->dp_rxq != 0) {
if (strcmp(sa->dp_rx->dp.name, SFC_KVARG_DATAPATH_EFX) == 0)
evq->callbacks = &sfc_ev_callbacks_efx_rx;
else
evq->callbacks = &sfc_ev_callbacks_dp_rx;
- } else if (evq->txq != 0) {
- evq->callbacks = &sfc_ev_callbacks_tx;
+ } else if (evq->dp_txq != 0) {
+ if (strcmp(sa->dp_tx->dp.name, SFC_KVARG_DATAPATH_EFX) == 0)
+ evq->callbacks = &sfc_ev_callbacks_efx_tx;
+ else
+ evq->callbacks = &sfc_ev_callbacks_dp_tx;
} else {
evq->callbacks = &sfc_ev_callbacks;
}
#ifndef _SFC_EV_H_
#define _SFC_EV_H_
+#include <rte_ethdev.h>
+
#include "efx.h"
+#include "sfc.h"
+
#ifdef __cplusplus
extern "C" {
#endif
struct sfc_adapter;
struct sfc_dp_rxq;
-struct sfc_txq;
+struct sfc_dp_txq;
enum sfc_evq_state {
SFC_EVQ_UNINITIALIZED = 0,
boolean_t exception;
efsys_mem_t mem;
struct sfc_dp_rxq *dp_rxq;
- struct sfc_txq *txq;
+ struct sfc_dp_txq *dp_txq;
/* Not used on datapath */
struct sfc_adapter *sa;
SFC_KVARG_MCDI_LOGGING,
SFC_KVARG_PERF_PROFILE,
SFC_KVARG_RX_DATAPATH,
+ SFC_KVARG_TX_DATAPATH,
NULL,
};
"[" SFC_KVARG_DATAPATH_EFX "|" \
SFC_KVARG_DATAPATH_EF10 "]"
+#define SFC_KVARG_TX_DATAPATH "tx_datapath"
+#define SFC_KVARG_VALUES_TX_DATAPATH \
+ "[" SFC_KVARG_DATAPATH_EFX "]"
+
struct sfc_adapter;
int sfc_kvargs_parse(struct sfc_adapter *sa);
#define SFC_TSO_OPDESCS_IDX_SHIFT 2
int
-sfc_tso_alloc_tsoh_objs(struct sfc_tx_sw_desc *sw_ring,
- unsigned int txq_entries, unsigned int socket_id)
+sfc_efx_tso_alloc_tsoh_objs(struct sfc_efx_tx_sw_desc *sw_ring,
+ unsigned int txq_entries, unsigned int socket_id)
{
unsigned int i;
for (i = 0; i < txq_entries; ++i) {
- sw_ring[i].tsoh = rte_malloc_socket("sfc-txq-tsoh-obj",
+ sw_ring[i].tsoh = rte_malloc_socket("sfc-efx-txq-tsoh-obj",
SFC_TSOH_STD_LEN,
RTE_CACHE_LINE_SIZE,
socket_id);
}
void
-sfc_tso_free_tsoh_objs(struct sfc_tx_sw_desc *sw_ring, unsigned int txq_entries)
+sfc_efx_tso_free_tsoh_objs(struct sfc_efx_tx_sw_desc *sw_ring,
+ unsigned int txq_entries)
{
unsigned int i;
}
static void
-sfc_tso_prepare_header(struct sfc_txq *txq, struct rte_mbuf **in_seg,
- size_t *in_off, unsigned int idx, size_t bytes_left)
+sfc_efx_tso_prepare_header(struct sfc_efx_txq *txq, struct rte_mbuf **in_seg,
+ size_t *in_off, unsigned int idx, size_t bytes_left)
{
struct rte_mbuf *m = *in_seg;
size_t bytes_to_copy = 0;
}
int
-sfc_tso_do(struct sfc_txq *txq, unsigned int idx, struct rte_mbuf **in_seg,
- size_t *in_off, efx_desc_t **pend, unsigned int *pkt_descs,
- size_t *pkt_len)
+sfc_efx_tso_do(struct sfc_efx_txq *txq, unsigned int idx,
+ struct rte_mbuf **in_seg, size_t *in_off, efx_desc_t **pend,
+ unsigned int *pkt_descs, size_t *pkt_len)
{
uint8_t *tsoh;
const struct tcp_hdr *th;
* limitations on address boundaries crossing by DMA descriptor data.
*/
if (m->data_len < header_len) {
- sfc_tso_prepare_header(txq, in_seg, in_off, idx, header_len);
+ sfc_efx_tso_prepare_header(txq, in_seg, in_off, idx,
+ header_len);
tsoh = txq->sw_ring[idx & txq->ptr_mask].tsoh;
header_paddr = rte_malloc_virt2phy((void *)tsoh);
#include "sfc_ev.h"
#include "sfc_tx.h"
#include "sfc_tweak.h"
+#include "sfc_kvargs.h"
/*
* Maximum number of TX queue flush attempts in case of
txq->state &= ~SFC_TXQ_FLUSHING;
}
-static void
-sfc_tx_reap(struct sfc_txq *txq)
-{
- unsigned int completed;
-
-
- sfc_ev_qpoll(txq->evq);
-
- for (completed = txq->completed;
- completed != txq->pending; completed++) {
- struct sfc_tx_sw_desc *txd;
-
- txd = &txq->sw_ring[completed & txq->ptr_mask];
-
- if (txd->mbuf != NULL) {
- rte_pktmbuf_free(txd->mbuf);
- txd->mbuf = NULL;
- }
- }
-
- txq->completed = completed;
-}
-
int
sfc_tx_qinit(struct sfc_adapter *sa, unsigned int sw_index,
uint16_t nb_tx_desc, unsigned int socket_id,
struct sfc_txq *txq;
unsigned int evq_index = sfc_evq_index_by_txq_sw_index(sa, sw_index);
int rc = 0;
+ struct sfc_dp_tx_qcreate_info info;
sfc_log_init(sa, "TxQ = %u", sw_index);
if (txq == NULL)
goto fail_txq_alloc;
+ txq_info->txq = txq;
+
+ txq->hw_index = sw_index;
+ txq->evq = evq;
+ txq->free_thresh =
+ (tx_conf->tx_free_thresh) ? tx_conf->tx_free_thresh :
+ SFC_TX_DEFAULT_FREE_THRESH;
+ txq->flags = tx_conf->txq_flags;
+
rc = sfc_dma_alloc(sa, "txq", sw_index, EFX_TXQ_SIZE(txq_info->entries),
socket_id, &txq->mem);
if (rc != 0)
goto fail_dma_alloc;
- rc = ENOMEM;
- txq->pend_desc = rte_calloc_socket("sfc-txq-pend-desc",
- EFX_TXQ_LIMIT(txq_info->entries),
- sizeof(efx_desc_t), 0, socket_id);
- if (txq->pend_desc == NULL)
- goto fail_pend_desc_alloc;
+ memset(&info, 0, sizeof(info));
+ info.free_thresh = txq->free_thresh;
+ info.flags = tx_conf->txq_flags;
+ info.txq_entries = txq_info->entries;
+ info.dma_desc_size_max = encp->enc_tx_dma_desc_size_max;
- rc = ENOMEM;
- txq->sw_ring = rte_calloc_socket("sfc-txq-desc", txq_info->entries,
- sizeof(*txq->sw_ring), 0, socket_id);
- if (txq->sw_ring == NULL)
- goto fail_desc_alloc;
+ rc = sa->dp_tx->qcreate(sa->eth_dev->data->port_id, sw_index,
+ &SFC_DEV_TO_PCI(sa->eth_dev)->addr,
+ socket_id, &info, &txq->dp);
+ if (rc != 0)
+ goto fail_dp_tx_qinit;
- if (sa->tso) {
- rc = sfc_tso_alloc_tsoh_objs(txq->sw_ring, txq_info->entries,
- socket_id);
- if (rc != 0)
- goto fail_alloc_tsoh_objs;
- }
+ evq->dp_txq = txq->dp;
txq->state = SFC_TXQ_INITIALIZED;
- txq->ptr_mask = txq_info->entries - 1;
- txq->free_thresh = (tx_conf->tx_free_thresh) ? tx_conf->tx_free_thresh :
- SFC_TX_DEFAULT_FREE_THRESH;
- txq->dma_desc_size_max = encp->enc_tx_dma_desc_size_max;
- txq->hw_index = sw_index;
- txq->flags = tx_conf->txq_flags;
- txq->evq = evq;
- evq->txq = txq;
-
- txq_info->txq = txq;
txq_info->deferred_start = (tx_conf->tx_deferred_start != 0);
return 0;
-fail_alloc_tsoh_objs:
- rte_free(txq->sw_ring);
-
-fail_desc_alloc:
- rte_free(txq->pend_desc);
-
-fail_pend_desc_alloc:
+fail_dp_tx_qinit:
sfc_dma_free(sa, &txq->mem);
fail_dma_alloc:
+ txq_info->txq = NULL;
rte_free(txq);
fail_txq_alloc:
SFC_ASSERT(txq != NULL);
SFC_ASSERT(txq->state == SFC_TXQ_INITIALIZED);
- sfc_tso_free_tsoh_objs(txq->sw_ring, txq_info->entries);
+ sa->dp_tx->qdestroy(txq->dp);
+ txq->dp = NULL;
txq_info->txq = NULL;
txq_info->entries = 0;
- rte_free(txq->sw_ring);
- rte_free(txq->pend_desc);
sfc_dma_free(sa, &txq->mem);
rte_free(txq);
}
goto fail_tx_qcreate;
}
- txq->added = txq->pending = txq->completed = desc_index;
- txq->hw_vlan_tci = 0;
-
efx_tx_qenable(txq->common);
- txq->state |= (SFC_TXQ_STARTED | SFC_TXQ_RUNNING);
+ txq->state |= SFC_TXQ_STARTED;
+
+ rc = sa->dp_tx->qstart(txq->dp, evq->read_ptr, desc_index);
+ if (rc != 0)
+ goto fail_dp_qstart;
/*
* It seems to be used by DPDK for debug purposes only ('rte_ether')
return 0;
+fail_dp_qstart:
+ txq->state = SFC_TXQ_INITIALIZED;
+ efx_tx_qdestroy(txq->common);
+
fail_tx_qcreate:
sfc_ev_qstop(sa, evq->evq_index);
struct sfc_txq *txq;
unsigned int retry_count;
unsigned int wait_count;
- unsigned int txds;
sfc_log_init(sa, "TxQ = %u", sw_index);
SFC_ASSERT(txq->state & SFC_TXQ_STARTED);
- txq->state &= ~SFC_TXQ_RUNNING;
+ sa->dp_tx->qstop(txq->dp, &txq->evq->read_ptr);
/*
* Retry TX queue flushing in case of flush failed or
sfc_info(sa, "TxQ %u flushed", sw_index);
}
- sfc_tx_reap(txq);
-
- for (txds = 0; txds < txq_info->entries; txds++) {
- if (txq->sw_ring[txds].mbuf != NULL) {
- rte_pktmbuf_free(txq->sw_ring[txds].mbuf);
- txq->sw_ring[txds].mbuf = NULL;
- }
- }
+ sa->dp_tx->qreap(txq->dp);
txq->state = SFC_TXQ_INITIALIZED;
efx_tx_fini(sa->nic);
}
+static void
+sfc_efx_tx_reap(struct sfc_efx_txq *txq)
+{
+ unsigned int completed;
+
+ sfc_ev_qpoll(txq->evq);
+
+ for (completed = txq->completed;
+ completed != txq->pending; completed++) {
+ struct sfc_efx_tx_sw_desc *txd;
+
+ txd = &txq->sw_ring[completed & txq->ptr_mask];
+
+ if (txd->mbuf != NULL) {
+ rte_pktmbuf_free(txd->mbuf);
+ txd->mbuf = NULL;
+ }
+ }
+
+ txq->completed = completed;
+}
+
/*
* The function is used to insert or update VLAN tag;
* the firmware has state of the firmware tag to insert per TxQ
* the function will update it
*/
static unsigned int
-sfc_tx_maybe_insert_tag(struct sfc_txq *txq, struct rte_mbuf *m,
- efx_desc_t **pend)
+sfc_efx_tx_maybe_insert_tag(struct sfc_efx_txq *txq, struct rte_mbuf *m,
+ efx_desc_t **pend)
{
uint16_t this_tag = ((m->ol_flags & PKT_TX_VLAN_PKT) ?
m->vlan_tci : 0);
return 1;
}
-uint16_t
-sfc_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+static uint16_t
+sfc_efx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
{
- struct sfc_txq *txq = (struct sfc_txq *)tx_queue;
+ struct sfc_dp_txq *dp_txq = (struct sfc_dp_txq *)tx_queue;
+ struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq);
unsigned int added = txq->added;
unsigned int pushed = added;
unsigned int pkts_sent = 0;
int rc __rte_unused;
struct rte_mbuf **pktp;
- if (unlikely((txq->state & SFC_TXQ_RUNNING) == 0))
+ if (unlikely((txq->flags & SFC_EFX_TXQ_FLAG_RUNNING) == 0))
goto done;
/*
reap_done = (fill_level > soft_max_fill);
if (reap_done) {
- sfc_tx_reap(txq);
+ sfc_efx_tx_reap(txq);
/*
* Recalculate fill level since 'txq->completed'
* might have changed on reap
* DEV_TX_VLAN_OFFLOAD and pushes VLAN TCI, then
* TX_ERROR will occur
*/
- pkt_descs += sfc_tx_maybe_insert_tag(txq, m_seg, &pend);
+ pkt_descs += sfc_efx_tx_maybe_insert_tag(txq, m_seg, &pend);
if (m_seg->ol_flags & PKT_TX_TCP_SEG) {
/*
* We expect correct 'pkt->l[2, 3, 4]_len' values
* to be set correctly by the caller
*/
- if (sfc_tso_do(txq, added, &m_seg, &in_off, &pend,
- &pkt_descs, &pkt_len) != 0) {
+ if (sfc_efx_tso_do(txq, added, &m_seg, &in_off, &pend,
+ &pkt_descs, &pkt_len) != 0) {
/* We may have reached this place for
* one of the following reasons:
*
* Try to reap (if we haven't yet).
*/
if (!reap_done) {
- sfc_tx_reap(txq);
+ sfc_efx_tx_reap(txq);
reap_done = B_TRUE;
fill_level = added - txq->completed;
if (fill_level > hard_max_fill) {
#if SFC_TX_XMIT_PKTS_REAP_AT_LEAST_ONCE
if (!reap_done)
- sfc_tx_reap(txq);
+ sfc_efx_tx_reap(txq);
#endif
done:
return pkts_sent;
}
+
+struct sfc_txq *
+sfc_txq_by_dp_txq(const struct sfc_dp_txq *dp_txq)
+{
+ const struct sfc_dp_queue *dpq = &dp_txq->dpq;
+ struct rte_eth_dev *eth_dev;
+ struct sfc_adapter *sa;
+ struct sfc_txq *txq;
+
+ SFC_ASSERT(rte_eth_dev_is_valid_port(dpq->port_id));
+ eth_dev = &rte_eth_devices[dpq->port_id];
+
+ sa = eth_dev->data->dev_private;
+
+ SFC_ASSERT(dpq->queue_id < sa->txq_count);
+ txq = sa->txq_info[dpq->queue_id].txq;
+
+ SFC_ASSERT(txq != NULL);
+ return txq;
+}
+
+static sfc_dp_tx_qcreate_t sfc_efx_tx_qcreate;
+static int
+sfc_efx_tx_qcreate(uint16_t port_id, uint16_t queue_id,
+ const struct rte_pci_addr *pci_addr,
+ int socket_id,
+ const struct sfc_dp_tx_qcreate_info *info,
+ struct sfc_dp_txq **dp_txqp)
+{
+ struct sfc_efx_txq *txq;
+ struct sfc_txq *ctrl_txq;
+ int rc;
+
+ rc = ENOMEM;
+ txq = rte_zmalloc_socket("sfc-efx-txq", sizeof(*txq),
+ RTE_CACHE_LINE_SIZE, socket_id);
+ if (txq == NULL)
+ goto fail_txq_alloc;
+
+ sfc_dp_queue_init(&txq->dp.dpq, port_id, queue_id, pci_addr);
+
+ rc = ENOMEM;
+ txq->pend_desc = rte_calloc_socket("sfc-efx-txq-pend-desc",
+ EFX_TXQ_LIMIT(info->txq_entries),
+ sizeof(*txq->pend_desc), 0,
+ socket_id);
+ if (txq->pend_desc == NULL)
+ goto fail_pend_desc_alloc;
+
+ rc = ENOMEM;
+ txq->sw_ring = rte_calloc_socket("sfc-efx-txq-sw_ring",
+ info->txq_entries,
+ sizeof(*txq->sw_ring),
+ RTE_CACHE_LINE_SIZE, socket_id);
+ if (txq->sw_ring == NULL)
+ goto fail_sw_ring_alloc;
+
+ ctrl_txq = sfc_txq_by_dp_txq(&txq->dp);
+ if (ctrl_txq->evq->sa->tso) {
+ rc = sfc_efx_tso_alloc_tsoh_objs(txq->sw_ring,
+ info->txq_entries, socket_id);
+ if (rc != 0)
+ goto fail_alloc_tsoh_objs;
+ }
+
+ txq->evq = ctrl_txq->evq;
+ txq->ptr_mask = info->txq_entries - 1;
+ txq->free_thresh = info->free_thresh;
+ txq->dma_desc_size_max = info->dma_desc_size_max;
+
+ *dp_txqp = &txq->dp;
+ return 0;
+
+fail_alloc_tsoh_objs:
+ rte_free(txq->sw_ring);
+
+fail_sw_ring_alloc:
+ rte_free(txq->pend_desc);
+
+fail_pend_desc_alloc:
+ rte_free(txq);
+
+fail_txq_alloc:
+ return rc;
+}
+
+static sfc_dp_tx_qdestroy_t sfc_efx_tx_qdestroy;
+static void
+sfc_efx_tx_qdestroy(struct sfc_dp_txq *dp_txq)
+{
+ struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq);
+
+ sfc_efx_tso_free_tsoh_objs(txq->sw_ring, txq->ptr_mask + 1);
+ rte_free(txq->sw_ring);
+ rte_free(txq->pend_desc);
+ rte_free(txq);
+}
+
+static sfc_dp_tx_qstart_t sfc_efx_tx_qstart;
+static int
+sfc_efx_tx_qstart(struct sfc_dp_txq *dp_txq,
+ __rte_unused unsigned int evq_read_ptr,
+ unsigned int txq_desc_index)
+{
+ /* libefx-based datapath is specific to libefx-based PMD */
+ struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq);
+ struct sfc_txq *ctrl_txq = sfc_txq_by_dp_txq(dp_txq);
+
+ txq->common = ctrl_txq->common;
+
+ txq->pending = txq->completed = txq->added = txq_desc_index;
+ txq->hw_vlan_tci = 0;
+
+ txq->flags |= (SFC_EFX_TXQ_FLAG_STARTED | SFC_EFX_TXQ_FLAG_RUNNING);
+
+ return 0;
+}
+
+static sfc_dp_tx_qstop_t sfc_efx_tx_qstop;
+static void
+sfc_efx_tx_qstop(struct sfc_dp_txq *dp_txq,
+ __rte_unused unsigned int *evq_read_ptr)
+{
+ struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq);
+
+ txq->flags &= ~SFC_EFX_TXQ_FLAG_RUNNING;
+}
+
+static sfc_dp_tx_qreap_t sfc_efx_tx_qreap;
+static void
+sfc_efx_tx_qreap(struct sfc_dp_txq *dp_txq)
+{
+ struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq);
+ unsigned int txds;
+
+ sfc_efx_tx_reap(txq);
+
+ for (txds = 0; txds <= txq->ptr_mask; txds++) {
+ if (txq->sw_ring[txds].mbuf != NULL) {
+ rte_pktmbuf_free(txq->sw_ring[txds].mbuf);
+ txq->sw_ring[txds].mbuf = NULL;
+ }
+ }
+
+ txq->flags &= ~SFC_EFX_TXQ_FLAG_STARTED;
+}
+
+struct sfc_dp_tx sfc_efx_tx = {
+ .dp = {
+ .name = SFC_KVARG_DATAPATH_EFX,
+ .type = SFC_DP_TX,
+ .hw_fw_caps = 0,
+ },
+ .qcreate = sfc_efx_tx_qcreate,
+ .qdestroy = sfc_efx_tx_qdestroy,
+ .qstart = sfc_efx_tx_qstart,
+ .qstop = sfc_efx_tx_qstop,
+ .qreap = sfc_efx_tx_qreap,
+ .pkt_burst = sfc_efx_xmit_pkts,
+};
#include "efx.h"
+#include "sfc_dp_tx.h"
+
#ifdef __cplusplus
extern "C" {
#endif
struct sfc_adapter;
struct sfc_evq;
-struct sfc_tx_sw_desc {
+/**
+ * Software Tx descriptor information associated with hardware Tx
+ * descriptor.
+ */
+struct sfc_efx_tx_sw_desc {
struct rte_mbuf *mbuf;
uint8_t *tsoh; /* Buffer to store TSO header */
};
#define SFC_TXQ_INITIALIZED (1 << SFC_TXQ_INITIALIZED_BIT)
SFC_TXQ_STARTED_BIT,
#define SFC_TXQ_STARTED (1 << SFC_TXQ_STARTED_BIT)
- SFC_TXQ_RUNNING_BIT,
-#define SFC_TXQ_RUNNING (1 << SFC_TXQ_RUNNING_BIT)
SFC_TXQ_FLUSHING_BIT,
#define SFC_TXQ_FLUSHING (1 << SFC_TXQ_FLUSHING_BIT)
SFC_TXQ_FLUSHED_BIT,
#define SFC_TXQ_FLUSHED (1 << SFC_TXQ_FLUSHED_BIT)
};
+/**
+ * Transmit queue control information. Not used on datapath.
+ * Allocated on the socket specified on the queue setup.
+ */
struct sfc_txq {
- struct sfc_evq *evq;
- struct sfc_tx_sw_desc *sw_ring;
- unsigned int state;
- unsigned int ptr_mask;
- efx_desc_t *pend_desc;
- efx_txq_t *common;
- efsys_mem_t mem;
- unsigned int added;
- unsigned int pending;
- unsigned int completed;
- unsigned int free_thresh;
- uint16_t hw_vlan_tci;
- uint16_t dma_desc_size_max;
-
- unsigned int hw_index;
- unsigned int flags;
+ unsigned int state;
+ unsigned int hw_index;
+ struct sfc_evq *evq;
+ efsys_mem_t mem;
+ struct sfc_dp_txq *dp;
+ efx_txq_t *common;
+ unsigned int free_thresh;
+ unsigned int flags;
};
+static inline unsigned int
+sfc_txq_sw_index_by_hw_index(unsigned int hw_index)
+{
+ return hw_index;
+}
+
static inline unsigned int
sfc_txq_sw_index(const struct sfc_txq *txq)
{
- return txq->hw_index;
+ return sfc_txq_sw_index_by_hw_index(txq->hw_index);
+}
+
+struct sfc_txq *sfc_txq_by_dp_txq(const struct sfc_dp_txq *dp_txq);
+
+/**
+ * Transmit queue information used on libefx-based data path.
+ * Allocated on the socket specified on the queue setup.
+ */
+struct sfc_efx_txq {
+ struct sfc_evq *evq;
+ struct sfc_efx_tx_sw_desc *sw_ring;
+ unsigned int ptr_mask;
+ efx_desc_t *pend_desc;
+ efx_txq_t *common;
+ unsigned int added;
+ unsigned int pending;
+ unsigned int completed;
+ unsigned int free_thresh;
+ uint16_t hw_vlan_tci;
+ uint16_t dma_desc_size_max;
+
+ unsigned int hw_index;
+ unsigned int flags;
+#define SFC_EFX_TXQ_FLAG_STARTED 0x1
+#define SFC_EFX_TXQ_FLAG_RUNNING 0x2
+
+ /* Datapath transmit queue anchor */
+ struct sfc_dp_txq dp;
+};
+
+static inline struct sfc_efx_txq *
+sfc_efx_txq_by_dp_txq(struct sfc_dp_txq *dp_txq)
+{
+ return container_of(dp_txq, struct sfc_efx_txq, dp);
}
struct sfc_txq_info {
int sfc_tx_start(struct sfc_adapter *sa);
void sfc_tx_stop(struct sfc_adapter *sa);
-uint16_t sfc_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
- uint16_t nb_pkts);
-
/* From 'sfc_tso.c' */
-int sfc_tso_alloc_tsoh_objs(struct sfc_tx_sw_desc *sw_ring,
- unsigned int txq_entries, unsigned int socket_id);
-void sfc_tso_free_tsoh_objs(struct sfc_tx_sw_desc *sw_ring,
- unsigned int txq_entries);
-int sfc_tso_do(struct sfc_txq *txq, unsigned int idx, struct rte_mbuf **in_seg,
- size_t *in_off, efx_desc_t **pend, unsigned int *pkt_descs,
- size_t *pkt_len);
+int sfc_efx_tso_alloc_tsoh_objs(struct sfc_efx_tx_sw_desc *sw_ring,
+ unsigned int txq_entries,
+ unsigned int socket_id);
+void sfc_efx_tso_free_tsoh_objs(struct sfc_efx_tx_sw_desc *sw_ring,
+ unsigned int txq_entries);
+int sfc_efx_tso_do(struct sfc_efx_txq *txq, unsigned int idx,
+ struct rte_mbuf **in_seg, size_t *in_off, efx_desc_t **pend,
+ unsigned int *pkt_descs, size_t *pkt_len);
#ifdef __cplusplus
}
git.droids-corp.org / dpdk.git / commitdiff