1 /* SPDX-License-Identifier: BSD-3-Clause
3 * Copyright(c) 2019-2020 Xilinx, Inc.
4 * Copyright(c) 2016-2019 Solarflare Communications Inc.
6 * This software was jointly developed between OKTET Labs (under contract
7 * for Solarflare) and Solarflare Communications, Inc.
13 #include <rte_ethdev_driver.h>
16 #include "sfc_debug.h"
24 * Generic transmit queue information used on data path.
25 * It must be kept as small as it is possible since it is built into
26 * the structure used on datapath.
29 struct sfc_dp_queue dpq;
32 /** Datapath transmit queue descriptor number limitations */
33 struct sfc_dp_tx_hw_limits {
34 unsigned int txq_max_entries;
35 unsigned int txq_min_entries;
39 * Datapath transmit queue creation information.
41 * The structure is used just to pass information from control path to
42 * datapath. It could be just function arguments, but it would be hardly
45 struct sfc_dp_tx_qcreate_info {
46 /** Maximum number of pushed Tx descriptors */
47 unsigned int max_fill_level;
48 /** Minimum number of unused Tx descriptors to do reap */
49 unsigned int free_thresh;
50 /** Offloads enabled on the transmit queue */
53 unsigned int txq_entries;
54 /** Maximum size of data in the DMA descriptor */
55 uint16_t dma_desc_size_max;
56 /** DMA-mapped Tx descriptors ring */
58 /** Associated event queue size */
59 unsigned int evq_entries;
60 /** Hardware event ring */
62 /** The queue index in hardware (required to push right doorbell) */
63 unsigned int hw_index;
64 /** Virtual address of the memory-mapped BAR to push Tx doorbell */
65 volatile void *mem_bar;
66 /** VI window size shift */
67 unsigned int vi_window_shift;
69 * Maximum number of bytes into the packet the TCP header can start for
70 * the hardware to apply TSO packet edits.
72 uint16_t tso_tcp_header_offset_limit;
76 * Get Tx datapath specific device info.
78 * @param dev_info Device info to be adjusted
80 typedef void (sfc_dp_tx_get_dev_info_t)(struct rte_eth_dev_info *dev_info);
83 * Get size of transmit and event queue rings by the number of Tx
86 * @param nb_tx_desc Number of Tx descriptors
87 * @param txq_entries Location for number of Tx ring entries
88 * @param evq_entries Location for number of event ring entries
89 * @param txq_max_fill_level Location for maximum Tx ring fill level
91 * @return 0 or positive errno.
93 typedef int (sfc_dp_tx_qsize_up_rings_t)(uint16_t nb_tx_desc,
94 struct sfc_dp_tx_hw_limits *limits,
95 unsigned int *txq_entries,
96 unsigned int *evq_entries,
97 unsigned int *txq_max_fill_level);
100 * Allocate and initialize datapath transmit queue.
102 * @param port_id The port identifier
103 * @param queue_id The queue identifier
104 * @param pci_addr PCI function address
105 * @param socket_id Socket identifier to allocate memory
106 * @param info Tx queue details wrapped in structure
107 * @param dp_txqp Location for generic datapath transmit queue pointer
109 * @return 0 or positive errno.
111 typedef int (sfc_dp_tx_qcreate_t)(uint16_t port_id, uint16_t queue_id,
112 const struct rte_pci_addr *pci_addr,
114 const struct sfc_dp_tx_qcreate_info *info,
115 struct sfc_dp_txq **dp_txqp);
118 * Free resources allocated for datapath transmit queue.
120 typedef void (sfc_dp_tx_qdestroy_t)(struct sfc_dp_txq *dp_txq);
123 * Transmit queue start callback.
125 * It handovers EvQ to the datapath.
127 typedef int (sfc_dp_tx_qstart_t)(struct sfc_dp_txq *dp_txq,
128 unsigned int evq_read_ptr,
129 unsigned int txq_desc_index);
132 * Transmit queue stop function called before the queue flush.
134 * It returns EvQ to the control path.
136 typedef void (sfc_dp_tx_qstop_t)(struct sfc_dp_txq *dp_txq,
137 unsigned int *evq_read_ptr);
140 * Transmit event handler used during queue flush only.
142 typedef bool (sfc_dp_tx_qtx_ev_t)(struct sfc_dp_txq *dp_txq, unsigned int id);
145 * Transmit queue function called after the queue flush.
147 typedef void (sfc_dp_tx_qreap_t)(struct sfc_dp_txq *dp_txq);
150 * Check Tx descriptor status
152 typedef int (sfc_dp_tx_qdesc_status_t)(struct sfc_dp_txq *dp_txq,
155 /** Transmit datapath definition */
159 unsigned int features;
160 #define SFC_DP_TX_FEAT_MULTI_PROCESS 0x1
162 * Tx offload capabilities supported by the datapath on device
163 * level only if HW/FW supports it.
165 uint64_t dev_offload_capa;
167 * Tx offload capabilities supported by the datapath per-queue
168 * if HW/FW supports it.
170 uint64_t queue_offload_capa;
171 sfc_dp_tx_get_dev_info_t *get_dev_info;
172 sfc_dp_tx_qsize_up_rings_t *qsize_up_rings;
173 sfc_dp_tx_qcreate_t *qcreate;
174 sfc_dp_tx_qdestroy_t *qdestroy;
175 sfc_dp_tx_qstart_t *qstart;
176 sfc_dp_tx_qstop_t *qstop;
177 sfc_dp_tx_qtx_ev_t *qtx_ev;
178 sfc_dp_tx_qreap_t *qreap;
179 sfc_dp_tx_qdesc_status_t *qdesc_status;
180 eth_tx_prep_t pkt_prepare;
181 eth_tx_burst_t pkt_burst;
184 static inline struct sfc_dp_tx *
185 sfc_dp_find_tx_by_name(struct sfc_dp_list *head, const char *name)
187 struct sfc_dp *p = sfc_dp_find_by_name(head, SFC_DP_TX, name);
189 return (p == NULL) ? NULL : container_of(p, struct sfc_dp_tx, dp);
192 static inline struct sfc_dp_tx *
193 sfc_dp_find_tx_by_caps(struct sfc_dp_list *head, unsigned int avail_caps)
195 struct sfc_dp *p = sfc_dp_find_by_caps(head, SFC_DP_TX, avail_caps);
197 return (p == NULL) ? NULL : container_of(p, struct sfc_dp_tx, dp);
200 /** Get Tx datapath ops by the datapath TxQ handle */
201 const struct sfc_dp_tx *sfc_dp_tx_by_dp_txq(const struct sfc_dp_txq *dp_txq);
203 static inline uint64_t
204 sfc_dp_tx_offload_capa(const struct sfc_dp_tx *dp_tx)
206 return dp_tx->dev_offload_capa | dp_tx->queue_offload_capa;
209 static inline unsigned int
210 sfc_dp_tx_pkt_extra_hdr_segs(struct rte_mbuf **m_seg,
211 unsigned int *header_len_remaining)
213 unsigned int nb_extra_header_segs = 0;
215 while (rte_pktmbuf_data_len(*m_seg) < *header_len_remaining) {
216 *header_len_remaining -= rte_pktmbuf_data_len(*m_seg);
217 *m_seg = (*m_seg)->next;
218 ++nb_extra_header_segs;
221 return nb_extra_header_segs;
225 sfc_dp_tx_prepare_pkt(struct rte_mbuf *m,
226 unsigned int max_nb_header_segs,
227 unsigned int tso_bounce_buffer_len,
228 uint32_t tso_tcp_header_offset_limit,
229 unsigned int max_fill_level,
230 unsigned int nb_tso_descs,
231 unsigned int nb_vlan_descs)
233 unsigned int descs_required = m->nb_segs;
234 unsigned int tcph_off = ((m->ol_flags & PKT_TX_TUNNEL_MASK) ?
235 m->outer_l2_len + m->outer_l3_len : 0) +
236 m->l2_len + m->l3_len;
237 unsigned int header_len = tcph_off + m->l4_len;
238 unsigned int header_len_remaining = header_len;
239 unsigned int nb_header_segs = 1;
240 struct rte_mbuf *m_seg = m;
242 #ifdef RTE_LIBRTE_SFC_EFX_DEBUG
245 ret = rte_validate_tx_offload(m);
248 * Negative error code is returned by rte_validate_tx_offload(),
249 * but positive are used inside net/sfc PMD.
256 if (max_nb_header_segs != 0) {
257 /* There is a limit on the number of header segments. */
260 sfc_dp_tx_pkt_extra_hdr_segs(&m_seg,
261 &header_len_remaining);
263 if (unlikely(nb_header_segs > max_nb_header_segs)) {
265 * The number of header segments is too large.
267 * If TSO is requested and if the datapath supports
268 * linearisation of TSO headers, allow the packet
269 * to proceed with additional checks below.
270 * Otherwise, throw an error.
272 if ((m->ol_flags & PKT_TX_TCP_SEG) == 0 ||
273 tso_bounce_buffer_len == 0)
278 if (m->ol_flags & PKT_TX_TCP_SEG) {
279 switch (m->ol_flags & PKT_TX_TUNNEL_MASK) {
282 case PKT_TX_TUNNEL_VXLAN:
284 case PKT_TX_TUNNEL_GENEVE:
286 (PKT_TX_OUTER_IPV4 | PKT_TX_OUTER_IPV6)))
290 if (unlikely(tcph_off > tso_tcp_header_offset_limit))
293 descs_required += nb_tso_descs;
296 * If headers segments are already counted above, here
297 * nothing is done since remaining length is smaller
298 * then current segment size.
301 sfc_dp_tx_pkt_extra_hdr_segs(&m_seg,
302 &header_len_remaining);
305 * Extra descriptor which is required when (a part of) payload
306 * shares the same segment with (a part of) the header.
308 if (rte_pktmbuf_data_len(m_seg) > header_len_remaining)
311 if (tso_bounce_buffer_len != 0) {
312 if (nb_header_segs > 1 &&
313 unlikely(header_len > tso_bounce_buffer_len)) {
315 * Header linearization is required and
316 * the header is too big to be linearized
324 * The number of VLAN descriptors is added regardless of requested
325 * VLAN offload since VLAN is sticky and sending packet without VLAN
326 * insertion may require VLAN descriptor to reset the sticky to 0.
328 descs_required += nb_vlan_descs;
331 * Max fill level must be sufficient to hold all required descriptors
332 * to send the packet entirely.
334 if (descs_required > max_fill_level)
340 extern struct sfc_dp_tx sfc_efx_tx;
341 extern struct sfc_dp_tx sfc_ef10_tx;
342 extern struct sfc_dp_tx sfc_ef10_simple_tx;
343 extern struct sfc_dp_tx sfc_ef100_tx;
348 #endif /* _SFC_DP_TX_H */