1 /* SPDX-License-Identifier: BSD-3-Clause
3 * Copyright(c) 2019-2020 Xilinx, Inc.
4 * Copyright(c) 2016-2019 Solarflare Communications Inc.
6 * This software was jointly developed between OKTET Labs (under contract
7 * for Solarflare) and Solarflare Communications, Inc.
13 #include <rte_ethdev_driver.h>
16 #include "sfc_debug.h"
24 * Generic transmit queue information used on data path.
25 * It must be kept as small as it is possible since it is built into
26 * the structure used on datapath.
29 struct sfc_dp_queue dpq;
32 /** Datapath transmit queue descriptor number limitations */
33 struct sfc_dp_tx_hw_limits {
34 unsigned int txq_max_entries;
35 unsigned int txq_min_entries;
39 * Datapath transmit queue creation information.
41 * The structure is used just to pass information from control path to
42 * datapath. It could be just function arguments, but it would be hardly
45 struct sfc_dp_tx_qcreate_info {
46 /** Maximum number of pushed Tx descriptors */
47 unsigned int max_fill_level;
48 /** Minimum number of unused Tx descriptors to do reap */
49 unsigned int free_thresh;
50 /** Offloads enabled on the transmit queue */
53 unsigned int txq_entries;
54 /** Maximum size of data in the DMA descriptor */
55 uint16_t dma_desc_size_max;
56 /** DMA-mapped Tx descriptors ring */
58 /** Associated event queue size */
59 unsigned int evq_entries;
60 /** Hardware event ring */
62 /** The queue index in hardware (required to push right doorbell) */
63 unsigned int hw_index;
64 /** Virtual address of the memory-mapped BAR to push Tx doorbell */
65 volatile void *mem_bar;
66 /** VI window size shift */
67 unsigned int vi_window_shift;
69 * Maximum number of bytes into the packet the TCP header can start for
70 * the hardware to apply TSO packet edits.
72 uint16_t tso_tcp_header_offset_limit;
73 /** Maximum number of header DMA descriptors per TSOv3 transaction */
74 uint16_t tso_max_nb_header_descs;
75 /** Maximum header length acceptable by TSOv3 transaction */
76 uint16_t tso_max_header_len;
77 /** Maximum number of payload DMA descriptors per TSOv3 transaction */
78 uint16_t tso_max_nb_payload_descs;
79 /** Maximum payload length per TSOv3 transaction */
80 uint32_t tso_max_payload_len;
81 /** Maximum number of frames to be generated per TSOv3 transaction */
82 uint32_t tso_max_nb_outgoing_frames;
86 * Get Tx datapath specific device info.
88 * @param dev_info Device info to be adjusted
90 typedef void (sfc_dp_tx_get_dev_info_t)(struct rte_eth_dev_info *dev_info);
93 * Get size of transmit and event queue rings by the number of Tx
96 * @param nb_tx_desc Number of Tx descriptors
97 * @param txq_entries Location for number of Tx ring entries
98 * @param evq_entries Location for number of event ring entries
99 * @param txq_max_fill_level Location for maximum Tx ring fill level
101 * @return 0 or positive errno.
103 typedef int (sfc_dp_tx_qsize_up_rings_t)(uint16_t nb_tx_desc,
104 struct sfc_dp_tx_hw_limits *limits,
105 unsigned int *txq_entries,
106 unsigned int *evq_entries,
107 unsigned int *txq_max_fill_level);
110 * Allocate and initialize datapath transmit queue.
112 * @param port_id The port identifier
113 * @param queue_id The queue identifier
114 * @param pci_addr PCI function address
115 * @param socket_id Socket identifier to allocate memory
116 * @param info Tx queue details wrapped in structure
117 * @param dp_txqp Location for generic datapath transmit queue pointer
119 * @return 0 or positive errno.
121 typedef int (sfc_dp_tx_qcreate_t)(uint16_t port_id, uint16_t queue_id,
122 const struct rte_pci_addr *pci_addr,
124 const struct sfc_dp_tx_qcreate_info *info,
125 struct sfc_dp_txq **dp_txqp);
128 * Free resources allocated for datapath transmit queue.
130 typedef void (sfc_dp_tx_qdestroy_t)(struct sfc_dp_txq *dp_txq);
133 * Transmit queue start callback.
135 * It handovers EvQ to the datapath.
137 typedef int (sfc_dp_tx_qstart_t)(struct sfc_dp_txq *dp_txq,
138 unsigned int evq_read_ptr,
139 unsigned int txq_desc_index);
142 * Transmit queue stop function called before the queue flush.
144 * It returns EvQ to the control path.
146 typedef void (sfc_dp_tx_qstop_t)(struct sfc_dp_txq *dp_txq,
147 unsigned int *evq_read_ptr);
150 * Transmit event handler used during queue flush only.
152 typedef bool (sfc_dp_tx_qtx_ev_t)(struct sfc_dp_txq *dp_txq, unsigned int id);
155 * Transmit queue function called after the queue flush.
157 typedef void (sfc_dp_tx_qreap_t)(struct sfc_dp_txq *dp_txq);
160 * Check Tx descriptor status
162 typedef int (sfc_dp_tx_qdesc_status_t)(struct sfc_dp_txq *dp_txq,
165 /** Transmit datapath definition */
169 unsigned int features;
170 #define SFC_DP_TX_FEAT_MULTI_PROCESS 0x1
172 * Tx offload capabilities supported by the datapath on device
173 * level only if HW/FW supports it.
175 uint64_t dev_offload_capa;
177 * Tx offload capabilities supported by the datapath per-queue
178 * if HW/FW supports it.
180 uint64_t queue_offload_capa;
181 sfc_dp_tx_get_dev_info_t *get_dev_info;
182 sfc_dp_tx_qsize_up_rings_t *qsize_up_rings;
183 sfc_dp_tx_qcreate_t *qcreate;
184 sfc_dp_tx_qdestroy_t *qdestroy;
185 sfc_dp_tx_qstart_t *qstart;
186 sfc_dp_tx_qstop_t *qstop;
187 sfc_dp_tx_qtx_ev_t *qtx_ev;
188 sfc_dp_tx_qreap_t *qreap;
189 sfc_dp_tx_qdesc_status_t *qdesc_status;
190 eth_tx_prep_t pkt_prepare;
191 eth_tx_burst_t pkt_burst;
194 static inline struct sfc_dp_tx *
195 sfc_dp_find_tx_by_name(struct sfc_dp_list *head, const char *name)
197 struct sfc_dp *p = sfc_dp_find_by_name(head, SFC_DP_TX, name);
199 return (p == NULL) ? NULL : container_of(p, struct sfc_dp_tx, dp);
202 static inline struct sfc_dp_tx *
203 sfc_dp_find_tx_by_caps(struct sfc_dp_list *head, unsigned int avail_caps)
205 struct sfc_dp *p = sfc_dp_find_by_caps(head, SFC_DP_TX, avail_caps);
207 return (p == NULL) ? NULL : container_of(p, struct sfc_dp_tx, dp);
210 /** Get Tx datapath ops by the datapath TxQ handle */
211 const struct sfc_dp_tx *sfc_dp_tx_by_dp_txq(const struct sfc_dp_txq *dp_txq);
213 static inline uint64_t
214 sfc_dp_tx_offload_capa(const struct sfc_dp_tx *dp_tx)
216 return dp_tx->dev_offload_capa | dp_tx->queue_offload_capa;
219 static inline unsigned int
220 sfc_dp_tx_pkt_extra_hdr_segs(struct rte_mbuf **m_seg,
221 unsigned int *header_len_remaining)
223 unsigned int nb_extra_header_segs = 0;
225 while (rte_pktmbuf_data_len(*m_seg) < *header_len_remaining) {
226 *header_len_remaining -= rte_pktmbuf_data_len(*m_seg);
227 *m_seg = (*m_seg)->next;
228 ++nb_extra_header_segs;
231 return nb_extra_header_segs;
235 sfc_dp_tx_prepare_pkt(struct rte_mbuf *m,
236 unsigned int max_nb_header_segs,
237 unsigned int tso_bounce_buffer_len,
238 uint32_t tso_tcp_header_offset_limit,
239 unsigned int max_fill_level,
240 unsigned int nb_tso_descs,
241 unsigned int nb_vlan_descs)
243 unsigned int descs_required = m->nb_segs;
244 unsigned int tcph_off = ((m->ol_flags & PKT_TX_TUNNEL_MASK) ?
245 m->outer_l2_len + m->outer_l3_len : 0) +
246 m->l2_len + m->l3_len;
247 unsigned int header_len = tcph_off + m->l4_len;
248 unsigned int header_len_remaining = header_len;
249 unsigned int nb_header_segs = 1;
250 struct rte_mbuf *m_seg = m;
252 #ifdef RTE_LIBRTE_SFC_EFX_DEBUG
255 ret = rte_validate_tx_offload(m);
258 * Negative error code is returned by rte_validate_tx_offload(),
259 * but positive are used inside net/sfc PMD.
266 if (max_nb_header_segs != 0) {
267 /* There is a limit on the number of header segments. */
270 sfc_dp_tx_pkt_extra_hdr_segs(&m_seg,
271 &header_len_remaining);
273 if (unlikely(nb_header_segs > max_nb_header_segs)) {
275 * The number of header segments is too large.
277 * If TSO is requested and if the datapath supports
278 * linearisation of TSO headers, allow the packet
279 * to proceed with additional checks below.
280 * Otherwise, throw an error.
282 if ((m->ol_flags & PKT_TX_TCP_SEG) == 0 ||
283 tso_bounce_buffer_len == 0)
288 if (m->ol_flags & PKT_TX_TCP_SEG) {
289 switch (m->ol_flags & PKT_TX_TUNNEL_MASK) {
292 case PKT_TX_TUNNEL_VXLAN:
294 case PKT_TX_TUNNEL_GENEVE:
296 (PKT_TX_OUTER_IPV4 | PKT_TX_OUTER_IPV6)))
300 if (unlikely(tcph_off > tso_tcp_header_offset_limit))
303 descs_required += nb_tso_descs;
306 * If headers segments are already counted above, here
307 * nothing is done since remaining length is smaller
308 * then current segment size.
311 sfc_dp_tx_pkt_extra_hdr_segs(&m_seg,
312 &header_len_remaining);
315 * Extra descriptor which is required when (a part of) payload
316 * shares the same segment with (a part of) the header.
318 if (rte_pktmbuf_data_len(m_seg) > header_len_remaining)
321 if (tso_bounce_buffer_len != 0) {
322 if (nb_header_segs > 1 &&
323 unlikely(header_len > tso_bounce_buffer_len)) {
325 * Header linearization is required and
326 * the header is too big to be linearized
334 * The number of VLAN descriptors is added regardless of requested
335 * VLAN offload since VLAN is sticky and sending packet without VLAN
336 * insertion may require VLAN descriptor to reset the sticky to 0.
338 descs_required += nb_vlan_descs;
341 * Max fill level must be sufficient to hold all required descriptors
342 * to send the packet entirely.
344 if (descs_required > max_fill_level)
350 extern struct sfc_dp_tx sfc_efx_tx;
351 extern struct sfc_dp_tx sfc_ef10_tx;
352 extern struct sfc_dp_tx sfc_ef10_simple_tx;
353 extern struct sfc_dp_tx sfc_ef100_tx;
358 #endif /* _SFC_DP_TX_H */