1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2021 6WIND S.A.
3 * Copyright 2021 Mellanox Technologies, Ltd
6 #ifndef RTE_PMD_MLX5_TX_H_
7 #define RTE_PMD_MLX5_TX_H_
10 #include <sys/queue.h>
13 #include <rte_mempool.h>
14 #include <rte_common.h>
15 #include <rte_spinlock.h>
17 #include <mlx5_common_mr.h>
20 #include "mlx5_autoconf.h"
22 /* TX burst subroutines return codes. */
23 enum mlx5_txcmp_code {
24 MLX5_TXCMP_CODE_EXIT = 0,
25 MLX5_TXCMP_CODE_ERROR,
26 MLX5_TXCMP_CODE_SINGLE,
27 MLX5_TXCMP_CODE_MULTI,
33 * These defines are used to configure Tx burst routine option set supported
34 * at compile time. The not specified options are optimized out due to if
35 * conditions can be explicitly calculated at compile time.
36 * The offloads with bigger runtime check (require more CPU cycles toskip)
37 * overhead should have the bigger index - this is needed to select the better
38 * matching routine function if no exact match and some offloads are not
41 #define MLX5_TXOFF_CONFIG_MULTI (1u << 0) /* Multi-segment packets.*/
42 #define MLX5_TXOFF_CONFIG_TSO (1u << 1) /* TCP send offload supported.*/
43 #define MLX5_TXOFF_CONFIG_SWP (1u << 2) /* Tunnels/SW Parser offloads.*/
44 #define MLX5_TXOFF_CONFIG_CSUM (1u << 3) /* Check Sums offloaded. */
45 #define MLX5_TXOFF_CONFIG_INLINE (1u << 4) /* Data inlining supported. */
46 #define MLX5_TXOFF_CONFIG_VLAN (1u << 5) /* VLAN insertion supported.*/
47 #define MLX5_TXOFF_CONFIG_METADATA (1u << 6) /* Flow metadata. */
48 #define MLX5_TXOFF_CONFIG_EMPW (1u << 8) /* Enhanced MPW supported.*/
49 #define MLX5_TXOFF_CONFIG_MPW (1u << 9) /* Legacy MPW supported.*/
50 #define MLX5_TXOFF_CONFIG_TXPP (1u << 10) /* Scheduling on timestamp.*/
52 /* The most common offloads groups. */
53 #define MLX5_TXOFF_CONFIG_NONE 0
54 #define MLX5_TXOFF_CONFIG_FULL (MLX5_TXOFF_CONFIG_MULTI | \
55 MLX5_TXOFF_CONFIG_TSO | \
56 MLX5_TXOFF_CONFIG_SWP | \
57 MLX5_TXOFF_CONFIG_CSUM | \
58 MLX5_TXOFF_CONFIG_INLINE | \
59 MLX5_TXOFF_CONFIG_VLAN | \
60 MLX5_TXOFF_CONFIG_METADATA)
62 #define MLX5_TXOFF_CONFIG(mask) (olx & MLX5_TXOFF_CONFIG_##mask)
64 #define MLX5_TXOFF_PRE_DECL(func) \
65 uint16_t mlx5_tx_burst_##func(void *txq, \
66 struct rte_mbuf **pkts, \
69 #define MLX5_TXOFF_DECL(func, olx) \
70 uint16_t mlx5_tx_burst_##func(void *txq, \
71 struct rte_mbuf **pkts, \
74 return mlx5_tx_burst_tmpl((struct mlx5_txq_data *)txq, \
75 pkts, pkts_n, (olx)); \
78 /* Mbuf dynamic flag offset for inline. */
79 extern uint64_t rte_net_mlx5_dynf_inline_mask;
80 #define RTE_MBUF_F_TX_DYNF_NOINLINE rte_net_mlx5_dynf_inline_mask
82 extern uint32_t mlx5_ptype_table[] __rte_cache_aligned;
83 extern uint8_t mlx5_cksum_table[1 << 10] __rte_cache_aligned;
84 extern uint8_t mlx5_swp_types_table[1 << 10] __rte_cache_aligned;
86 struct mlx5_txq_stats {
87 #ifdef MLX5_PMD_SOFT_COUNTERS
88 uint64_t opackets; /**< Total of successfully sent packets. */
89 uint64_t obytes; /**< Total of successfully sent bytes. */
91 uint64_t oerrors; /**< Total number of failed transmitted packets. */
94 /* TX queue send local data. */
96 struct mlx5_txq_local {
97 struct mlx5_wqe *wqe_last; /* last sent WQE pointer. */
98 struct rte_mbuf *mbuf; /* first mbuf to process. */
99 uint16_t pkts_copy; /* packets copied to elts. */
100 uint16_t pkts_sent; /* packets sent. */
101 uint16_t pkts_loop; /* packets sent on loop entry. */
102 uint16_t elts_free; /* available elts remain. */
103 uint16_t wqe_free; /* available wqe remain. */
104 uint16_t mbuf_off; /* data offset in current mbuf. */
105 uint16_t mbuf_nseg; /* number of remaining mbuf. */
106 uint16_t mbuf_free; /* number of inline mbufs to free. */
109 /* TX queue descriptor. */
111 struct mlx5_txq_data {
112 uint16_t elts_head; /* Current counter in (*elts)[]. */
113 uint16_t elts_tail; /* Counter of first element awaiting completion. */
114 uint16_t elts_comp; /* elts index since last completion request. */
115 uint16_t elts_s; /* Number of mbuf elements. */
116 uint16_t elts_m; /* Mask for mbuf elements indices. */
117 /* Fields related to elts mbuf storage. */
118 uint16_t wqe_ci; /* Consumer index for work queue. */
119 uint16_t wqe_pi; /* Producer index for work queue. */
120 uint16_t wqe_s; /* Number of WQ elements. */
121 uint16_t wqe_m; /* Mask Number for WQ elements. */
122 uint16_t wqe_comp; /* WQE index since last completion request. */
123 uint16_t wqe_thres; /* WQE threshold to request completion in CQ. */
124 /* WQ related fields. */
125 uint16_t cq_ci; /* Consumer index for completion queue. */
126 uint16_t cq_pi; /* Production index for completion queue. */
127 uint16_t cqe_s; /* Number of CQ elements. */
128 uint16_t cqe_m; /* Mask for CQ indices. */
129 /* CQ related fields. */
130 uint16_t elts_n:4; /* elts[] length (in log2). */
131 uint16_t cqe_n:4; /* Number of CQ elements (in log2). */
132 uint16_t wqe_n:4; /* Number of WQ elements (in log2). */
133 uint16_t tso_en:1; /* When set hardware TSO is enabled. */
134 uint16_t tunnel_en:1;
135 /* When set TX offload for tunneled packets are supported. */
136 uint16_t swp_en:1; /* Whether SW parser is enabled. */
137 uint16_t vlan_en:1; /* VLAN insertion in WQE is supported. */
138 uint16_t db_nc:1; /* Doorbell mapped to non-cached region. */
139 uint16_t db_heu:1; /* Doorbell heuristic write barrier. */
140 uint16_t fast_free:1; /* mbuf fast free on Tx is enabled. */
141 uint16_t inlen_send; /* Ordinary send data inline size. */
142 uint16_t inlen_empw; /* eMPW max packet size to inline. */
143 uint16_t inlen_mode; /* Minimal data length to inline. */
144 uint32_t qp_num_8s; /* QP number shifted by 8. */
145 uint64_t offloads; /* Offloads for Tx Queue. */
146 struct mlx5_mr_ctrl mr_ctrl; /* MR control descriptor. */
147 struct mlx5_wqe *wqes; /* Work queue. */
148 struct mlx5_wqe *wqes_end; /* Work queue array limit. */
149 #ifdef RTE_LIBRTE_MLX5_DEBUG
150 uint32_t *fcqs; /* Free completion queue (debug extended). */
152 uint16_t *fcqs; /* Free completion queue. */
154 volatile struct mlx5_cqe *cqes; /* Completion queue. */
155 volatile uint32_t *qp_db; /* Work queue doorbell. */
156 volatile uint32_t *cq_db; /* Completion queue doorbell. */
157 uint16_t port_id; /* Port ID of device. */
158 uint16_t idx; /* Queue index. */
159 uint64_t ts_mask; /* Timestamp flag dynamic mask. */
160 int32_t ts_offset; /* Timestamp field dynamic offset. */
161 struct mlx5_dev_ctx_shared *sh; /* Shared context. */
162 struct mlx5_txq_stats stats; /* TX queue counters. */
164 rte_spinlock_t *uar_lock;
165 /* UAR access lock required for 32bit implementations */
167 struct rte_mbuf *elts[0];
168 /* Storage for queued packets, must be the last field. */
169 } __rte_cache_aligned;
172 MLX5_TXQ_TYPE_STANDARD, /* Standard Tx queue. */
173 MLX5_TXQ_TYPE_HAIRPIN, /* Hairpin Tx queue. */
176 /* TX queue control descriptor. */
177 struct mlx5_txq_ctrl {
178 LIST_ENTRY(mlx5_txq_ctrl) next; /* Pointer to the next element. */
179 uint32_t refcnt; /* Reference counter. */
180 unsigned int socket; /* CPU socket ID for allocations. */
181 enum mlx5_txq_type type; /* The txq ctrl type. */
182 unsigned int max_inline_data; /* Max inline data. */
183 unsigned int max_tso_header; /* Max TSO header size. */
184 struct mlx5_txq_obj *obj; /* Verbs/DevX queue object. */
185 struct mlx5_priv *priv; /* Back pointer to private data. */
186 off_t uar_mmap_offset; /* UAR mmap offset for non-primary process. */
187 void *bf_reg; /* BlueFlame register from Verbs. */
188 uint16_t dump_file_n; /* Number of dump files. */
189 struct rte_eth_hairpin_conf hairpin_conf; /* Hairpin configuration. */
190 uint32_t hairpin_status; /* Hairpin binding status. */
191 struct mlx5_txq_data txq; /* Data path structure. */
192 /* Must be the last field in the structure, contains elts[]. */
197 int mlx5_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id);
198 int mlx5_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id);
199 int mlx5_tx_queue_start_primary(struct rte_eth_dev *dev, uint16_t queue_id);
200 int mlx5_tx_queue_stop_primary(struct rte_eth_dev *dev, uint16_t queue_id);
201 int mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
202 unsigned int socket, const struct rte_eth_txconf *conf);
203 int mlx5_tx_hairpin_queue_setup
204 (struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
205 const struct rte_eth_hairpin_conf *hairpin_conf);
206 void mlx5_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid);
207 void txq_uar_init(struct mlx5_txq_ctrl *txq_ctrl);
208 int mlx5_tx_uar_init_secondary(struct rte_eth_dev *dev, int fd);
209 void mlx5_tx_uar_uninit_secondary(struct rte_eth_dev *dev);
210 int mlx5_txq_obj_verify(struct rte_eth_dev *dev);
211 struct mlx5_txq_ctrl *mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx,
212 uint16_t desc, unsigned int socket,
213 const struct rte_eth_txconf *conf);
214 struct mlx5_txq_ctrl *mlx5_txq_hairpin_new
215 (struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
216 const struct rte_eth_hairpin_conf *hairpin_conf);
217 struct mlx5_txq_ctrl *mlx5_txq_get(struct rte_eth_dev *dev, uint16_t idx);
218 int mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx);
219 int mlx5_txq_releasable(struct rte_eth_dev *dev, uint16_t idx);
220 int mlx5_txq_verify(struct rte_eth_dev *dev);
221 void txq_alloc_elts(struct mlx5_txq_ctrl *txq_ctrl);
222 void txq_free_elts(struct mlx5_txq_ctrl *txq_ctrl);
223 uint64_t mlx5_get_tx_port_offloads(struct rte_eth_dev *dev);
224 void mlx5_txq_dynf_timestamp_set(struct rte_eth_dev *dev);
228 uint16_t removed_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts,
230 void mlx5_tx_handle_completion(struct mlx5_txq_data *__rte_restrict txq,
231 unsigned int olx __rte_unused);
232 int mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset);
233 void mlx5_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
234 struct rte_eth_txq_info *qinfo);
235 int mlx5_tx_burst_mode_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
236 struct rte_eth_burst_mode *mode);
240 MLX5_TXOFF_PRE_DECL(full_empw);
241 MLX5_TXOFF_PRE_DECL(none_empw);
242 MLX5_TXOFF_PRE_DECL(md_empw);
243 MLX5_TXOFF_PRE_DECL(mt_empw);
244 MLX5_TXOFF_PRE_DECL(mtsc_empw);
245 MLX5_TXOFF_PRE_DECL(mti_empw);
246 MLX5_TXOFF_PRE_DECL(mtv_empw);
247 MLX5_TXOFF_PRE_DECL(mtiv_empw);
248 MLX5_TXOFF_PRE_DECL(sc_empw);
249 MLX5_TXOFF_PRE_DECL(sci_empw);
250 MLX5_TXOFF_PRE_DECL(scv_empw);
251 MLX5_TXOFF_PRE_DECL(sciv_empw);
252 MLX5_TXOFF_PRE_DECL(i_empw);
253 MLX5_TXOFF_PRE_DECL(v_empw);
254 MLX5_TXOFF_PRE_DECL(iv_empw);
256 /* mlx5_tx_nompw.c */
258 MLX5_TXOFF_PRE_DECL(full);
259 MLX5_TXOFF_PRE_DECL(none);
260 MLX5_TXOFF_PRE_DECL(md);
261 MLX5_TXOFF_PRE_DECL(mt);
262 MLX5_TXOFF_PRE_DECL(mtsc);
263 MLX5_TXOFF_PRE_DECL(mti);
264 MLX5_TXOFF_PRE_DECL(mtv);
265 MLX5_TXOFF_PRE_DECL(mtiv);
266 MLX5_TXOFF_PRE_DECL(sc);
267 MLX5_TXOFF_PRE_DECL(sci);
268 MLX5_TXOFF_PRE_DECL(scv);
269 MLX5_TXOFF_PRE_DECL(sciv);
270 MLX5_TXOFF_PRE_DECL(i);
271 MLX5_TXOFF_PRE_DECL(v);
272 MLX5_TXOFF_PRE_DECL(iv);
276 MLX5_TXOFF_PRE_DECL(full_ts_nompw);
277 MLX5_TXOFF_PRE_DECL(full_ts_nompwi);
278 MLX5_TXOFF_PRE_DECL(full_ts);
279 MLX5_TXOFF_PRE_DECL(full_ts_noi);
280 MLX5_TXOFF_PRE_DECL(none_ts);
281 MLX5_TXOFF_PRE_DECL(mdi_ts);
282 MLX5_TXOFF_PRE_DECL(mti_ts);
283 MLX5_TXOFF_PRE_DECL(mtiv_ts);
287 MLX5_TXOFF_PRE_DECL(none_mpw);
288 MLX5_TXOFF_PRE_DECL(mci_mpw);
289 MLX5_TXOFF_PRE_DECL(mc_mpw);
290 MLX5_TXOFF_PRE_DECL(i_mpw);
292 static __rte_always_inline uint64_t *
293 mlx5_tx_bfreg(struct mlx5_txq_data *txq)
295 return MLX5_PROC_PRIV(txq->port_id)->uar_table[txq->idx];
299 * Provide safe 64bit store operation to mlx5 UAR region for both 32bit and
300 * 64bit architectures.
303 * value to write in CPU endian format.
305 * Address to write to.
307 * Address of the lock to use for that UAR access.
309 static __rte_always_inline void
310 __mlx5_uar_write64_relaxed(uint64_t val, void *addr,
311 rte_spinlock_t *lock __rte_unused)
314 *(uint64_t *)addr = val;
315 #else /* !RTE_ARCH_64 */
316 rte_spinlock_lock(lock);
317 *(uint32_t *)addr = val;
319 *((uint32_t *)addr + 1) = val >> 32;
320 rte_spinlock_unlock(lock);
325 * Provide safe 64bit store operation to mlx5 UAR region for both 32bit and
326 * 64bit architectures while guaranteeing the order of execution with the
327 * code being executed.
330 * value to write in CPU endian format.
332 * Address to write to.
334 * Address of the lock to use for that UAR access.
336 static __rte_always_inline void
337 __mlx5_uar_write64(uint64_t val, void *addr, rte_spinlock_t *lock)
340 __mlx5_uar_write64_relaxed(val, addr, lock);
343 /* Assist macros, used instead of directly calling the functions they wrap. */
345 #define mlx5_uar_write64_relaxed(val, dst, lock) \
346 __mlx5_uar_write64_relaxed(val, dst, NULL)
347 #define mlx5_uar_write64(val, dst, lock) __mlx5_uar_write64(val, dst, NULL)
349 #define mlx5_uar_write64_relaxed(val, dst, lock) \
350 __mlx5_uar_write64_relaxed(val, dst, lock)
351 #define mlx5_uar_write64(val, dst, lock) __mlx5_uar_write64(val, dst, lock)
355 * Query LKey from a packet buffer for Tx.
358 * Pointer to Tx queue structure.
363 * Searched LKey on success, UINT32_MAX on no match.
365 static __rte_always_inline uint32_t
366 mlx5_tx_mb2mr(struct mlx5_txq_data *txq, struct rte_mbuf *mb)
368 struct mlx5_mr_ctrl *mr_ctrl = &txq->mr_ctrl;
369 struct mlx5_txq_ctrl *txq_ctrl =
370 container_of(txq, struct mlx5_txq_ctrl, txq);
372 /* Take slower bottom-half on miss. */
373 return mlx5_mr_mb2mr(mr_ctrl, mb, &txq_ctrl->priv->mp_id);
377 * Ring TX queue doorbell and flush the update if requested.
380 * Pointer to TX queue structure.
382 * Pointer to the last WQE posted in the NIC.
384 * Request for write memory barrier after BlueFlame update.
386 static __rte_always_inline void
387 mlx5_tx_dbrec_cond_wmb(struct mlx5_txq_data *txq, volatile struct mlx5_wqe *wqe,
390 uint64_t *dst = mlx5_tx_bfreg(txq);
391 volatile uint64_t *src = ((volatile uint64_t *)wqe);
394 *txq->qp_db = rte_cpu_to_be_32(txq->wqe_ci);
395 /* Ensure ordering between DB record and BF copy. */
397 mlx5_uar_write64_relaxed(*src, dst, txq->uar_lock);
403 * Ring TX queue doorbell and flush the update by write memory barrier.
406 * Pointer to TX queue structure.
408 * Pointer to the last WQE posted in the NIC.
410 static __rte_always_inline void
411 mlx5_tx_dbrec(struct mlx5_txq_data *txq, volatile struct mlx5_wqe *wqe)
413 mlx5_tx_dbrec_cond_wmb(txq, wqe, 1);
417 * Convert timestamp from mbuf format to linear counter
418 * of Clock Queue completions (24 bits).
421 * Pointer to the device shared context to fetch Tx
422 * packet pacing timestamp and parameters.
424 * Timestamp from mbuf to convert.
426 * positive or zero value - completion ID to wait.
427 * negative value - conversion error.
429 static __rte_always_inline int32_t
430 mlx5_txpp_convert_tx_ts(struct mlx5_dev_ctx_shared *sh, uint64_t mts)
437 * Read atomically two uint64_t fields and compare lsb bits.
438 * It there is no match - the timestamp was updated in
439 * the service thread, data should be re-read.
441 rte_compiler_barrier();
442 ci = __atomic_load_n(&sh->txpp.ts.ci_ts, __ATOMIC_RELAXED);
443 ts = __atomic_load_n(&sh->txpp.ts.ts, __ATOMIC_RELAXED);
444 rte_compiler_barrier();
445 if (!((ts ^ ci) << (64 - MLX5_CQ_INDEX_WIDTH)))
448 /* Perform the skew correction, positive value to send earlier. */
449 mts -= sh->txpp.skew;
451 if (unlikely(mts >= UINT64_MAX / 2)) {
452 /* We have negative integer, mts is in the past. */
453 __atomic_fetch_add(&sh->txpp.err_ts_past,
454 1, __ATOMIC_RELAXED);
457 tick = sh->txpp.tick;
459 /* Convert delta to completions, round up. */
460 mts = (mts + tick - 1) / tick;
461 if (unlikely(mts >= (1 << MLX5_CQ_INDEX_WIDTH) / 2 - 1)) {
462 /* We have mts is too distant future. */
463 __atomic_fetch_add(&sh->txpp.err_ts_future,
464 1, __ATOMIC_RELAXED);
467 mts <<= 64 - MLX5_CQ_INDEX_WIDTH;
469 ci >>= 64 - MLX5_CQ_INDEX_WIDTH;
474 * Set Software Parser flags and offsets in Ethernet Segment of WQE.
475 * Flags must be preliminary initialized to zero.
478 * Pointer to burst routine local context.
480 * Pointer to store Software Parser flags.
482 * Configured Tx offloads mask. It is fully defined at
483 * compile time and may be used for optimization.
486 * Software Parser offsets packed in dword.
487 * Software Parser flags are set by pointer.
489 static __rte_always_inline uint32_t
490 txq_mbuf_to_swp(struct mlx5_txq_local *__rte_restrict loc,
495 unsigned int idx, off;
498 if (!MLX5_TXOFF_CONFIG(SWP))
500 ol = loc->mbuf->ol_flags;
501 tunnel = ol & RTE_MBUF_F_TX_TUNNEL_MASK;
503 * Check whether Software Parser is required.
504 * Only customized tunnels may ask for.
506 if (likely(tunnel != RTE_MBUF_F_TX_TUNNEL_UDP && tunnel != RTE_MBUF_F_TX_TUNNEL_IP))
509 * The index should have:
510 * bit[0:1] = RTE_MBUF_F_TX_L4_MASK
511 * bit[4] = RTE_MBUF_F_TX_IPV6
512 * bit[8] = RTE_MBUF_F_TX_OUTER_IPV6
513 * bit[9] = RTE_MBUF_F_TX_OUTER_UDP
515 idx = (ol & (RTE_MBUF_F_TX_L4_MASK | RTE_MBUF_F_TX_IPV6 | RTE_MBUF_F_TX_OUTER_IPV6)) >> 52;
516 idx |= (tunnel == RTE_MBUF_F_TX_TUNNEL_UDP) ? (1 << 9) : 0;
517 *swp_flags = mlx5_swp_types_table[idx];
519 * Set offsets for SW parser. Since ConnectX-5, SW parser just
520 * complements HW parser. SW parser starts to engage only if HW parser
521 * can't reach a header. For the older devices, HW parser will not kick
522 * in if any of SWP offsets is set. Therefore, all of the L3 offsets
523 * should be set regardless of HW offload.
525 off = loc->mbuf->outer_l2_len;
526 if (MLX5_TXOFF_CONFIG(VLAN) && ol & RTE_MBUF_F_TX_VLAN)
527 off += sizeof(struct rte_vlan_hdr);
528 set = (off >> 1) << 8; /* Outer L3 offset. */
529 off += loc->mbuf->outer_l3_len;
530 if (tunnel == RTE_MBUF_F_TX_TUNNEL_UDP)
531 set |= off >> 1; /* Outer L4 offset. */
532 if (ol & (RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_IPV6)) { /* Inner IP. */
533 const uint64_t csum = ol & RTE_MBUF_F_TX_L4_MASK;
534 off += loc->mbuf->l2_len;
535 set |= (off >> 1) << 24; /* Inner L3 offset. */
536 if (csum == RTE_MBUF_F_TX_TCP_CKSUM ||
537 csum == RTE_MBUF_F_TX_UDP_CKSUM ||
538 (MLX5_TXOFF_CONFIG(TSO) && ol & RTE_MBUF_F_TX_TCP_SEG)) {
539 off += loc->mbuf->l3_len;
540 set |= (off >> 1) << 16; /* Inner L4 offset. */
543 set = rte_cpu_to_le_32(set);
548 * Convert the Checksum offloads to Verbs.
551 * Pointer to the mbuf.
554 * Converted checksum flags.
556 static __rte_always_inline uint8_t
557 txq_ol_cksum_to_cs(struct rte_mbuf *buf)
560 uint8_t is_tunnel = !!(buf->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK);
561 const uint64_t ol_flags_mask = RTE_MBUF_F_TX_TCP_SEG | RTE_MBUF_F_TX_L4_MASK |
562 RTE_MBUF_F_TX_IP_CKSUM | RTE_MBUF_F_TX_OUTER_IP_CKSUM;
565 * The index should have:
566 * bit[0] = RTE_MBUF_F_TX_TCP_SEG
567 * bit[2:3] = RTE_MBUF_F_TX_UDP_CKSUM, RTE_MBUF_F_TX_TCP_CKSUM
568 * bit[4] = RTE_MBUF_F_TX_IP_CKSUM
569 * bit[8] = RTE_MBUF_F_TX_OUTER_IP_CKSUM
572 idx = ((buf->ol_flags & ol_flags_mask) >> 50) | (!!is_tunnel << 9);
573 return mlx5_cksum_table[idx];
577 * Free the mbufs from the linear array of pointers.
580 * Pointer to Tx queue structure.
582 * Pointer to array of packets to be free.
584 * Number of packets to be freed.
586 * Configured Tx offloads mask. It is fully defined at
587 * compile time and may be used for optimization.
589 static __rte_always_inline void
590 mlx5_tx_free_mbuf(struct mlx5_txq_data *__rte_restrict txq,
591 struct rte_mbuf **__rte_restrict pkts,
593 unsigned int olx __rte_unused)
595 struct rte_mempool *pool = NULL;
596 struct rte_mbuf **p_free = NULL;
597 struct rte_mbuf *mbuf;
598 unsigned int n_free = 0;
601 * The implemented algorithm eliminates
602 * copying pointers to temporary array
603 * for rte_mempool_put_bulk() calls.
608 * Free mbufs directly to the pool in bulk
609 * if fast free offload is engaged
611 if (!MLX5_TXOFF_CONFIG(MULTI) && txq->fast_free) {
614 rte_mempool_put_bulk(pool, (void *)pkts, pkts_n);
620 * Decrement mbuf reference counter, detach
621 * indirect and external buffers if needed.
623 mbuf = rte_pktmbuf_prefree_seg(*pkts);
624 if (likely(mbuf != NULL)) {
625 MLX5_ASSERT(mbuf == *pkts);
626 if (likely(n_free != 0)) {
627 if (unlikely(pool != mbuf->pool))
628 /* From different pool. */
631 /* Start new scan array. */
638 if (unlikely(pkts_n == 0)) {
644 * This happens if mbuf is still referenced.
645 * We can't put it back to the pool, skip.
649 if (unlikely(n_free != 0))
650 /* There is some array to free.*/
652 if (unlikely(pkts_n == 0))
653 /* Last mbuf, nothing to free. */
659 * This loop is implemented to avoid multiple
660 * inlining of rte_mempool_put_bulk().
666 * Free the array of pre-freed mbufs
667 * belonging to the same memory pool.
669 rte_mempool_put_bulk(pool, (void *)p_free, n_free);
670 if (unlikely(mbuf != NULL)) {
671 /* There is the request to start new scan. */
676 if (likely(pkts_n != 0))
679 * This is the last mbuf to be freed.
680 * Do one more loop iteration to complete.
681 * This is rare case of the last unique mbuf.
686 if (likely(pkts_n == 0))
695 * No inline version to free buffers for optimal call
696 * on the tx_burst completion.
698 static __rte_noinline void
699 __mlx5_tx_free_mbuf(struct mlx5_txq_data *__rte_restrict txq,
700 struct rte_mbuf **__rte_restrict pkts,
702 unsigned int olx __rte_unused)
704 mlx5_tx_free_mbuf(txq, pkts, pkts_n, olx);
708 * Free the mbuf from the elts ring buffer till new tail.
711 * Pointer to Tx queue structure.
713 * Index in elts to free up to, becomes new elts tail.
715 * Configured Tx offloads mask. It is fully defined at
716 * compile time and may be used for optimization.
718 static __rte_always_inline void
719 mlx5_tx_free_elts(struct mlx5_txq_data *__rte_restrict txq,
721 unsigned int olx __rte_unused)
723 uint16_t n_elts = tail - txq->elts_tail;
726 MLX5_ASSERT(n_elts <= txq->elts_s);
728 * Implement a loop to support ring buffer wraparound
729 * with single inlining of mlx5_tx_free_mbuf().
734 part = txq->elts_s - (txq->elts_tail & txq->elts_m);
735 part = RTE_MIN(part, n_elts);
737 MLX5_ASSERT(part <= txq->elts_s);
738 mlx5_tx_free_mbuf(txq,
739 &txq->elts[txq->elts_tail & txq->elts_m],
741 txq->elts_tail += part;
747 * Store the mbuf being sent into elts ring buffer.
748 * On Tx completion these mbufs will be freed.
751 * Pointer to Tx queue structure.
753 * Pointer to array of packets to be stored.
755 * Number of packets to be stored.
757 * Configured Tx offloads mask. It is fully defined at
758 * compile time and may be used for optimization.
760 static __rte_always_inline void
761 mlx5_tx_copy_elts(struct mlx5_txq_data *__rte_restrict txq,
762 struct rte_mbuf **__rte_restrict pkts,
764 unsigned int olx __rte_unused)
767 struct rte_mbuf **elts = (struct rte_mbuf **)txq->elts;
771 part = txq->elts_s - (txq->elts_head & txq->elts_m);
773 MLX5_ASSERT(part <= txq->elts_s);
774 /* This code is a good candidate for vectorizing with SIMD. */
775 rte_memcpy((void *)(elts + (txq->elts_head & txq->elts_m)),
777 RTE_MIN(part, pkts_n) * sizeof(struct rte_mbuf *));
778 txq->elts_head += pkts_n;
779 if (unlikely(part < pkts_n))
780 /* The copy is wrapping around the elts array. */
781 rte_memcpy((void *)elts, (void *)(pkts + part),
782 (pkts_n - part) * sizeof(struct rte_mbuf *));
786 * Check if the completion request flag should be set in the last WQE.
787 * Both pushed mbufs and WQEs are monitored and the completion request
788 * flag is set if any of thresholds is reached.
791 * Pointer to TX queue structure.
793 * Pointer to burst routine local context.
795 * Configured Tx offloads mask. It is fully defined at
796 * compile time and may be used for optimization.
798 static __rte_always_inline void
799 mlx5_tx_request_completion(struct mlx5_txq_data *__rte_restrict txq,
800 struct mlx5_txq_local *__rte_restrict loc,
803 uint16_t head = txq->elts_head;
806 part = MLX5_TXOFF_CONFIG(INLINE) ?
807 0 : loc->pkts_sent - loc->pkts_copy;
809 if ((uint16_t)(head - txq->elts_comp) >= MLX5_TX_COMP_THRESH ||
810 (MLX5_TXOFF_CONFIG(INLINE) &&
811 (uint16_t)(txq->wqe_ci - txq->wqe_comp) >= txq->wqe_thres)) {
812 volatile struct mlx5_wqe *last = loc->wqe_last;
815 txq->elts_comp = head;
816 if (MLX5_TXOFF_CONFIG(INLINE))
817 txq->wqe_comp = txq->wqe_ci;
818 /* Request unconditional completion on last WQE. */
819 last->cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS <<
820 MLX5_COMP_MODE_OFFSET);
821 /* Save elts_head in dedicated free on completion queue. */
822 #ifdef RTE_LIBRTE_MLX5_DEBUG
823 txq->fcqs[txq->cq_pi++ & txq->cqe_m] = head |
824 (last->cseg.opcode >> 8) << 16;
826 txq->fcqs[txq->cq_pi++ & txq->cqe_m] = head;
828 /* A CQE slot must always be available. */
829 MLX5_ASSERT((txq->cq_pi - txq->cq_ci) <= txq->cqe_s);
834 * Build the Control Segment with specified opcode:
836 * - MLX5_OPCODE_ENHANCED_MPSW
840 * Pointer to TX queue structure.
842 * Pointer to burst routine local context.
844 * Pointer to WQE to fill with built Control Segment.
846 * Supposed length of WQE in segments.
848 * SQ WQE opcode to put into Control Segment.
850 * Configured Tx offloads mask. It is fully defined at
851 * compile time and may be used for optimization.
853 static __rte_always_inline void
854 mlx5_tx_cseg_init(struct mlx5_txq_data *__rte_restrict txq,
855 struct mlx5_txq_local *__rte_restrict loc __rte_unused,
856 struct mlx5_wqe *__rte_restrict wqe,
859 unsigned int olx __rte_unused)
861 struct mlx5_wqe_cseg *__rte_restrict cs = &wqe->cseg;
863 /* For legacy MPW replace the EMPW by TSO with modifier. */
864 if (MLX5_TXOFF_CONFIG(MPW) && opcode == MLX5_OPCODE_ENHANCED_MPSW)
865 opcode = MLX5_OPCODE_TSO | MLX5_OPC_MOD_MPW << 24;
866 cs->opcode = rte_cpu_to_be_32((txq->wqe_ci << 8) | opcode);
867 cs->sq_ds = rte_cpu_to_be_32(txq->qp_num_8s | ds);
868 cs->flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR <<
869 MLX5_COMP_MODE_OFFSET);
870 cs->misc = RTE_BE32(0);
874 * Build the Synchronize Queue Segment with specified completion index.
877 * Pointer to TX queue structure.
879 * Pointer to burst routine local context.
881 * Pointer to WQE to fill with built Control Segment.
883 * Completion index in Clock Queue to wait.
885 * Configured Tx offloads mask. It is fully defined at
886 * compile time and may be used for optimization.
888 static __rte_always_inline void
889 mlx5_tx_wseg_init(struct mlx5_txq_data *restrict txq,
890 struct mlx5_txq_local *restrict loc __rte_unused,
891 struct mlx5_wqe *restrict wqe,
893 unsigned int olx __rte_unused)
895 struct mlx5_wqe_qseg *qs;
897 qs = RTE_PTR_ADD(wqe, MLX5_WSEG_SIZE);
898 qs->max_index = rte_cpu_to_be_32(wci);
899 qs->qpn_cqn = rte_cpu_to_be_32(txq->sh->txpp.clock_queue.cq_obj.cq->id);
900 qs->reserved0 = RTE_BE32(0);
901 qs->reserved1 = RTE_BE32(0);
905 * Build the Ethernet Segment without inlined data.
906 * Supports Software Parser, Checksums and VLAN insertion Tx offload features.
909 * Pointer to TX queue structure.
911 * Pointer to burst routine local context.
913 * Pointer to WQE to fill with built Ethernet Segment.
915 * Configured Tx offloads mask. It is fully defined at
916 * compile time and may be used for optimization.
918 static __rte_always_inline void
919 mlx5_tx_eseg_none(struct mlx5_txq_data *__rte_restrict txq __rte_unused,
920 struct mlx5_txq_local *__rte_restrict loc,
921 struct mlx5_wqe *__rte_restrict wqe,
924 struct mlx5_wqe_eseg *__rte_restrict es = &wqe->eseg;
928 * Calculate and set check sum flags first, dword field
929 * in segment may be shared with Software Parser flags.
931 csum = MLX5_TXOFF_CONFIG(CSUM) ? txq_ol_cksum_to_cs(loc->mbuf) : 0;
932 es->flags = rte_cpu_to_le_32(csum);
934 * Calculate and set Software Parser offsets and flags.
935 * These flags a set for custom UDP and IP tunnel packets.
937 es->swp_offs = txq_mbuf_to_swp(loc, &es->swp_flags, olx);
938 /* Fill metadata field if needed. */
939 es->metadata = MLX5_TXOFF_CONFIG(METADATA) ?
940 loc->mbuf->ol_flags & RTE_MBUF_DYNFLAG_TX_METADATA ?
941 rte_cpu_to_be_32(*RTE_FLOW_DYNF_METADATA(loc->mbuf)) :
943 /* Engage VLAN tag insertion feature if requested. */
944 if (MLX5_TXOFF_CONFIG(VLAN) &&
945 loc->mbuf->ol_flags & RTE_MBUF_F_TX_VLAN) {
947 * We should get here only if device support
948 * this feature correctly.
950 MLX5_ASSERT(txq->vlan_en);
951 es->inline_hdr = rte_cpu_to_be_32(MLX5_ETH_WQE_VLAN_INSERT |
952 loc->mbuf->vlan_tci);
954 es->inline_hdr = RTE_BE32(0);
959 * Build the Ethernet Segment with minimal inlined data
960 * of MLX5_ESEG_MIN_INLINE_SIZE bytes length. This is
961 * used to fill the gap in single WQEBB WQEs.
962 * Supports Software Parser, Checksums and VLAN
963 * insertion Tx offload features.
966 * Pointer to TX queue structure.
968 * Pointer to burst routine local context.
970 * Pointer to WQE to fill with built Ethernet Segment.
972 * Length of VLAN tag insertion if any.
974 * Configured Tx offloads mask. It is fully defined at
975 * compile time and may be used for optimization.
977 static __rte_always_inline void
978 mlx5_tx_eseg_dmin(struct mlx5_txq_data *__rte_restrict txq __rte_unused,
979 struct mlx5_txq_local *__rte_restrict loc,
980 struct mlx5_wqe *__rte_restrict wqe,
984 struct mlx5_wqe_eseg *__rte_restrict es = &wqe->eseg;
986 uint8_t *psrc, *pdst;
989 * Calculate and set check sum flags first, dword field
990 * in segment may be shared with Software Parser flags.
992 csum = MLX5_TXOFF_CONFIG(CSUM) ? txq_ol_cksum_to_cs(loc->mbuf) : 0;
993 es->flags = rte_cpu_to_le_32(csum);
995 * Calculate and set Software Parser offsets and flags.
996 * These flags a set for custom UDP and IP tunnel packets.
998 es->swp_offs = txq_mbuf_to_swp(loc, &es->swp_flags, olx);
999 /* Fill metadata field if needed. */
1000 es->metadata = MLX5_TXOFF_CONFIG(METADATA) ?
1001 loc->mbuf->ol_flags & RTE_MBUF_DYNFLAG_TX_METADATA ?
1002 rte_cpu_to_be_32(*RTE_FLOW_DYNF_METADATA(loc->mbuf)) :
1004 psrc = rte_pktmbuf_mtod(loc->mbuf, uint8_t *);
1005 es->inline_hdr_sz = RTE_BE16(MLX5_ESEG_MIN_INLINE_SIZE);
1006 es->inline_data = *(unaligned_uint16_t *)psrc;
1007 psrc += sizeof(uint16_t);
1008 pdst = (uint8_t *)(es + 1);
1009 if (MLX5_TXOFF_CONFIG(VLAN) && vlan) {
1010 /* Implement VLAN tag insertion as part inline data. */
1011 memcpy(pdst, psrc, 2 * RTE_ETHER_ADDR_LEN - sizeof(uint16_t));
1012 pdst += 2 * RTE_ETHER_ADDR_LEN - sizeof(uint16_t);
1013 psrc += 2 * RTE_ETHER_ADDR_LEN - sizeof(uint16_t);
1014 /* Insert VLAN ethertype + VLAN tag. */
1015 *(unaligned_uint32_t *)pdst = rte_cpu_to_be_32
1016 ((RTE_ETHER_TYPE_VLAN << 16) |
1017 loc->mbuf->vlan_tci);
1018 pdst += sizeof(struct rte_vlan_hdr);
1019 /* Copy the rest two bytes from packet data. */
1020 MLX5_ASSERT(pdst == RTE_PTR_ALIGN(pdst, sizeof(uint16_t)));
1021 *(uint16_t *)pdst = *(unaligned_uint16_t *)psrc;
1023 /* Fill the gap in the title WQEBB with inline data. */
1024 rte_mov16(pdst, psrc);
1029 * Build the Ethernet Segment with entire packet data inlining. Checks the
1030 * boundary of WQEBB and ring buffer wrapping, supports Software Parser,
1031 * Checksums and VLAN insertion Tx offload features.
1034 * Pointer to TX queue structure.
1036 * Pointer to burst routine local context.
1038 * Pointer to WQE to fill with built Ethernet Segment.
1040 * Length of VLAN tag insertion if any.
1042 * Length of data to inline (VLAN included, if any).
1044 * TSO flag, set mss field from the packet.
1046 * Configured Tx offloads mask. It is fully defined at
1047 * compile time and may be used for optimization.
1050 * Pointer to the next Data Segment (aligned and wrapped around).
1052 static __rte_always_inline struct mlx5_wqe_dseg *
1053 mlx5_tx_eseg_data(struct mlx5_txq_data *__rte_restrict txq,
1054 struct mlx5_txq_local *__rte_restrict loc,
1055 struct mlx5_wqe *__rte_restrict wqe,
1061 struct mlx5_wqe_eseg *__rte_restrict es = &wqe->eseg;
1063 uint8_t *psrc, *pdst;
1067 * Calculate and set check sum flags first, dword field
1068 * in segment may be shared with Software Parser flags.
1070 csum = MLX5_TXOFF_CONFIG(CSUM) ? txq_ol_cksum_to_cs(loc->mbuf) : 0;
1073 csum |= loc->mbuf->tso_segsz;
1074 es->flags = rte_cpu_to_be_32(csum);
1076 es->flags = rte_cpu_to_le_32(csum);
1079 * Calculate and set Software Parser offsets and flags.
1080 * These flags a set for custom UDP and IP tunnel packets.
1082 es->swp_offs = txq_mbuf_to_swp(loc, &es->swp_flags, olx);
1083 /* Fill metadata field if needed. */
1084 es->metadata = MLX5_TXOFF_CONFIG(METADATA) ?
1085 loc->mbuf->ol_flags & RTE_MBUF_DYNFLAG_TX_METADATA ?
1086 rte_cpu_to_be_32(*RTE_FLOW_DYNF_METADATA(loc->mbuf)) :
1088 psrc = rte_pktmbuf_mtod(loc->mbuf, uint8_t *);
1089 es->inline_hdr_sz = rte_cpu_to_be_16(inlen);
1090 es->inline_data = *(unaligned_uint16_t *)psrc;
1091 psrc += sizeof(uint16_t);
1092 pdst = (uint8_t *)(es + 1);
1093 if (MLX5_TXOFF_CONFIG(VLAN) && vlan) {
1094 /* Implement VLAN tag insertion as part inline data. */
1095 memcpy(pdst, psrc, 2 * RTE_ETHER_ADDR_LEN - sizeof(uint16_t));
1096 pdst += 2 * RTE_ETHER_ADDR_LEN - sizeof(uint16_t);
1097 psrc += 2 * RTE_ETHER_ADDR_LEN - sizeof(uint16_t);
1098 /* Insert VLAN ethertype + VLAN tag. */
1099 *(unaligned_uint32_t *)pdst = rte_cpu_to_be_32
1100 ((RTE_ETHER_TYPE_VLAN << 16) |
1101 loc->mbuf->vlan_tci);
1102 pdst += sizeof(struct rte_vlan_hdr);
1103 /* Copy the rest two bytes from packet data. */
1104 MLX5_ASSERT(pdst == RTE_PTR_ALIGN(pdst, sizeof(uint16_t)));
1105 *(uint16_t *)pdst = *(unaligned_uint16_t *)psrc;
1106 psrc += sizeof(uint16_t);
1108 /* Fill the gap in the title WQEBB with inline data. */
1109 rte_mov16(pdst, psrc);
1110 psrc += sizeof(rte_v128u32_t);
1112 pdst = (uint8_t *)(es + 2);
1113 MLX5_ASSERT(inlen >= MLX5_ESEG_MIN_INLINE_SIZE);
1114 MLX5_ASSERT(pdst < (uint8_t *)txq->wqes_end);
1115 inlen -= MLX5_ESEG_MIN_INLINE_SIZE;
1117 MLX5_ASSERT(pdst == RTE_PTR_ALIGN(pdst, MLX5_WSEG_SIZE));
1118 return (struct mlx5_wqe_dseg *)pdst;
1121 * The WQEBB space availability is checked by caller.
1122 * Here we should be aware of WQE ring buffer wraparound only.
1124 part = (uint8_t *)txq->wqes_end - pdst;
1125 part = RTE_MIN(part, inlen);
1127 rte_memcpy(pdst, psrc, part);
1129 if (likely(!inlen)) {
1131 * If return value is not used by the caller
1132 * the code below will be optimized out.
1135 pdst = RTE_PTR_ALIGN(pdst, MLX5_WSEG_SIZE);
1136 if (unlikely(pdst >= (uint8_t *)txq->wqes_end))
1137 pdst = (uint8_t *)txq->wqes;
1138 return (struct mlx5_wqe_dseg *)pdst;
1140 pdst = (uint8_t *)txq->wqes;
1147 * Copy data from chain of mbuf to the specified linear buffer.
1148 * Checksums and VLAN insertion Tx offload features. If data
1149 * from some mbuf copied completely this mbuf is freed. Local
1150 * structure is used to keep the byte stream state.
1153 * Pointer to the destination linear buffer.
1155 * Pointer to burst routine local context.
1157 * Length of data to be copied.
1159 * Length of data to be copied ignoring no inline hint.
1161 * Configured Tx offloads mask. It is fully defined at
1162 * compile time and may be used for optimization.
1165 * Number of actual copied data bytes. This is always greater than or
1166 * equal to must parameter and might be lesser than len in no inline
1167 * hint flag is encountered.
1169 static __rte_always_inline unsigned int
1170 mlx5_tx_mseg_memcpy(uint8_t *pdst,
1171 struct mlx5_txq_local *__rte_restrict loc,
1174 unsigned int olx __rte_unused)
1176 struct rte_mbuf *mbuf;
1177 unsigned int part, dlen, copy = 0;
1181 MLX5_ASSERT(must <= len);
1183 /* Allow zero length packets, must check first. */
1184 dlen = rte_pktmbuf_data_len(loc->mbuf);
1185 if (dlen <= loc->mbuf_off) {
1186 /* Exhausted packet, just free. */
1188 loc->mbuf = mbuf->next;
1189 rte_pktmbuf_free_seg(mbuf);
1191 MLX5_ASSERT(loc->mbuf_nseg > 1);
1192 MLX5_ASSERT(loc->mbuf);
1194 if (loc->mbuf->ol_flags & RTE_MBUF_F_TX_DYNF_NOINLINE) {
1199 * We already copied the minimal
1200 * requested amount of data.
1205 if (diff <= rte_pktmbuf_data_len(loc->mbuf)) {
1207 * Copy only the minimal required
1208 * part of the data buffer.
1215 dlen -= loc->mbuf_off;
1216 psrc = rte_pktmbuf_mtod_offset(loc->mbuf, uint8_t *,
1218 part = RTE_MIN(len, dlen);
1219 rte_memcpy(pdst, psrc, part);
1221 loc->mbuf_off += part;
1224 if (loc->mbuf_off >= rte_pktmbuf_data_len(loc->mbuf)) {
1226 /* Exhausted packet, just free. */
1228 loc->mbuf = mbuf->next;
1229 rte_pktmbuf_free_seg(mbuf);
1231 MLX5_ASSERT(loc->mbuf_nseg >= 1);
1241 * Build the Ethernet Segment with inlined data from multi-segment packet.
1242 * Checks the boundary of WQEBB and ring buffer wrapping, supports Software
1243 * Parser, Checksums and VLAN insertion Tx offload features.
1246 * Pointer to TX queue structure.
1248 * Pointer to burst routine local context.
1250 * Pointer to WQE to fill with built Ethernet Segment.
1252 * Length of VLAN tag insertion if any.
1254 * Length of data to inline (VLAN included, if any).
1256 * TSO flag, set mss field from the packet.
1258 * Configured Tx offloads mask. It is fully defined at
1259 * compile time and may be used for optimization.
1262 * Pointer to the next Data Segment (aligned and possible NOT wrapped
1263 * around - caller should do wrapping check on its own).
1265 static __rte_always_inline struct mlx5_wqe_dseg *
1266 mlx5_tx_eseg_mdat(struct mlx5_txq_data *__rte_restrict txq,
1267 struct mlx5_txq_local *__rte_restrict loc,
1268 struct mlx5_wqe *__rte_restrict wqe,
1274 struct mlx5_wqe_eseg *__rte_restrict es = &wqe->eseg;
1277 unsigned int part, tlen = 0;
1280 * Calculate and set check sum flags first, uint32_t field
1281 * in segment may be shared with Software Parser flags.
1283 csum = MLX5_TXOFF_CONFIG(CSUM) ? txq_ol_cksum_to_cs(loc->mbuf) : 0;
1286 csum |= loc->mbuf->tso_segsz;
1287 es->flags = rte_cpu_to_be_32(csum);
1289 es->flags = rte_cpu_to_le_32(csum);
1292 * Calculate and set Software Parser offsets and flags.
1293 * These flags a set for custom UDP and IP tunnel packets.
1295 es->swp_offs = txq_mbuf_to_swp(loc, &es->swp_flags, olx);
1296 /* Fill metadata field if needed. */
1297 es->metadata = MLX5_TXOFF_CONFIG(METADATA) ?
1298 loc->mbuf->ol_flags & RTE_MBUF_DYNFLAG_TX_METADATA ?
1299 rte_cpu_to_be_32(*RTE_FLOW_DYNF_METADATA(loc->mbuf)) :
1301 MLX5_ASSERT(inlen >= MLX5_ESEG_MIN_INLINE_SIZE);
1302 pdst = (uint8_t *)&es->inline_data;
1303 if (MLX5_TXOFF_CONFIG(VLAN) && vlan) {
1304 /* Implement VLAN tag insertion as part inline data. */
1305 mlx5_tx_mseg_memcpy(pdst, loc,
1306 2 * RTE_ETHER_ADDR_LEN,
1307 2 * RTE_ETHER_ADDR_LEN, olx);
1308 pdst += 2 * RTE_ETHER_ADDR_LEN;
1309 *(unaligned_uint32_t *)pdst = rte_cpu_to_be_32
1310 ((RTE_ETHER_TYPE_VLAN << 16) |
1311 loc->mbuf->vlan_tci);
1312 pdst += sizeof(struct rte_vlan_hdr);
1313 tlen += 2 * RTE_ETHER_ADDR_LEN + sizeof(struct rte_vlan_hdr);
1315 MLX5_ASSERT(pdst < (uint8_t *)txq->wqes_end);
1317 * The WQEBB space availability is checked by caller.
1318 * Here we should be aware of WQE ring buffer wraparound only.
1320 part = (uint8_t *)txq->wqes_end - pdst;
1321 part = RTE_MIN(part, inlen - tlen);
1327 * Copying may be interrupted inside the routine
1328 * if run into no inline hint flag.
1330 copy = tso ? inlen : txq->inlen_mode;
1331 copy = tlen >= copy ? 0 : (copy - tlen);
1332 copy = mlx5_tx_mseg_memcpy(pdst, loc, part, copy, olx);
1334 if (likely(inlen <= tlen) || copy < part) {
1335 es->inline_hdr_sz = rte_cpu_to_be_16(tlen);
1337 pdst = RTE_PTR_ALIGN(pdst, MLX5_WSEG_SIZE);
1338 return (struct mlx5_wqe_dseg *)pdst;
1340 pdst = (uint8_t *)txq->wqes;
1341 part = inlen - tlen;
1346 * Build the Data Segment of pointer type.
1349 * Pointer to TX queue structure.
1351 * Pointer to burst routine local context.
1353 * Pointer to WQE to fill with built Data Segment.
1355 * Data buffer to point.
1357 * Data buffer length.
1359 * Configured Tx offloads mask. It is fully defined at
1360 * compile time and may be used for optimization.
1362 static __rte_always_inline void
1363 mlx5_tx_dseg_ptr(struct mlx5_txq_data *__rte_restrict txq,
1364 struct mlx5_txq_local *__rte_restrict loc,
1365 struct mlx5_wqe_dseg *__rte_restrict dseg,
1368 unsigned int olx __rte_unused)
1372 dseg->bcount = rte_cpu_to_be_32(len);
1373 dseg->lkey = mlx5_tx_mb2mr(txq, loc->mbuf);
1374 dseg->pbuf = rte_cpu_to_be_64((uintptr_t)buf);
1378 * Build the Data Segment of pointer type or inline if data length is less than
1379 * buffer in minimal Data Segment size.
1382 * Pointer to TX queue structure.
1384 * Pointer to burst routine local context.
1386 * Pointer to WQE to fill with built Data Segment.
1388 * Data buffer to point.
1390 * Data buffer length.
1392 * Configured Tx offloads mask. It is fully defined at
1393 * compile time and may be used for optimization.
1395 static __rte_always_inline void
1396 mlx5_tx_dseg_iptr(struct mlx5_txq_data *__rte_restrict txq,
1397 struct mlx5_txq_local *__rte_restrict loc,
1398 struct mlx5_wqe_dseg *__rte_restrict dseg,
1401 unsigned int olx __rte_unused)
1407 if (len > MLX5_DSEG_MIN_INLINE_SIZE) {
1408 dseg->bcount = rte_cpu_to_be_32(len);
1409 dseg->lkey = mlx5_tx_mb2mr(txq, loc->mbuf);
1410 dseg->pbuf = rte_cpu_to_be_64((uintptr_t)buf);
1414 dseg->bcount = rte_cpu_to_be_32(len | MLX5_ETH_WQE_DATA_INLINE);
1415 /* Unrolled implementation of generic rte_memcpy. */
1416 dst = (uintptr_t)&dseg->inline_data[0];
1417 src = (uintptr_t)buf;
1419 #ifdef RTE_ARCH_STRICT_ALIGN
1420 MLX5_ASSERT(dst == RTE_PTR_ALIGN(dst, sizeof(uint32_t)));
1421 *(uint32_t *)dst = *(unaligned_uint32_t *)src;
1422 dst += sizeof(uint32_t);
1423 src += sizeof(uint32_t);
1424 *(uint32_t *)dst = *(unaligned_uint32_t *)src;
1425 dst += sizeof(uint32_t);
1426 src += sizeof(uint32_t);
1428 *(uint64_t *)dst = *(unaligned_uint64_t *)src;
1429 dst += sizeof(uint64_t);
1430 src += sizeof(uint64_t);
1434 *(uint32_t *)dst = *(unaligned_uint32_t *)src;
1435 dst += sizeof(uint32_t);
1436 src += sizeof(uint32_t);
1439 *(uint16_t *)dst = *(unaligned_uint16_t *)src;
1440 dst += sizeof(uint16_t);
1441 src += sizeof(uint16_t);
1444 *(uint8_t *)dst = *(uint8_t *)src;
1448 * Build the Data Segment of inlined data from single
1449 * segment packet, no VLAN insertion.
1452 * Pointer to TX queue structure.
1454 * Pointer to burst routine local context.
1456 * Pointer to WQE to fill with built Data Segment.
1458 * Data buffer to point.
1460 * Data buffer length.
1462 * Configured Tx offloads mask. It is fully defined at
1463 * compile time and may be used for optimization.
1466 * Pointer to the next Data Segment after inlined data.
1467 * Ring buffer wraparound check is needed. We do not do it here because it
1468 * may not be needed for the last packet in the eMPW session.
1470 static __rte_always_inline struct mlx5_wqe_dseg *
1471 mlx5_tx_dseg_empw(struct mlx5_txq_data *__rte_restrict txq,
1472 struct mlx5_txq_local *__rte_restrict loc __rte_unused,
1473 struct mlx5_wqe_dseg *__rte_restrict dseg,
1476 unsigned int olx __rte_unused)
1481 if (!MLX5_TXOFF_CONFIG(MPW)) {
1482 /* Store the descriptor byte counter for eMPW sessions. */
1483 dseg->bcount = rte_cpu_to_be_32(len | MLX5_ETH_WQE_DATA_INLINE);
1484 pdst = &dseg->inline_data[0];
1486 /* The entire legacy MPW session counter is stored on close. */
1487 pdst = (uint8_t *)dseg;
1490 * The WQEBB space availability is checked by caller.
1491 * Here we should be aware of WQE ring buffer wraparound only.
1493 part = (uint8_t *)txq->wqes_end - pdst;
1494 part = RTE_MIN(part, len);
1496 rte_memcpy(pdst, buf, part);
1500 if (!MLX5_TXOFF_CONFIG(MPW))
1501 pdst = RTE_PTR_ALIGN(pdst, MLX5_WSEG_SIZE);
1502 /* Note: no final wraparound check here. */
1503 return (struct mlx5_wqe_dseg *)pdst;
1505 pdst = (uint8_t *)txq->wqes;
1512 * Build the Data Segment of inlined data from single
1513 * segment packet with VLAN insertion.
1516 * Pointer to TX queue structure.
1518 * Pointer to burst routine local context.
1520 * Pointer to the dseg fill with built Data Segment.
1522 * Data buffer to point.
1524 * Data buffer length.
1526 * Configured Tx offloads mask. It is fully defined at
1527 * compile time and may be used for optimization.
1530 * Pointer to the next Data Segment after inlined data.
1531 * Ring buffer wraparound check is needed.
1533 static __rte_always_inline struct mlx5_wqe_dseg *
1534 mlx5_tx_dseg_vlan(struct mlx5_txq_data *__rte_restrict txq,
1535 struct mlx5_txq_local *__rte_restrict loc __rte_unused,
1536 struct mlx5_wqe_dseg *__rte_restrict dseg,
1539 unsigned int olx __rte_unused)
1545 MLX5_ASSERT(len > MLX5_ESEG_MIN_INLINE_SIZE);
1546 if (!MLX5_TXOFF_CONFIG(MPW)) {
1547 /* Store the descriptor byte counter for eMPW sessions. */
1548 dseg->bcount = rte_cpu_to_be_32
1549 ((len + sizeof(struct rte_vlan_hdr)) |
1550 MLX5_ETH_WQE_DATA_INLINE);
1551 pdst = &dseg->inline_data[0];
1553 /* The entire legacy MPW session counter is stored on close. */
1554 pdst = (uint8_t *)dseg;
1556 memcpy(pdst, buf, MLX5_DSEG_MIN_INLINE_SIZE);
1557 buf += MLX5_DSEG_MIN_INLINE_SIZE;
1558 pdst += MLX5_DSEG_MIN_INLINE_SIZE;
1559 len -= MLX5_DSEG_MIN_INLINE_SIZE;
1560 /* Insert VLAN ethertype + VLAN tag. Pointer is aligned. */
1561 MLX5_ASSERT(pdst == RTE_PTR_ALIGN(pdst, MLX5_WSEG_SIZE));
1562 if (unlikely(pdst >= (uint8_t *)txq->wqes_end))
1563 pdst = (uint8_t *)txq->wqes;
1564 *(uint32_t *)pdst = rte_cpu_to_be_32((RTE_ETHER_TYPE_VLAN << 16) |
1565 loc->mbuf->vlan_tci);
1566 pdst += sizeof(struct rte_vlan_hdr);
1568 * The WQEBB space availability is checked by caller.
1569 * Here we should be aware of WQE ring buffer wraparound only.
1571 part = (uint8_t *)txq->wqes_end - pdst;
1572 part = RTE_MIN(part, len);
1574 rte_memcpy(pdst, buf, part);
1578 if (!MLX5_TXOFF_CONFIG(MPW))
1579 pdst = RTE_PTR_ALIGN(pdst, MLX5_WSEG_SIZE);
1580 /* Note: no final wraparound check here. */
1581 return (struct mlx5_wqe_dseg *)pdst;
1583 pdst = (uint8_t *)txq->wqes;
1590 * Build the Ethernet Segment with optionally inlined data with
1591 * VLAN insertion and following Data Segments (if any) from
1592 * multi-segment packet. Used by ordinary send and TSO.
1595 * Pointer to TX queue structure.
1597 * Pointer to burst routine local context.
1599 * Pointer to WQE to fill with built Ethernet/Data Segments.
1601 * Length of VLAN header to insert, 0 means no VLAN insertion.
1603 * Data length to inline. For TSO this parameter specifies exact value,
1604 * for ordinary send routine can be aligned by caller to provide better WQE
1605 * space saving and data buffer start address alignment.
1606 * This length includes VLAN header being inserted.
1608 * Zero means ordinary send, inlined data can be extended,
1609 * otherwise this is TSO, inlined data length is fixed.
1611 * Configured Tx offloads mask. It is fully defined at
1612 * compile time and may be used for optimization.
1615 * Actual size of built WQE in segments.
1617 static __rte_always_inline unsigned int
1618 mlx5_tx_mseg_build(struct mlx5_txq_data *__rte_restrict txq,
1619 struct mlx5_txq_local *__rte_restrict loc,
1620 struct mlx5_wqe *__rte_restrict wqe,
1624 unsigned int olx __rte_unused)
1626 struct mlx5_wqe_dseg *__rte_restrict dseg;
1629 MLX5_ASSERT((rte_pktmbuf_pkt_len(loc->mbuf) + vlan) >= inlen);
1630 loc->mbuf_nseg = NB_SEGS(loc->mbuf);
1633 dseg = mlx5_tx_eseg_mdat(txq, loc, wqe, vlan, inlen, tso, olx);
1634 if (!loc->mbuf_nseg)
1637 * There are still some mbuf remaining, not inlined.
1638 * The first mbuf may be partially inlined and we
1639 * must process the possible non-zero data offset.
1641 if (loc->mbuf_off) {
1646 * Exhausted packets must be dropped before.
1647 * Non-zero offset means there are some data
1648 * remained in the packet.
1650 MLX5_ASSERT(loc->mbuf_off < rte_pktmbuf_data_len(loc->mbuf));
1651 MLX5_ASSERT(rte_pktmbuf_data_len(loc->mbuf));
1652 dptr = rte_pktmbuf_mtod_offset(loc->mbuf, uint8_t *,
1654 dlen = rte_pktmbuf_data_len(loc->mbuf) - loc->mbuf_off;
1656 * Build the pointer/minimal Data Segment.
1657 * Do ring buffer wrapping check in advance.
1659 if ((uintptr_t)dseg >= (uintptr_t)txq->wqes_end)
1660 dseg = (struct mlx5_wqe_dseg *)txq->wqes;
1661 mlx5_tx_dseg_iptr(txq, loc, dseg, dptr, dlen, olx);
1662 /* Store the mbuf to be freed on completion. */
1663 MLX5_ASSERT(loc->elts_free);
1664 txq->elts[txq->elts_head++ & txq->elts_m] = loc->mbuf;
1667 if (--loc->mbuf_nseg == 0)
1669 loc->mbuf = loc->mbuf->next;
1673 if (unlikely(!rte_pktmbuf_data_len(loc->mbuf))) {
1674 struct rte_mbuf *mbuf;
1676 /* Zero length segment found, just skip. */
1678 loc->mbuf = loc->mbuf->next;
1679 rte_pktmbuf_free_seg(mbuf);
1680 if (--loc->mbuf_nseg == 0)
1683 if ((uintptr_t)dseg >= (uintptr_t)txq->wqes_end)
1684 dseg = (struct mlx5_wqe_dseg *)txq->wqes;
1687 rte_pktmbuf_mtod(loc->mbuf, uint8_t *),
1688 rte_pktmbuf_data_len(loc->mbuf), olx);
1689 MLX5_ASSERT(loc->elts_free);
1690 txq->elts[txq->elts_head++ & txq->elts_m] = loc->mbuf;
1693 if (--loc->mbuf_nseg == 0)
1695 loc->mbuf = loc->mbuf->next;
1700 /* Calculate actual segments used from the dseg pointer. */
1701 if ((uintptr_t)wqe < (uintptr_t)dseg)
1702 ds = ((uintptr_t)dseg - (uintptr_t)wqe) / MLX5_WSEG_SIZE;
1704 ds = (((uintptr_t)dseg - (uintptr_t)wqe) +
1705 txq->wqe_s * MLX5_WQE_SIZE) / MLX5_WSEG_SIZE;
1710 * The routine checks timestamp flag in the current packet,
1711 * and push WAIT WQE into the queue if scheduling is required.
1714 * Pointer to TX queue structure.
1716 * Pointer to burst routine local context.
1718 * Configured Tx offloads mask. It is fully defined at
1719 * compile time and may be used for optimization.
1722 * MLX5_TXCMP_CODE_EXIT - sending is done or impossible.
1723 * MLX5_TXCMP_CODE_SINGLE - continue processing with the packet.
1724 * MLX5_TXCMP_CODE_MULTI - the WAIT inserted, continue processing.
1725 * Local context variables partially updated.
1727 static __rte_always_inline enum mlx5_txcmp_code
1728 mlx5_tx_schedule_send(struct mlx5_txq_data *restrict txq,
1729 struct mlx5_txq_local *restrict loc,
1732 if (MLX5_TXOFF_CONFIG(TXPP) &&
1733 loc->mbuf->ol_flags & txq->ts_mask) {
1734 struct mlx5_wqe *wqe;
1739 * Estimate the required space quickly and roughly.
1740 * We would like to ensure the packet can be pushed
1741 * to the queue and we won't get the orphan WAIT WQE.
1743 if (loc->wqe_free <= MLX5_WQE_SIZE_MAX / MLX5_WQE_SIZE ||
1744 loc->elts_free < NB_SEGS(loc->mbuf))
1745 return MLX5_TXCMP_CODE_EXIT;
1746 /* Convert the timestamp into completion to wait. */
1747 ts = *RTE_MBUF_DYNFIELD(loc->mbuf, txq->ts_offset, uint64_t *);
1748 wci = mlx5_txpp_convert_tx_ts(txq->sh, ts);
1749 if (unlikely(wci < 0))
1750 return MLX5_TXCMP_CODE_SINGLE;
1751 /* Build the WAIT WQE with specified completion. */
1752 wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m);
1753 mlx5_tx_cseg_init(txq, loc, wqe, 2, MLX5_OPCODE_WAIT, olx);
1754 mlx5_tx_wseg_init(txq, loc, wqe, wci, olx);
1757 return MLX5_TXCMP_CODE_MULTI;
1759 return MLX5_TXCMP_CODE_SINGLE;
1763 * Tx one packet function for multi-segment TSO. Supports all
1764 * types of Tx offloads, uses MLX5_OPCODE_TSO to build WQEs,
1765 * sends one packet per WQE.
1767 * This routine is responsible for storing processed mbuf
1768 * into elts ring buffer and update elts_head.
1771 * Pointer to TX queue structure.
1773 * Pointer to burst routine local context.
1775 * Configured Tx offloads mask. It is fully defined at
1776 * compile time and may be used for optimization.
1779 * MLX5_TXCMP_CODE_EXIT - sending is done or impossible.
1780 * MLX5_TXCMP_CODE_ERROR - some unrecoverable error occurred.
1781 * Local context variables partially updated.
1783 static __rte_always_inline enum mlx5_txcmp_code
1784 mlx5_tx_packet_multi_tso(struct mlx5_txq_data *__rte_restrict txq,
1785 struct mlx5_txq_local *__rte_restrict loc,
1788 struct mlx5_wqe *__rte_restrict wqe;
1789 unsigned int ds, dlen, inlen, ntcp, vlan = 0;
1791 if (MLX5_TXOFF_CONFIG(TXPP)) {
1792 enum mlx5_txcmp_code wret;
1794 /* Generate WAIT for scheduling if requested. */
1795 wret = mlx5_tx_schedule_send(txq, loc, olx);
1796 if (wret == MLX5_TXCMP_CODE_EXIT)
1797 return MLX5_TXCMP_CODE_EXIT;
1798 if (wret == MLX5_TXCMP_CODE_ERROR)
1799 return MLX5_TXCMP_CODE_ERROR;
1802 * Calculate data length to be inlined to estimate
1803 * the required space in WQE ring buffer.
1805 dlen = rte_pktmbuf_pkt_len(loc->mbuf);
1806 if (MLX5_TXOFF_CONFIG(VLAN) && loc->mbuf->ol_flags & RTE_MBUF_F_TX_VLAN)
1807 vlan = sizeof(struct rte_vlan_hdr);
1808 inlen = loc->mbuf->l2_len + vlan +
1809 loc->mbuf->l3_len + loc->mbuf->l4_len;
1810 if (unlikely((!inlen || !loc->mbuf->tso_segsz)))
1811 return MLX5_TXCMP_CODE_ERROR;
1812 if (loc->mbuf->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK)
1813 inlen += loc->mbuf->outer_l2_len + loc->mbuf->outer_l3_len;
1814 /* Packet must contain all TSO headers. */
1815 if (unlikely(inlen > MLX5_MAX_TSO_HEADER ||
1816 inlen <= MLX5_ESEG_MIN_INLINE_SIZE ||
1817 inlen > (dlen + vlan)))
1818 return MLX5_TXCMP_CODE_ERROR;
1819 MLX5_ASSERT(inlen >= txq->inlen_mode);
1821 * Check whether there are enough free WQEBBs:
1823 * - Ethernet Segment
1824 * - First Segment of inlined Ethernet data
1825 * - ... data continued ...
1826 * - Data Segments of pointer/min inline type
1828 ds = NB_SEGS(loc->mbuf) + 2 + (inlen -
1829 MLX5_ESEG_MIN_INLINE_SIZE +
1831 MLX5_WSEG_SIZE - 1) / MLX5_WSEG_SIZE;
1832 if (unlikely(loc->wqe_free < ((ds + 3) / 4)))
1833 return MLX5_TXCMP_CODE_EXIT;
1834 /* Check for maximal WQE size. */
1835 if (unlikely((MLX5_WQE_SIZE_MAX / MLX5_WSEG_SIZE) < ((ds + 3) / 4)))
1836 return MLX5_TXCMP_CODE_ERROR;
1837 #ifdef MLX5_PMD_SOFT_COUNTERS
1838 /* Update sent data bytes/packets counters. */
1839 ntcp = (dlen - (inlen - vlan) + loc->mbuf->tso_segsz - 1) /
1840 loc->mbuf->tso_segsz;
1842 * One will be added for mbuf itself at the end of the mlx5_tx_burst
1843 * from loc->pkts_sent field.
1846 txq->stats.opackets += ntcp;
1847 txq->stats.obytes += dlen + vlan + ntcp * inlen;
1849 wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m);
1850 loc->wqe_last = wqe;
1851 mlx5_tx_cseg_init(txq, loc, wqe, 0, MLX5_OPCODE_TSO, olx);
1852 ds = mlx5_tx_mseg_build(txq, loc, wqe, vlan, inlen, 1, olx);
1853 wqe->cseg.sq_ds = rte_cpu_to_be_32(txq->qp_num_8s | ds);
1854 txq->wqe_ci += (ds + 3) / 4;
1855 loc->wqe_free -= (ds + 3) / 4;
1856 return MLX5_TXCMP_CODE_MULTI;
1860 * Tx one packet function for multi-segment SEND. Supports all types of Tx
1861 * offloads, uses MLX5_OPCODE_SEND to build WQEs, sends one packet per WQE,
1862 * without any data inlining in Ethernet Segment.
1864 * This routine is responsible for storing processed mbuf
1865 * into elts ring buffer and update elts_head.
1868 * Pointer to TX queue structure.
1870 * Pointer to burst routine local context.
1872 * Configured Tx offloads mask. It is fully defined at
1873 * compile time and may be used for optimization.
1876 * MLX5_TXCMP_CODE_EXIT - sending is done or impossible.
1877 * MLX5_TXCMP_CODE_ERROR - some unrecoverable error occurred.
1878 * Local context variables partially updated.
1880 static __rte_always_inline enum mlx5_txcmp_code
1881 mlx5_tx_packet_multi_send(struct mlx5_txq_data *__rte_restrict txq,
1882 struct mlx5_txq_local *__rte_restrict loc,
1885 struct mlx5_wqe_dseg *__rte_restrict dseg;
1886 struct mlx5_wqe *__rte_restrict wqe;
1887 unsigned int ds, nseg;
1889 MLX5_ASSERT(NB_SEGS(loc->mbuf) > 1);
1890 if (MLX5_TXOFF_CONFIG(TXPP)) {
1891 enum mlx5_txcmp_code wret;
1893 /* Generate WAIT for scheduling if requested. */
1894 wret = mlx5_tx_schedule_send(txq, loc, olx);
1895 if (wret == MLX5_TXCMP_CODE_EXIT)
1896 return MLX5_TXCMP_CODE_EXIT;
1897 if (wret == MLX5_TXCMP_CODE_ERROR)
1898 return MLX5_TXCMP_CODE_ERROR;
1901 * No inline at all, it means the CPU cycles saving is prioritized at
1902 * configuration, we should not copy any packet data to WQE.
1904 nseg = NB_SEGS(loc->mbuf);
1906 if (unlikely(loc->wqe_free < ((ds + 3) / 4)))
1907 return MLX5_TXCMP_CODE_EXIT;
1908 /* Check for maximal WQE size. */
1909 if (unlikely((MLX5_WQE_SIZE_MAX / MLX5_WSEG_SIZE) < ((ds + 3) / 4)))
1910 return MLX5_TXCMP_CODE_ERROR;
1912 * Some Tx offloads may cause an error if packet is not long enough,
1913 * check against assumed minimal length.
1915 if (rte_pktmbuf_pkt_len(loc->mbuf) <= MLX5_ESEG_MIN_INLINE_SIZE)
1916 return MLX5_TXCMP_CODE_ERROR;
1917 #ifdef MLX5_PMD_SOFT_COUNTERS
1918 /* Update sent data bytes counter. */
1919 txq->stats.obytes += rte_pktmbuf_pkt_len(loc->mbuf);
1920 if (MLX5_TXOFF_CONFIG(VLAN) &&
1921 loc->mbuf->ol_flags & RTE_MBUF_F_TX_VLAN)
1922 txq->stats.obytes += sizeof(struct rte_vlan_hdr);
1925 * SEND WQE, one WQEBB:
1926 * - Control Segment, SEND opcode
1927 * - Ethernet Segment, optional VLAN, no inline
1928 * - Data Segments, pointer only type
1930 wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m);
1931 loc->wqe_last = wqe;
1932 mlx5_tx_cseg_init(txq, loc, wqe, ds, MLX5_OPCODE_SEND, olx);
1933 mlx5_tx_eseg_none(txq, loc, wqe, olx);
1934 dseg = &wqe->dseg[0];
1936 if (unlikely(!rte_pktmbuf_data_len(loc->mbuf))) {
1937 struct rte_mbuf *mbuf;
1940 * Zero length segment found, have to correct total
1941 * size of WQE in segments.
1942 * It is supposed to be rare occasion, so in normal
1943 * case (no zero length segments) we avoid extra
1944 * writing to the Control Segment.
1947 wqe->cseg.sq_ds -= RTE_BE32(1);
1949 loc->mbuf = mbuf->next;
1950 rte_pktmbuf_free_seg(mbuf);
1956 rte_pktmbuf_mtod(loc->mbuf, uint8_t *),
1957 rte_pktmbuf_data_len(loc->mbuf), olx);
1958 txq->elts[txq->elts_head++ & txq->elts_m] = loc->mbuf;
1963 if ((uintptr_t)dseg >= (uintptr_t)txq->wqes_end)
1964 dseg = (struct mlx5_wqe_dseg *)txq->wqes;
1965 loc->mbuf = loc->mbuf->next;
1968 txq->wqe_ci += (ds + 3) / 4;
1969 loc->wqe_free -= (ds + 3) / 4;
1970 return MLX5_TXCMP_CODE_MULTI;
1974 * Tx one packet function for multi-segment SEND. Supports all
1975 * types of Tx offloads, uses MLX5_OPCODE_SEND to build WQEs,
1976 * sends one packet per WQE, with data inlining in
1977 * Ethernet Segment and minimal Data Segments.
1979 * This routine is responsible for storing processed mbuf
1980 * into elts ring buffer and update elts_head.
1983 * Pointer to TX queue structure.
1985 * Pointer to burst routine local context.
1987 * Configured Tx offloads mask. It is fully defined at
1988 * compile time and may be used for optimization.
1991 * MLX5_TXCMP_CODE_EXIT - sending is done or impossible.
1992 * MLX5_TXCMP_CODE_ERROR - some unrecoverable error occurred.
1993 * Local context variables partially updated.
1995 static __rte_always_inline enum mlx5_txcmp_code
1996 mlx5_tx_packet_multi_inline(struct mlx5_txq_data *__rte_restrict txq,
1997 struct mlx5_txq_local *__rte_restrict loc,
2000 struct mlx5_wqe *__rte_restrict wqe;
2001 unsigned int ds, inlen, dlen, vlan = 0;
2003 MLX5_ASSERT(MLX5_TXOFF_CONFIG(INLINE));
2004 MLX5_ASSERT(NB_SEGS(loc->mbuf) > 1);
2005 if (MLX5_TXOFF_CONFIG(TXPP)) {
2006 enum mlx5_txcmp_code wret;
2008 /* Generate WAIT for scheduling if requested. */
2009 wret = mlx5_tx_schedule_send(txq, loc, olx);
2010 if (wret == MLX5_TXCMP_CODE_EXIT)
2011 return MLX5_TXCMP_CODE_EXIT;
2012 if (wret == MLX5_TXCMP_CODE_ERROR)
2013 return MLX5_TXCMP_CODE_ERROR;
2016 * First calculate data length to be inlined
2017 * to estimate the required space for WQE.
2019 dlen = rte_pktmbuf_pkt_len(loc->mbuf);
2020 if (MLX5_TXOFF_CONFIG(VLAN) && loc->mbuf->ol_flags & RTE_MBUF_F_TX_VLAN)
2021 vlan = sizeof(struct rte_vlan_hdr);
2022 inlen = dlen + vlan;
2023 /* Check against minimal length. */
2024 if (inlen <= MLX5_ESEG_MIN_INLINE_SIZE)
2025 return MLX5_TXCMP_CODE_ERROR;
2026 MLX5_ASSERT(txq->inlen_send >= MLX5_ESEG_MIN_INLINE_SIZE);
2027 if (inlen > txq->inlen_send ||
2028 loc->mbuf->ol_flags & RTE_MBUF_F_TX_DYNF_NOINLINE) {
2029 struct rte_mbuf *mbuf;
2034 nxlen = rte_pktmbuf_data_len(mbuf);
2036 * Packet length exceeds the allowed inline data length,
2037 * check whether the minimal inlining is required.
2039 if (txq->inlen_mode) {
2040 MLX5_ASSERT(txq->inlen_mode >=
2041 MLX5_ESEG_MIN_INLINE_SIZE);
2042 MLX5_ASSERT(txq->inlen_mode <= txq->inlen_send);
2043 inlen = txq->inlen_mode;
2044 } else if (vlan && !txq->vlan_en) {
2046 * VLAN insertion is requested and hardware does not
2047 * support the offload, will do with software inline.
2049 inlen = MLX5_ESEG_MIN_INLINE_SIZE;
2050 } else if (mbuf->ol_flags & RTE_MBUF_F_TX_DYNF_NOINLINE ||
2051 nxlen > txq->inlen_send) {
2052 return mlx5_tx_packet_multi_send(txq, loc, olx);
2057 * Now we know the minimal amount of data is requested
2058 * to inline. Check whether we should inline the buffers
2059 * from the chain beginning to eliminate some mbufs.
2061 if (unlikely(nxlen <= txq->inlen_send)) {
2062 /* We can inline first mbuf at least. */
2063 if (nxlen < inlen) {
2066 /* Scan mbufs till inlen filled. */
2071 nxlen = rte_pktmbuf_data_len(mbuf);
2073 } while (unlikely(nxlen < inlen));
2074 if (unlikely(nxlen > txq->inlen_send)) {
2075 /* We cannot inline entire mbuf. */
2076 smlen = inlen - smlen;
2077 start = rte_pktmbuf_mtod_offset
2078 (mbuf, uintptr_t, smlen);
2086 /* There should be not end of packet. */
2088 nxlen = inlen + rte_pktmbuf_data_len(mbuf);
2089 } while (unlikely(nxlen < txq->inlen_send));
2091 start = rte_pktmbuf_mtod(mbuf, uintptr_t);
2093 * Check whether we can do inline to align start
2094 * address of data buffer to cacheline.
2097 start = (~start + 1) & (RTE_CACHE_LINE_SIZE - 1);
2098 if (unlikely(start)) {
2100 if (start <= txq->inlen_send)
2105 * Check whether there are enough free WQEBBs:
2107 * - Ethernet Segment
2108 * - First Segment of inlined Ethernet data
2109 * - ... data continued ...
2110 * - Data Segments of pointer/min inline type
2112 * Estimate the number of Data Segments conservatively,
2113 * supposing no any mbufs is being freed during inlining.
2115 MLX5_ASSERT(inlen <= txq->inlen_send);
2116 ds = NB_SEGS(loc->mbuf) + 2 + (inlen -
2117 MLX5_ESEG_MIN_INLINE_SIZE +
2119 MLX5_WSEG_SIZE - 1) / MLX5_WSEG_SIZE;
2120 if (unlikely(loc->wqe_free < ((ds + 3) / 4)))
2121 return MLX5_TXCMP_CODE_EXIT;
2122 /* Check for maximal WQE size. */
2123 if (unlikely((MLX5_WQE_SIZE_MAX / MLX5_WSEG_SIZE) < ((ds + 3) / 4)))
2124 return MLX5_TXCMP_CODE_ERROR;
2125 #ifdef MLX5_PMD_SOFT_COUNTERS
2126 /* Update sent data bytes/packets counters. */
2127 txq->stats.obytes += dlen + vlan;
2129 wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m);
2130 loc->wqe_last = wqe;
2131 mlx5_tx_cseg_init(txq, loc, wqe, 0, MLX5_OPCODE_SEND, olx);
2132 ds = mlx5_tx_mseg_build(txq, loc, wqe, vlan, inlen, 0, olx);
2133 wqe->cseg.sq_ds = rte_cpu_to_be_32(txq->qp_num_8s | ds);
2134 txq->wqe_ci += (ds + 3) / 4;
2135 loc->wqe_free -= (ds + 3) / 4;
2136 return MLX5_TXCMP_CODE_MULTI;
2140 * Tx burst function for multi-segment packets. Supports all
2141 * types of Tx offloads, uses MLX5_OPCODE_SEND/TSO to build WQEs,
2142 * sends one packet per WQE. Function stops sending if it
2143 * encounters the single-segment packet.
2145 * This routine is responsible for storing processed mbuf
2146 * into elts ring buffer and update elts_head.
2149 * Pointer to TX queue structure.
2151 * Packets to transmit.
2153 * Number of packets in array.
2155 * Pointer to burst routine local context.
2157 * Configured Tx offloads mask. It is fully defined at
2158 * compile time and may be used for optimization.
2161 * MLX5_TXCMP_CODE_EXIT - sending is done or impossible.
2162 * MLX5_TXCMP_CODE_ERROR - some unrecoverable error occurred.
2163 * MLX5_TXCMP_CODE_SINGLE - single-segment packet encountered.
2164 * MLX5_TXCMP_CODE_TSO - TSO single-segment packet encountered.
2165 * Local context variables updated.
2167 static __rte_always_inline enum mlx5_txcmp_code
2168 mlx5_tx_burst_mseg(struct mlx5_txq_data *__rte_restrict txq,
2169 struct rte_mbuf **__rte_restrict pkts,
2170 unsigned int pkts_n,
2171 struct mlx5_txq_local *__rte_restrict loc,
2174 MLX5_ASSERT(loc->elts_free && loc->wqe_free);
2175 MLX5_ASSERT(pkts_n > loc->pkts_sent);
2176 pkts += loc->pkts_sent + 1;
2177 pkts_n -= loc->pkts_sent;
2179 enum mlx5_txcmp_code ret;
2181 MLX5_ASSERT(NB_SEGS(loc->mbuf) > 1);
2183 * Estimate the number of free elts quickly but conservatively.
2184 * Some segment may be fully inlined and freed,
2185 * ignore this here - precise estimation is costly.
2187 if (loc->elts_free < NB_SEGS(loc->mbuf))
2188 return MLX5_TXCMP_CODE_EXIT;
2189 if (MLX5_TXOFF_CONFIG(TSO) &&
2190 unlikely(loc->mbuf->ol_flags & RTE_MBUF_F_TX_TCP_SEG)) {
2191 /* Proceed with multi-segment TSO. */
2192 ret = mlx5_tx_packet_multi_tso(txq, loc, olx);
2193 } else if (MLX5_TXOFF_CONFIG(INLINE)) {
2194 /* Proceed with multi-segment SEND with inlining. */
2195 ret = mlx5_tx_packet_multi_inline(txq, loc, olx);
2197 /* Proceed with multi-segment SEND w/o inlining. */
2198 ret = mlx5_tx_packet_multi_send(txq, loc, olx);
2200 if (ret == MLX5_TXCMP_CODE_EXIT)
2201 return MLX5_TXCMP_CODE_EXIT;
2202 if (ret == MLX5_TXCMP_CODE_ERROR)
2203 return MLX5_TXCMP_CODE_ERROR;
2204 /* WQE is built, go to the next packet. */
2207 if (unlikely(!pkts_n || !loc->elts_free || !loc->wqe_free))
2208 return MLX5_TXCMP_CODE_EXIT;
2209 loc->mbuf = *pkts++;
2211 rte_prefetch0(*pkts);
2212 if (likely(NB_SEGS(loc->mbuf) > 1))
2214 /* Here ends the series of multi-segment packets. */
2215 if (MLX5_TXOFF_CONFIG(TSO) &&
2216 unlikely(loc->mbuf->ol_flags & RTE_MBUF_F_TX_TCP_SEG))
2217 return MLX5_TXCMP_CODE_TSO;
2218 return MLX5_TXCMP_CODE_SINGLE;
2224 * Tx burst function for single-segment packets with TSO.
2225 * Supports all types of Tx offloads, except multi-packets.
2226 * Uses MLX5_OPCODE_TSO to build WQEs, sends one packet per WQE.
2227 * Function stops sending if it encounters the multi-segment
2228 * packet or packet without TSO requested.
2230 * The routine is responsible for storing processed mbuf into elts ring buffer
2231 * and update elts_head if inline offloads is requested due to possible early
2232 * freeing of the inlined mbufs (can not store pkts array in elts as a batch).
2235 * Pointer to TX queue structure.
2237 * Packets to transmit.
2239 * Number of packets in array.
2241 * Pointer to burst routine local context.
2243 * Configured Tx offloads mask. It is fully defined at
2244 * compile time and may be used for optimization.
2247 * MLX5_TXCMP_CODE_EXIT - sending is done or impossible.
2248 * MLX5_TXCMP_CODE_ERROR - some unrecoverable error occurred.
2249 * MLX5_TXCMP_CODE_SINGLE - single-segment packet encountered.
2250 * MLX5_TXCMP_CODE_MULTI - multi-segment packet encountered.
2251 * Local context variables updated.
2253 static __rte_always_inline enum mlx5_txcmp_code
2254 mlx5_tx_burst_tso(struct mlx5_txq_data *__rte_restrict txq,
2255 struct rte_mbuf **__rte_restrict pkts,
2256 unsigned int pkts_n,
2257 struct mlx5_txq_local *__rte_restrict loc,
2260 MLX5_ASSERT(loc->elts_free && loc->wqe_free);
2261 MLX5_ASSERT(pkts_n > loc->pkts_sent);
2262 pkts += loc->pkts_sent + 1;
2263 pkts_n -= loc->pkts_sent;
2265 struct mlx5_wqe_dseg *__rte_restrict dseg;
2266 struct mlx5_wqe *__rte_restrict wqe;
2267 unsigned int ds, dlen, hlen, ntcp, vlan = 0;
2270 MLX5_ASSERT(NB_SEGS(loc->mbuf) == 1);
2271 if (MLX5_TXOFF_CONFIG(TXPP)) {
2272 enum mlx5_txcmp_code wret;
2274 /* Generate WAIT for scheduling if requested. */
2275 wret = mlx5_tx_schedule_send(txq, loc, olx);
2276 if (wret == MLX5_TXCMP_CODE_EXIT)
2277 return MLX5_TXCMP_CODE_EXIT;
2278 if (wret == MLX5_TXCMP_CODE_ERROR)
2279 return MLX5_TXCMP_CODE_ERROR;
2281 dlen = rte_pktmbuf_data_len(loc->mbuf);
2282 if (MLX5_TXOFF_CONFIG(VLAN) &&
2283 loc->mbuf->ol_flags & RTE_MBUF_F_TX_VLAN) {
2284 vlan = sizeof(struct rte_vlan_hdr);
2287 * First calculate the WQE size to check
2288 * whether we have enough space in ring buffer.
2290 hlen = loc->mbuf->l2_len + vlan +
2291 loc->mbuf->l3_len + loc->mbuf->l4_len;
2292 if (unlikely((!hlen || !loc->mbuf->tso_segsz)))
2293 return MLX5_TXCMP_CODE_ERROR;
2294 if (loc->mbuf->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK)
2295 hlen += loc->mbuf->outer_l2_len +
2296 loc->mbuf->outer_l3_len;
2297 /* Segment must contain all TSO headers. */
2298 if (unlikely(hlen > MLX5_MAX_TSO_HEADER ||
2299 hlen <= MLX5_ESEG_MIN_INLINE_SIZE ||
2300 hlen > (dlen + vlan)))
2301 return MLX5_TXCMP_CODE_ERROR;
2303 * Check whether there are enough free WQEBBs:
2305 * - Ethernet Segment
2306 * - First Segment of inlined Ethernet data
2307 * - ... data continued ...
2308 * - Finishing Data Segment of pointer type
2310 ds = 4 + (hlen - MLX5_ESEG_MIN_INLINE_SIZE +
2311 MLX5_WSEG_SIZE - 1) / MLX5_WSEG_SIZE;
2312 if (loc->wqe_free < ((ds + 3) / 4))
2313 return MLX5_TXCMP_CODE_EXIT;
2314 #ifdef MLX5_PMD_SOFT_COUNTERS
2315 /* Update sent data bytes/packets counters. */
2316 ntcp = (dlen + vlan - hlen +
2317 loc->mbuf->tso_segsz - 1) /
2318 loc->mbuf->tso_segsz;
2320 * One will be added for mbuf itself at the end
2321 * of the mlx5_tx_burst from loc->pkts_sent field.
2324 txq->stats.opackets += ntcp;
2325 txq->stats.obytes += dlen + vlan + ntcp * hlen;
2328 * Build the TSO WQE:
2330 * - Ethernet Segment with hlen bytes inlined
2331 * - Data Segment of pointer type
2333 wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m);
2334 loc->wqe_last = wqe;
2335 mlx5_tx_cseg_init(txq, loc, wqe, ds,
2336 MLX5_OPCODE_TSO, olx);
2337 dseg = mlx5_tx_eseg_data(txq, loc, wqe, vlan, hlen, 1, olx);
2338 dptr = rte_pktmbuf_mtod(loc->mbuf, uint8_t *) + hlen - vlan;
2339 dlen -= hlen - vlan;
2340 mlx5_tx_dseg_ptr(txq, loc, dseg, dptr, dlen, olx);
2342 * WQE is built, update the loop parameters
2343 * and go to the next packet.
2345 txq->wqe_ci += (ds + 3) / 4;
2346 loc->wqe_free -= (ds + 3) / 4;
2347 if (MLX5_TXOFF_CONFIG(INLINE))
2348 txq->elts[txq->elts_head++ & txq->elts_m] = loc->mbuf;
2352 if (unlikely(!pkts_n || !loc->elts_free || !loc->wqe_free))
2353 return MLX5_TXCMP_CODE_EXIT;
2354 loc->mbuf = *pkts++;
2356 rte_prefetch0(*pkts);
2357 if (MLX5_TXOFF_CONFIG(MULTI) &&
2358 unlikely(NB_SEGS(loc->mbuf) > 1))
2359 return MLX5_TXCMP_CODE_MULTI;
2360 if (likely(!(loc->mbuf->ol_flags & RTE_MBUF_F_TX_TCP_SEG)))
2361 return MLX5_TXCMP_CODE_SINGLE;
2362 /* Continue with the next TSO packet. */
2368 * Analyze the packet and select the best method to send.
2371 * Pointer to TX queue structure.
2373 * Pointer to burst routine local context.
2375 * Configured Tx offloads mask. It is fully defined at
2376 * compile time and may be used for optimization.
2378 * The predefined flag whether do complete check for
2379 * multi-segment packets and TSO.
2382 * MLX5_TXCMP_CODE_MULTI - multi-segment packet encountered.
2383 * MLX5_TXCMP_CODE_TSO - TSO required, use TSO/LSO.
2384 * MLX5_TXCMP_CODE_SINGLE - single-segment packet, use SEND.
2385 * MLX5_TXCMP_CODE_EMPW - single-segment packet, use MPW.
2387 static __rte_always_inline enum mlx5_txcmp_code
2388 mlx5_tx_able_to_empw(struct mlx5_txq_data *__rte_restrict txq,
2389 struct mlx5_txq_local *__rte_restrict loc,
2393 /* Check for multi-segment packet. */
2395 MLX5_TXOFF_CONFIG(MULTI) &&
2396 unlikely(NB_SEGS(loc->mbuf) > 1))
2397 return MLX5_TXCMP_CODE_MULTI;
2398 /* Check for TSO packet. */
2400 MLX5_TXOFF_CONFIG(TSO) &&
2401 unlikely(loc->mbuf->ol_flags & RTE_MBUF_F_TX_TCP_SEG))
2402 return MLX5_TXCMP_CODE_TSO;
2403 /* Check if eMPW is enabled at all. */
2404 if (!MLX5_TXOFF_CONFIG(EMPW))
2405 return MLX5_TXCMP_CODE_SINGLE;
2406 /* Check if eMPW can be engaged. */
2407 if (MLX5_TXOFF_CONFIG(VLAN) &&
2408 unlikely(loc->mbuf->ol_flags & RTE_MBUF_F_TX_VLAN) &&
2409 (!MLX5_TXOFF_CONFIG(INLINE) ||
2410 unlikely((rte_pktmbuf_data_len(loc->mbuf) +
2411 sizeof(struct rte_vlan_hdr)) > txq->inlen_empw))) {
2413 * eMPW does not support VLAN insertion offload, we have to
2414 * inline the entire packet but packet is too long for inlining.
2416 return MLX5_TXCMP_CODE_SINGLE;
2418 return MLX5_TXCMP_CODE_EMPW;
2422 * Check the next packet attributes to match with the eMPW batch ones.
2423 * In addition, for legacy MPW the packet length is checked either.
2426 * Pointer to TX queue structure.
2428 * Pointer to Ethernet Segment of eMPW batch.
2430 * Pointer to burst routine local context.
2432 * Length of previous packet in MPW descriptor.
2434 * Configured Tx offloads mask. It is fully defined at
2435 * compile time and may be used for optimization.
2438 * true - packet match with eMPW batch attributes.
2439 * false - no match, eMPW should be restarted.
2441 static __rte_always_inline bool
2442 mlx5_tx_match_empw(struct mlx5_txq_data *__rte_restrict txq,
2443 struct mlx5_wqe_eseg *__rte_restrict es,
2444 struct mlx5_txq_local *__rte_restrict loc,
2448 uint8_t swp_flags = 0;
2450 /* Compare the checksum flags, if any. */
2451 if (MLX5_TXOFF_CONFIG(CSUM) &&
2452 txq_ol_cksum_to_cs(loc->mbuf) != es->cs_flags)
2454 /* Compare the Software Parser offsets and flags. */
2455 if (MLX5_TXOFF_CONFIG(SWP) &&
2456 (es->swp_offs != txq_mbuf_to_swp(loc, &swp_flags, olx) ||
2457 es->swp_flags != swp_flags))
2459 /* Fill metadata field if needed. */
2460 if (MLX5_TXOFF_CONFIG(METADATA) &&
2461 es->metadata != (loc->mbuf->ol_flags & RTE_MBUF_DYNFLAG_TX_METADATA ?
2462 rte_cpu_to_be_32(*RTE_FLOW_DYNF_METADATA(loc->mbuf)) : 0))
2464 /* Legacy MPW can send packets with the same length only. */
2465 if (MLX5_TXOFF_CONFIG(MPW) &&
2466 dlen != rte_pktmbuf_data_len(loc->mbuf))
2468 /* There must be no VLAN packets in eMPW loop. */
2469 if (MLX5_TXOFF_CONFIG(VLAN))
2470 MLX5_ASSERT(!(loc->mbuf->ol_flags & RTE_MBUF_F_TX_VLAN));
2471 /* Check if the scheduling is requested. */
2472 if (MLX5_TXOFF_CONFIG(TXPP) &&
2473 loc->mbuf->ol_flags & txq->ts_mask)
2479 * Update send loop variables and WQE for eMPW loop without data inlining.
2480 * Number of Data Segments is equal to the number of sent packets.
2483 * Pointer to TX queue structure.
2485 * Pointer to burst routine local context.
2487 * Number of packets/Data Segments/Packets.
2489 * Accumulated statistics, bytes sent.
2491 * Configured Tx offloads mask. It is fully defined at
2492 * compile time and may be used for optimization.
2495 * true - packet match with eMPW batch attributes.
2496 * false - no match, eMPW should be restarted.
2498 static __rte_always_inline void
2499 mlx5_tx_sdone_empw(struct mlx5_txq_data *__rte_restrict txq,
2500 struct mlx5_txq_local *__rte_restrict loc,
2503 unsigned int olx __rte_unused)
2505 MLX5_ASSERT(!MLX5_TXOFF_CONFIG(INLINE));
2506 #ifdef MLX5_PMD_SOFT_COUNTERS
2507 /* Update sent data bytes counter. */
2508 txq->stats.obytes += slen;
2512 loc->elts_free -= ds;
2513 loc->pkts_sent += ds;
2515 loc->wqe_last->cseg.sq_ds = rte_cpu_to_be_32(txq->qp_num_8s | ds);
2516 txq->wqe_ci += (ds + 3) / 4;
2517 loc->wqe_free -= (ds + 3) / 4;
2521 * Update send loop variables and WQE for eMPW loop with data inlining.
2522 * Gets the size of pushed descriptors and data to the WQE.
2525 * Pointer to TX queue structure.
2527 * Pointer to burst routine local context.
2529 * Total size of descriptor/data in bytes.
2531 * Accumulated statistics, data bytes sent.
2533 * The base WQE for the eMPW/MPW descriptor.
2535 * Configured Tx offloads mask. It is fully defined at
2536 * compile time and may be used for optimization.
2539 * true - packet match with eMPW batch attributes.
2540 * false - no match, eMPW should be restarted.
2542 static __rte_always_inline void
2543 mlx5_tx_idone_empw(struct mlx5_txq_data *__rte_restrict txq,
2544 struct mlx5_txq_local *__rte_restrict loc,
2547 struct mlx5_wqe *__rte_restrict wqem,
2548 unsigned int olx __rte_unused)
2550 struct mlx5_wqe_dseg *dseg = &wqem->dseg[0];
2552 MLX5_ASSERT(MLX5_TXOFF_CONFIG(INLINE));
2553 #ifdef MLX5_PMD_SOFT_COUNTERS
2554 /* Update sent data bytes counter. */
2555 txq->stats.obytes += slen;
2559 if (MLX5_TXOFF_CONFIG(MPW) && dseg->bcount == RTE_BE32(0)) {
2561 * If the legacy MPW session contains the inline packets
2562 * we should set the only inline data segment length
2563 * and align the total length to the segment size.
2565 MLX5_ASSERT(len > sizeof(dseg->bcount));
2566 dseg->bcount = rte_cpu_to_be_32((len - sizeof(dseg->bcount)) |
2567 MLX5_ETH_WQE_DATA_INLINE);
2568 len = (len + MLX5_WSEG_SIZE - 1) / MLX5_WSEG_SIZE + 2;
2571 * The session is not legacy MPW or contains the
2572 * data buffer pointer segments.
2574 MLX5_ASSERT((len % MLX5_WSEG_SIZE) == 0);
2575 len = len / MLX5_WSEG_SIZE + 2;
2577 wqem->cseg.sq_ds = rte_cpu_to_be_32(txq->qp_num_8s | len);
2578 txq->wqe_ci += (len + 3) / 4;
2579 loc->wqe_free -= (len + 3) / 4;
2580 loc->wqe_last = wqem;
2584 * The set of Tx burst functions for single-segment packets without TSO
2585 * and with Multi-Packet Writing feature support.
2586 * Supports all types of Tx offloads, except multi-packets and TSO.
2588 * Uses MLX5_OPCODE_EMPW to build WQEs if possible and sends as many packet
2589 * per WQE as it can. If eMPW is not configured or packet can not be sent with
2590 * eMPW (VLAN insertion) the ordinary SEND opcode is used and only one packet
2593 * Functions stop sending if it encounters the multi-segment packet or packet
2594 * with TSO requested.
2596 * The routines are responsible for storing processed mbuf into elts ring buffer
2597 * and update elts_head if inlining offload is requested. Otherwise the copying
2598 * mbufs to elts can be postponed and completed at the end of burst routine.
2601 * Pointer to TX queue structure.
2603 * Packets to transmit.
2605 * Number of packets in array.
2607 * Pointer to burst routine local context.
2609 * Configured Tx offloads mask. It is fully defined at
2610 * compile time and may be used for optimization.
2613 * MLX5_TXCMP_CODE_EXIT - sending is done or impossible.
2614 * MLX5_TXCMP_CODE_ERROR - some unrecoverable error occurred.
2615 * MLX5_TXCMP_CODE_MULTI - multi-segment packet encountered.
2616 * MLX5_TXCMP_CODE_TSO - TSO packet encountered.
2617 * MLX5_TXCMP_CODE_SINGLE - used inside functions set.
2618 * MLX5_TXCMP_CODE_EMPW - used inside functions set.
2620 * Local context variables updated.
2623 * The routine sends packets with MLX5_OPCODE_EMPW
2624 * without inlining, this is dedicated optimized branch.
2625 * No VLAN insertion is supported.
2627 static __rte_always_inline enum mlx5_txcmp_code
2628 mlx5_tx_burst_empw_simple(struct mlx5_txq_data *__rte_restrict txq,
2629 struct rte_mbuf **__rte_restrict pkts,
2630 unsigned int pkts_n,
2631 struct mlx5_txq_local *__rte_restrict loc,
2635 * Subroutine is the part of mlx5_tx_burst_single() and sends
2636 * single-segment packet with eMPW opcode without data inlining.
2638 MLX5_ASSERT(!MLX5_TXOFF_CONFIG(INLINE));
2639 MLX5_ASSERT(MLX5_TXOFF_CONFIG(EMPW));
2640 MLX5_ASSERT(loc->elts_free && loc->wqe_free);
2641 MLX5_ASSERT(pkts_n > loc->pkts_sent);
2642 pkts += loc->pkts_sent + 1;
2643 pkts_n -= loc->pkts_sent;
2645 struct mlx5_wqe_dseg *__rte_restrict dseg;
2646 struct mlx5_wqe_eseg *__rte_restrict eseg;
2647 enum mlx5_txcmp_code ret;
2648 unsigned int part, loop;
2649 unsigned int slen = 0;
2652 MLX5_ASSERT(NB_SEGS(loc->mbuf) == 1);
2653 if (MLX5_TXOFF_CONFIG(TXPP)) {
2654 enum mlx5_txcmp_code wret;
2656 /* Generate WAIT for scheduling if requested. */
2657 wret = mlx5_tx_schedule_send(txq, loc, olx);
2658 if (wret == MLX5_TXCMP_CODE_EXIT)
2659 return MLX5_TXCMP_CODE_EXIT;
2660 if (wret == MLX5_TXCMP_CODE_ERROR)
2661 return MLX5_TXCMP_CODE_ERROR;
2663 part = RTE_MIN(pkts_n, MLX5_TXOFF_CONFIG(MPW) ?
2664 MLX5_MPW_MAX_PACKETS :
2665 MLX5_EMPW_MAX_PACKETS);
2666 if (unlikely(loc->elts_free < part)) {
2667 /* We have no enough elts to save all mbufs. */
2668 if (unlikely(loc->elts_free < MLX5_EMPW_MIN_PACKETS))
2669 return MLX5_TXCMP_CODE_EXIT;
2670 /* But we still able to send at least minimal eMPW. */
2671 part = loc->elts_free;
2673 /* Check whether we have enough WQEs */
2674 if (unlikely(loc->wqe_free < ((2 + part + 3) / 4))) {
2675 if (unlikely(loc->wqe_free <
2676 ((2 + MLX5_EMPW_MIN_PACKETS + 3) / 4)))
2677 return MLX5_TXCMP_CODE_EXIT;
2678 part = (loc->wqe_free * 4) - 2;
2680 if (likely(part > 1))
2681 rte_prefetch0(*pkts);
2682 loc->wqe_last = txq->wqes + (txq->wqe_ci & txq->wqe_m);
2684 * Build eMPW title WQEBB:
2685 * - Control Segment, eMPW opcode
2686 * - Ethernet Segment, no inline
2688 mlx5_tx_cseg_init(txq, loc, loc->wqe_last, part + 2,
2689 MLX5_OPCODE_ENHANCED_MPSW, olx);
2690 mlx5_tx_eseg_none(txq, loc, loc->wqe_last,
2691 olx & ~MLX5_TXOFF_CONFIG_VLAN);
2692 eseg = &loc->wqe_last->eseg;
2693 dseg = &loc->wqe_last->dseg[0];
2695 /* Store the packet length for legacy MPW. */
2696 if (MLX5_TXOFF_CONFIG(MPW))
2697 eseg->mss = rte_cpu_to_be_16
2698 (rte_pktmbuf_data_len(loc->mbuf));
2700 uint32_t dlen = rte_pktmbuf_data_len(loc->mbuf);
2701 #ifdef MLX5_PMD_SOFT_COUNTERS
2702 /* Update sent data bytes counter. */
2707 rte_pktmbuf_mtod(loc->mbuf, uint8_t *),
2709 if (unlikely(--loop == 0))
2711 loc->mbuf = *pkts++;
2712 if (likely(loop > 1))
2713 rte_prefetch0(*pkts);
2714 ret = mlx5_tx_able_to_empw(txq, loc, olx, true);
2716 * Unroll the completion code to avoid
2717 * returning variable value - it results in
2718 * unoptimized sequent checking in caller.
2720 if (ret == MLX5_TXCMP_CODE_MULTI) {
2722 mlx5_tx_sdone_empw(txq, loc, part, slen, olx);
2723 if (unlikely(!loc->elts_free ||
2725 return MLX5_TXCMP_CODE_EXIT;
2726 return MLX5_TXCMP_CODE_MULTI;
2728 MLX5_ASSERT(NB_SEGS(loc->mbuf) == 1);
2729 if (ret == MLX5_TXCMP_CODE_TSO) {
2731 mlx5_tx_sdone_empw(txq, loc, part, slen, olx);
2732 if (unlikely(!loc->elts_free ||
2734 return MLX5_TXCMP_CODE_EXIT;
2735 return MLX5_TXCMP_CODE_TSO;
2737 if (ret == MLX5_TXCMP_CODE_SINGLE) {
2739 mlx5_tx_sdone_empw(txq, loc, part, slen, olx);
2740 if (unlikely(!loc->elts_free ||
2742 return MLX5_TXCMP_CODE_EXIT;
2743 return MLX5_TXCMP_CODE_SINGLE;
2745 if (ret != MLX5_TXCMP_CODE_EMPW) {
2748 mlx5_tx_sdone_empw(txq, loc, part, slen, olx);
2749 return MLX5_TXCMP_CODE_ERROR;
2752 * Check whether packet parameters coincide
2753 * within assumed eMPW batch:
2754 * - check sum settings
2756 * - software parser settings
2757 * - packets length (legacy MPW only)
2758 * - scheduling is not required
2760 if (!mlx5_tx_match_empw(txq, eseg, loc, dlen, olx)) {
2763 mlx5_tx_sdone_empw(txq, loc, part, slen, olx);
2764 if (unlikely(!loc->elts_free ||
2766 return MLX5_TXCMP_CODE_EXIT;
2770 /* Packet attributes match, continue the same eMPW. */
2772 if ((uintptr_t)dseg >= (uintptr_t)txq->wqes_end)
2773 dseg = (struct mlx5_wqe_dseg *)txq->wqes;
2775 /* eMPW is built successfully, update loop parameters. */
2777 MLX5_ASSERT(pkts_n >= part);
2778 #ifdef MLX5_PMD_SOFT_COUNTERS
2779 /* Update sent data bytes counter. */
2780 txq->stats.obytes += slen;
2782 loc->elts_free -= part;
2783 loc->pkts_sent += part;
2784 txq->wqe_ci += (2 + part + 3) / 4;
2785 loc->wqe_free -= (2 + part + 3) / 4;
2787 if (unlikely(!pkts_n || !loc->elts_free || !loc->wqe_free))
2788 return MLX5_TXCMP_CODE_EXIT;
2789 loc->mbuf = *pkts++;
2790 ret = mlx5_tx_able_to_empw(txq, loc, olx, true);
2791 if (unlikely(ret != MLX5_TXCMP_CODE_EMPW))
2793 /* Continue sending eMPW batches. */
2799 * The routine sends packets with MLX5_OPCODE_EMPW
2800 * with inlining, optionally supports VLAN insertion.
2802 static __rte_always_inline enum mlx5_txcmp_code
2803 mlx5_tx_burst_empw_inline(struct mlx5_txq_data *__rte_restrict txq,
2804 struct rte_mbuf **__rte_restrict pkts,
2805 unsigned int pkts_n,
2806 struct mlx5_txq_local *__rte_restrict loc,
2810 * Subroutine is the part of mlx5_tx_burst_single() and sends
2811 * single-segment packet with eMPW opcode with data inlining.
2813 MLX5_ASSERT(MLX5_TXOFF_CONFIG(INLINE));
2814 MLX5_ASSERT(MLX5_TXOFF_CONFIG(EMPW));
2815 MLX5_ASSERT(loc->elts_free && loc->wqe_free);
2816 MLX5_ASSERT(pkts_n > loc->pkts_sent);
2817 pkts += loc->pkts_sent + 1;
2818 pkts_n -= loc->pkts_sent;
2820 struct mlx5_wqe_dseg *__rte_restrict dseg;
2821 struct mlx5_wqe *__rte_restrict wqem;
2822 enum mlx5_txcmp_code ret;
2823 unsigned int room, part, nlim;
2824 unsigned int slen = 0;
2826 MLX5_ASSERT(NB_SEGS(loc->mbuf) == 1);
2827 if (MLX5_TXOFF_CONFIG(TXPP)) {
2828 enum mlx5_txcmp_code wret;
2830 /* Generate WAIT for scheduling if requested. */
2831 wret = mlx5_tx_schedule_send(txq, loc, olx);
2832 if (wret == MLX5_TXCMP_CODE_EXIT)
2833 return MLX5_TXCMP_CODE_EXIT;
2834 if (wret == MLX5_TXCMP_CODE_ERROR)
2835 return MLX5_TXCMP_CODE_ERROR;
2838 * Limits the amount of packets in one WQE
2839 * to improve CQE latency generation.
2841 nlim = RTE_MIN(pkts_n, MLX5_TXOFF_CONFIG(MPW) ?
2842 MLX5_MPW_INLINE_MAX_PACKETS :
2843 MLX5_EMPW_MAX_PACKETS);
2844 /* Check whether we have minimal amount WQEs */
2845 if (unlikely(loc->wqe_free <
2846 ((2 + MLX5_EMPW_MIN_PACKETS + 3) / 4)))
2847 return MLX5_TXCMP_CODE_EXIT;
2848 if (likely(pkts_n > 1))
2849 rte_prefetch0(*pkts);
2850 wqem = txq->wqes + (txq->wqe_ci & txq->wqe_m);
2852 * Build eMPW title WQEBB:
2853 * - Control Segment, eMPW opcode, zero DS
2854 * - Ethernet Segment, no inline
2856 mlx5_tx_cseg_init(txq, loc, wqem, 0,
2857 MLX5_OPCODE_ENHANCED_MPSW, olx);
2858 mlx5_tx_eseg_none(txq, loc, wqem,
2859 olx & ~MLX5_TXOFF_CONFIG_VLAN);
2860 dseg = &wqem->dseg[0];
2861 /* Store the packet length for legacy MPW. */
2862 if (MLX5_TXOFF_CONFIG(MPW))
2863 wqem->eseg.mss = rte_cpu_to_be_16
2864 (rte_pktmbuf_data_len(loc->mbuf));
2865 room = RTE_MIN(MLX5_WQE_SIZE_MAX / MLX5_WQE_SIZE,
2866 loc->wqe_free) * MLX5_WQE_SIZE -
2867 MLX5_WQE_CSEG_SIZE -
2869 /* Limit the room for legacy MPW sessions for performance. */
2870 if (MLX5_TXOFF_CONFIG(MPW))
2871 room = RTE_MIN(room,
2872 RTE_MAX(txq->inlen_empw +
2873 sizeof(dseg->bcount) +
2874 (MLX5_TXOFF_CONFIG(VLAN) ?
2875 sizeof(struct rte_vlan_hdr) : 0),
2876 MLX5_MPW_INLINE_MAX_PACKETS *
2877 MLX5_WQE_DSEG_SIZE));
2878 /* Build WQE till we have space, packets and resources. */
2881 uint32_t dlen = rte_pktmbuf_data_len(loc->mbuf);
2882 uint8_t *dptr = rte_pktmbuf_mtod(loc->mbuf, uint8_t *);
2885 MLX5_ASSERT(room >= MLX5_WQE_DSEG_SIZE);
2886 MLX5_ASSERT((room % MLX5_WQE_DSEG_SIZE) == 0);
2887 MLX5_ASSERT((uintptr_t)dseg < (uintptr_t)txq->wqes_end);
2889 * Some Tx offloads may cause an error if packet is not
2890 * long enough, check against assumed minimal length.
2892 if (unlikely(dlen <= MLX5_ESEG_MIN_INLINE_SIZE)) {
2894 if (unlikely(!part))
2895 return MLX5_TXCMP_CODE_ERROR;
2897 * We have some successfully built
2898 * packet Data Segments to send.
2900 mlx5_tx_idone_empw(txq, loc, part,
2902 return MLX5_TXCMP_CODE_ERROR;
2904 /* Inline or not inline - that's the Question. */
2905 if (dlen > txq->inlen_empw ||
2906 loc->mbuf->ol_flags & RTE_MBUF_F_TX_DYNF_NOINLINE)
2908 if (MLX5_TXOFF_CONFIG(MPW)) {
2909 if (dlen > txq->inlen_send)
2913 /* Open new inline MPW session. */
2914 tlen += sizeof(dseg->bcount);
2915 dseg->bcount = RTE_BE32(0);
2917 (dseg, sizeof(dseg->bcount));
2920 * No pointer and inline descriptor
2921 * intermix for legacy MPW sessions.
2923 if (wqem->dseg[0].bcount)
2927 tlen = sizeof(dseg->bcount) + dlen;
2929 /* Inline entire packet, optional VLAN insertion. */
2930 if (MLX5_TXOFF_CONFIG(VLAN) &&
2931 loc->mbuf->ol_flags & RTE_MBUF_F_TX_VLAN) {
2933 * The packet length must be checked in
2934 * mlx5_tx_able_to_empw() and packet
2935 * fits into inline length guaranteed.
2938 sizeof(struct rte_vlan_hdr)) <=
2940 tlen += sizeof(struct rte_vlan_hdr);
2943 dseg = mlx5_tx_dseg_vlan(txq, loc, dseg,
2945 #ifdef MLX5_PMD_SOFT_COUNTERS
2946 /* Update sent data bytes counter. */
2947 slen += sizeof(struct rte_vlan_hdr);
2952 dseg = mlx5_tx_dseg_empw(txq, loc, dseg,
2955 if (!MLX5_TXOFF_CONFIG(MPW))
2956 tlen = RTE_ALIGN(tlen, MLX5_WSEG_SIZE);
2957 MLX5_ASSERT(room >= tlen);
2960 * Packet data are completely inline,
2961 * we can try to free the packet.
2963 if (likely(loc->pkts_sent == loc->mbuf_free)) {
2965 * All the packets from the burst beginning
2966 * are inline, we can free mbufs directly
2967 * from the origin array on tx_burst exit().
2973 * In order no to call rte_pktmbuf_free_seg() here,
2974 * in the most inner loop (that might be very
2975 * expensive) we just save the mbuf in elts.
2977 txq->elts[txq->elts_head++ & txq->elts_m] = loc->mbuf;
2982 * No pointer and inline descriptor
2983 * intermix for legacy MPW sessions.
2985 if (MLX5_TXOFF_CONFIG(MPW) &&
2987 wqem->dseg[0].bcount == RTE_BE32(0))
2990 * Not inlinable VLAN packets are
2991 * proceeded outside of this routine.
2993 MLX5_ASSERT(room >= MLX5_WQE_DSEG_SIZE);
2994 if (MLX5_TXOFF_CONFIG(VLAN))
2995 MLX5_ASSERT(!(loc->mbuf->ol_flags &
2996 RTE_MBUF_F_TX_VLAN));
2997 mlx5_tx_dseg_ptr(txq, loc, dseg, dptr, dlen, olx);
2998 /* We have to store mbuf in elts.*/
2999 txq->elts[txq->elts_head++ & txq->elts_m] = loc->mbuf;
3001 room -= MLX5_WQE_DSEG_SIZE;
3002 /* Ring buffer wraparound is checked at the loop end.*/
3005 #ifdef MLX5_PMD_SOFT_COUNTERS
3006 /* Update sent data bytes counter. */
3011 if (unlikely(!pkts_n || !loc->elts_free)) {
3013 * We have no resources/packets to
3014 * continue build descriptors.
3017 mlx5_tx_idone_empw(txq, loc, part,
3019 return MLX5_TXCMP_CODE_EXIT;
3021 loc->mbuf = *pkts++;
3022 if (likely(pkts_n > 1))
3023 rte_prefetch0(*pkts);
3024 ret = mlx5_tx_able_to_empw(txq, loc, olx, true);
3026 * Unroll the completion code to avoid
3027 * returning variable value - it results in
3028 * unoptimized sequent checking in caller.
3030 if (ret == MLX5_TXCMP_CODE_MULTI) {
3032 mlx5_tx_idone_empw(txq, loc, part,
3034 if (unlikely(!loc->elts_free ||
3036 return MLX5_TXCMP_CODE_EXIT;
3037 return MLX5_TXCMP_CODE_MULTI;
3039 MLX5_ASSERT(NB_SEGS(loc->mbuf) == 1);
3040 if (ret == MLX5_TXCMP_CODE_TSO) {
3042 mlx5_tx_idone_empw(txq, loc, part,
3044 if (unlikely(!loc->elts_free ||
3046 return MLX5_TXCMP_CODE_EXIT;
3047 return MLX5_TXCMP_CODE_TSO;
3049 if (ret == MLX5_TXCMP_CODE_SINGLE) {
3051 mlx5_tx_idone_empw(txq, loc, part,
3053 if (unlikely(!loc->elts_free ||
3055 return MLX5_TXCMP_CODE_EXIT;
3056 return MLX5_TXCMP_CODE_SINGLE;
3058 if (ret != MLX5_TXCMP_CODE_EMPW) {
3061 mlx5_tx_idone_empw(txq, loc, part,
3063 return MLX5_TXCMP_CODE_ERROR;
3065 /* Check if we have minimal room left. */
3067 if (unlikely(!nlim || room < MLX5_WQE_DSEG_SIZE))
3070 * Check whether packet parameters coincide
3071 * within assumed eMPW batch:
3072 * - check sum settings
3074 * - software parser settings
3075 * - packets length (legacy MPW only)
3076 * - scheduling is not required
3078 if (!mlx5_tx_match_empw(txq, &wqem->eseg,
3081 /* Packet attributes match, continue the same eMPW. */
3082 if ((uintptr_t)dseg >= (uintptr_t)txq->wqes_end)
3083 dseg = (struct mlx5_wqe_dseg *)txq->wqes;
3086 * We get here to close an existing eMPW
3087 * session and start the new one.
3089 MLX5_ASSERT(pkts_n);
3091 if (unlikely(!part))
3092 return MLX5_TXCMP_CODE_EXIT;
3093 mlx5_tx_idone_empw(txq, loc, part, slen, wqem, olx);
3094 if (unlikely(!loc->elts_free ||
3096 return MLX5_TXCMP_CODE_EXIT;
3097 /* Continue the loop with new eMPW session. */
3103 * The routine sends packets with ordinary MLX5_OPCODE_SEND.
3104 * Data inlining and VLAN insertion are supported.
3106 static __rte_always_inline enum mlx5_txcmp_code
3107 mlx5_tx_burst_single_send(struct mlx5_txq_data *__rte_restrict txq,
3108 struct rte_mbuf **__rte_restrict pkts,
3109 unsigned int pkts_n,
3110 struct mlx5_txq_local *__rte_restrict loc,
3114 * Subroutine is the part of mlx5_tx_burst_single()
3115 * and sends single-segment packet with SEND opcode.
3117 MLX5_ASSERT(loc->elts_free && loc->wqe_free);
3118 MLX5_ASSERT(pkts_n > loc->pkts_sent);
3119 pkts += loc->pkts_sent + 1;
3120 pkts_n -= loc->pkts_sent;
3122 struct mlx5_wqe *__rte_restrict wqe;
3123 enum mlx5_txcmp_code ret;
3125 MLX5_ASSERT(NB_SEGS(loc->mbuf) == 1);
3126 if (MLX5_TXOFF_CONFIG(TXPP)) {
3127 enum mlx5_txcmp_code wret;
3129 /* Generate WAIT for scheduling if requested. */
3130 wret = mlx5_tx_schedule_send(txq, loc, olx);
3131 if (wret == MLX5_TXCMP_CODE_EXIT)
3132 return MLX5_TXCMP_CODE_EXIT;
3133 if (wret == MLX5_TXCMP_CODE_ERROR)
3134 return MLX5_TXCMP_CODE_ERROR;
3136 if (MLX5_TXOFF_CONFIG(INLINE)) {
3137 unsigned int inlen, vlan = 0;
3139 inlen = rte_pktmbuf_data_len(loc->mbuf);
3140 if (MLX5_TXOFF_CONFIG(VLAN) &&
3141 loc->mbuf->ol_flags & RTE_MBUF_F_TX_VLAN) {
3142 vlan = sizeof(struct rte_vlan_hdr);
3146 * If inlining is enabled at configuration time
3147 * the limit must be not less than minimal size.
3148 * Otherwise we would do extra check for data
3149 * size to avoid crashes due to length overflow.
3151 MLX5_ASSERT(txq->inlen_send >=
3152 MLX5_ESEG_MIN_INLINE_SIZE);
3153 if (inlen <= txq->inlen_send) {
3154 unsigned int seg_n, wqe_n;
3156 rte_prefetch0(rte_pktmbuf_mtod
3157 (loc->mbuf, uint8_t *));
3158 /* Check against minimal length. */
3159 if (inlen <= MLX5_ESEG_MIN_INLINE_SIZE)
3160 return MLX5_TXCMP_CODE_ERROR;
3161 if (loc->mbuf->ol_flags &
3162 RTE_MBUF_F_TX_DYNF_NOINLINE) {
3164 * The hint flag not to inline packet
3165 * data is set. Check whether we can
3168 if ((!MLX5_TXOFF_CONFIG(EMPW) &&
3170 (MLX5_TXOFF_CONFIG(MPW) &&
3172 if (inlen <= txq->inlen_send)
3175 * The hardware requires the
3176 * minimal inline data header.
3178 goto single_min_inline;
3180 if (MLX5_TXOFF_CONFIG(VLAN) &&
3181 vlan && !txq->vlan_en) {
3183 * We must insert VLAN tag
3184 * by software means.
3186 goto single_part_inline;
3188 goto single_no_inline;
3192 * Completely inlined packet data WQE:
3193 * - Control Segment, SEND opcode
3194 * - Ethernet Segment, no VLAN insertion
3195 * - Data inlined, VLAN optionally inserted
3196 * - Alignment to MLX5_WSEG_SIZE
3197 * Have to estimate amount of WQEBBs
3199 seg_n = (inlen + 3 * MLX5_WSEG_SIZE -
3200 MLX5_ESEG_MIN_INLINE_SIZE +
3201 MLX5_WSEG_SIZE - 1) / MLX5_WSEG_SIZE;
3202 /* Check if there are enough WQEBBs. */
3203 wqe_n = (seg_n + 3) / 4;
3204 if (wqe_n > loc->wqe_free)
3205 return MLX5_TXCMP_CODE_EXIT;
3206 wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m);
3207 loc->wqe_last = wqe;
3208 mlx5_tx_cseg_init(txq, loc, wqe, seg_n,
3209 MLX5_OPCODE_SEND, olx);
3210 mlx5_tx_eseg_data(txq, loc, wqe,
3211 vlan, inlen, 0, olx);
3212 txq->wqe_ci += wqe_n;
3213 loc->wqe_free -= wqe_n;
3215 * Packet data are completely inlined,
3216 * free the packet immediately.
3218 rte_pktmbuf_free_seg(loc->mbuf);
3219 } else if ((!MLX5_TXOFF_CONFIG(EMPW) ||
3220 MLX5_TXOFF_CONFIG(MPW)) &&
3223 * If minimal inlining is requested the eMPW
3224 * feature should be disabled due to data is
3225 * inlined into Ethernet Segment, which can
3226 * not contain inlined data for eMPW due to
3227 * segment shared for all packets.
3229 struct mlx5_wqe_dseg *__rte_restrict dseg;
3234 * The inline-mode settings require
3235 * to inline the specified amount of
3236 * data bytes to the Ethernet Segment.
3237 * We should check the free space in
3238 * WQE ring buffer to inline partially.
3241 MLX5_ASSERT(txq->inlen_send >= txq->inlen_mode);
3242 MLX5_ASSERT(inlen > txq->inlen_mode);
3243 MLX5_ASSERT(txq->inlen_mode >=
3244 MLX5_ESEG_MIN_INLINE_SIZE);
3246 * Check whether there are enough free WQEBBs:
3248 * - Ethernet Segment
3249 * - First Segment of inlined Ethernet data
3250 * - ... data continued ...
3251 * - Finishing Data Segment of pointer type
3253 ds = (MLX5_WQE_CSEG_SIZE +
3254 MLX5_WQE_ESEG_SIZE +
3255 MLX5_WQE_DSEG_SIZE +
3257 MLX5_ESEG_MIN_INLINE_SIZE +
3258 MLX5_WQE_DSEG_SIZE +
3259 MLX5_WSEG_SIZE - 1) / MLX5_WSEG_SIZE;
3260 if (loc->wqe_free < ((ds + 3) / 4))
3261 return MLX5_TXCMP_CODE_EXIT;
3263 * Build the ordinary SEND WQE:
3265 * - Ethernet Segment, inline inlen_mode bytes
3266 * - Data Segment of pointer type
3268 wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m);
3269 loc->wqe_last = wqe;
3270 mlx5_tx_cseg_init(txq, loc, wqe, ds,
3271 MLX5_OPCODE_SEND, olx);
3272 dseg = mlx5_tx_eseg_data(txq, loc, wqe, vlan,
3275 dptr = rte_pktmbuf_mtod(loc->mbuf, uint8_t *) +
3276 txq->inlen_mode - vlan;
3277 inlen -= txq->inlen_mode;
3278 mlx5_tx_dseg_ptr(txq, loc, dseg,
3281 * WQE is built, update the loop parameters
3282 * and got to the next packet.
3284 txq->wqe_ci += (ds + 3) / 4;
3285 loc->wqe_free -= (ds + 3) / 4;
3286 /* We have to store mbuf in elts.*/
3287 MLX5_ASSERT(MLX5_TXOFF_CONFIG(INLINE));
3288 txq->elts[txq->elts_head++ & txq->elts_m] =
3296 * Partially inlined packet data WQE, we have
3297 * some space in title WQEBB, we can fill it
3298 * with some packet data. It takes one WQEBB,
3299 * it is available, no extra space check:
3300 * - Control Segment, SEND opcode
3301 * - Ethernet Segment, no VLAN insertion
3302 * - MLX5_ESEG_MIN_INLINE_SIZE bytes of Data
3303 * - Data Segment, pointer type
3305 * We also get here if VLAN insertion is not
3306 * supported by HW, the inline is enabled.
3309 wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m);
3310 loc->wqe_last = wqe;
3311 mlx5_tx_cseg_init(txq, loc, wqe, 4,
3312 MLX5_OPCODE_SEND, olx);
3313 mlx5_tx_eseg_dmin(txq, loc, wqe, vlan, olx);
3314 dptr = rte_pktmbuf_mtod(loc->mbuf, uint8_t *) +
3315 MLX5_ESEG_MIN_INLINE_SIZE - vlan;
3317 * The length check is performed above, by
3318 * comparing with txq->inlen_send. We should
3319 * not get overflow here.
3321 MLX5_ASSERT(inlen > MLX5_ESEG_MIN_INLINE_SIZE);
3322 dlen = inlen - MLX5_ESEG_MIN_INLINE_SIZE;
3323 mlx5_tx_dseg_ptr(txq, loc, &wqe->dseg[1],
3327 /* We have to store mbuf in elts.*/
3328 MLX5_ASSERT(MLX5_TXOFF_CONFIG(INLINE));
3329 txq->elts[txq->elts_head++ & txq->elts_m] =
3333 #ifdef MLX5_PMD_SOFT_COUNTERS
3334 /* Update sent data bytes counter. */
3335 txq->stats.obytes += vlan +
3336 rte_pktmbuf_data_len(loc->mbuf);
3340 * No inline at all, it means the CPU cycles saving
3341 * is prioritized at configuration, we should not
3342 * copy any packet data to WQE.
3344 * SEND WQE, one WQEBB:
3345 * - Control Segment, SEND opcode
3346 * - Ethernet Segment, optional VLAN, no inline
3347 * - Data Segment, pointer type
3350 wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m);
3351 loc->wqe_last = wqe;
3352 mlx5_tx_cseg_init(txq, loc, wqe, 3,
3353 MLX5_OPCODE_SEND, olx);
3354 mlx5_tx_eseg_none(txq, loc, wqe, olx);
3356 (txq, loc, &wqe->dseg[0],
3357 rte_pktmbuf_mtod(loc->mbuf, uint8_t *),
3358 rte_pktmbuf_data_len(loc->mbuf), olx);
3362 * We should not store mbuf pointer in elts
3363 * if no inlining is configured, this is done
3364 * by calling routine in a batch copy.
3366 MLX5_ASSERT(!MLX5_TXOFF_CONFIG(INLINE));
3368 #ifdef MLX5_PMD_SOFT_COUNTERS
3369 /* Update sent data bytes counter. */
3370 txq->stats.obytes += rte_pktmbuf_data_len(loc->mbuf);
3371 if (MLX5_TXOFF_CONFIG(VLAN) &&
3372 loc->mbuf->ol_flags & RTE_MBUF_F_TX_VLAN)
3373 txq->stats.obytes +=
3374 sizeof(struct rte_vlan_hdr);
3379 if (unlikely(!pkts_n || !loc->elts_free || !loc->wqe_free))
3380 return MLX5_TXCMP_CODE_EXIT;
3381 loc->mbuf = *pkts++;
3383 rte_prefetch0(*pkts);
3384 ret = mlx5_tx_able_to_empw(txq, loc, olx, true);
3385 if (unlikely(ret != MLX5_TXCMP_CODE_SINGLE))
3391 static __rte_always_inline enum mlx5_txcmp_code
3392 mlx5_tx_burst_single(struct mlx5_txq_data *__rte_restrict txq,
3393 struct rte_mbuf **__rte_restrict pkts,
3394 unsigned int pkts_n,
3395 struct mlx5_txq_local *__rte_restrict loc,
3398 enum mlx5_txcmp_code ret;
3400 ret = mlx5_tx_able_to_empw(txq, loc, olx, false);
3401 if (ret == MLX5_TXCMP_CODE_SINGLE)
3403 MLX5_ASSERT(ret == MLX5_TXCMP_CODE_EMPW);
3405 /* Optimize for inline/no inline eMPW send. */
3406 ret = (MLX5_TXOFF_CONFIG(INLINE)) ?
3407 mlx5_tx_burst_empw_inline
3408 (txq, pkts, pkts_n, loc, olx) :
3409 mlx5_tx_burst_empw_simple
3410 (txq, pkts, pkts_n, loc, olx);
3411 if (ret != MLX5_TXCMP_CODE_SINGLE)
3413 /* The resources to send one packet should remain. */
3414 MLX5_ASSERT(loc->elts_free && loc->wqe_free);
3416 ret = mlx5_tx_burst_single_send(txq, pkts, pkts_n, loc, olx);
3417 MLX5_ASSERT(ret != MLX5_TXCMP_CODE_SINGLE);
3418 if (ret != MLX5_TXCMP_CODE_EMPW)
3420 /* The resources to send one packet should remain. */
3421 MLX5_ASSERT(loc->elts_free && loc->wqe_free);
3426 * DPDK Tx callback template. This is configured template used to generate
3427 * routines optimized for specified offload setup.
3428 * One of this generated functions is chosen at SQ configuration time.
3431 * Generic pointer to TX queue structure.
3433 * Packets to transmit.
3435 * Number of packets in array.
3437 * Configured offloads mask, presents the bits of MLX5_TXOFF_CONFIG_xxx
3438 * values. Should be static to take compile time static configuration
3442 * Number of packets successfully transmitted (<= pkts_n).
3444 static __rte_always_inline uint16_t
3445 mlx5_tx_burst_tmpl(struct mlx5_txq_data *__rte_restrict txq,
3446 struct rte_mbuf **__rte_restrict pkts,
3450 struct mlx5_txq_local loc;
3451 enum mlx5_txcmp_code ret;
3454 MLX5_ASSERT(txq->elts_s >= (uint16_t)(txq->elts_head - txq->elts_tail));
3455 MLX5_ASSERT(txq->wqe_s >= (uint16_t)(txq->wqe_ci - txq->wqe_pi));
3456 if (unlikely(!pkts_n))
3458 if (MLX5_TXOFF_CONFIG(INLINE))
3462 loc.wqe_last = NULL;
3465 loc.pkts_loop = loc.pkts_sent;
3467 * Check if there are some CQEs, if any:
3468 * - process an encountered errors
3469 * - process the completed WQEs
3470 * - free related mbufs
3471 * - doorbell the NIC about processed CQEs
3473 rte_prefetch0(*(pkts + loc.pkts_sent));
3474 mlx5_tx_handle_completion(txq, olx);
3476 * Calculate the number of available resources - elts and WQEs.
3477 * There are two possible different scenarios:
3478 * - no data inlining into WQEs, one WQEBB may contains up to
3479 * four packets, in this case elts become scarce resource
3480 * - data inlining into WQEs, one packet may require multiple
3481 * WQEBBs, the WQEs become the limiting factor.
3483 MLX5_ASSERT(txq->elts_s >= (uint16_t)(txq->elts_head - txq->elts_tail));
3484 loc.elts_free = txq->elts_s -
3485 (uint16_t)(txq->elts_head - txq->elts_tail);
3486 MLX5_ASSERT(txq->wqe_s >= (uint16_t)(txq->wqe_ci - txq->wqe_pi));
3487 loc.wqe_free = txq->wqe_s -
3488 (uint16_t)(txq->wqe_ci - txq->wqe_pi);
3489 if (unlikely(!loc.elts_free || !loc.wqe_free))
3493 * Fetch the packet from array. Usually this is the first
3494 * packet in series of multi/single segment packets.
3496 loc.mbuf = *(pkts + loc.pkts_sent);
3497 /* Dedicated branch for multi-segment packets. */
3498 if (MLX5_TXOFF_CONFIG(MULTI) &&
3499 unlikely(NB_SEGS(loc.mbuf) > 1)) {
3501 * Multi-segment packet encountered.
3502 * Hardware is able to process it only
3503 * with SEND/TSO opcodes, one packet
3504 * per WQE, do it in dedicated routine.
3507 MLX5_ASSERT(loc.pkts_sent >= loc.pkts_copy);
3508 part = loc.pkts_sent - loc.pkts_copy;
3509 if (!MLX5_TXOFF_CONFIG(INLINE) && part) {
3511 * There are some single-segment mbufs not
3512 * stored in elts. The mbufs must be in the
3513 * same order as WQEs, so we must copy the
3514 * mbufs to elts here, before the coming
3515 * multi-segment packet mbufs is appended.
3517 mlx5_tx_copy_elts(txq, pkts + loc.pkts_copy,
3519 loc.pkts_copy = loc.pkts_sent;
3521 MLX5_ASSERT(pkts_n > loc.pkts_sent);
3522 ret = mlx5_tx_burst_mseg(txq, pkts, pkts_n, &loc, olx);
3523 if (!MLX5_TXOFF_CONFIG(INLINE))
3524 loc.pkts_copy = loc.pkts_sent;
3526 * These returned code checks are supposed
3527 * to be optimized out due to routine inlining.
3529 if (ret == MLX5_TXCMP_CODE_EXIT) {
3531 * The routine returns this code when
3532 * all packets are sent or there is no
3533 * enough resources to complete request.
3537 if (ret == MLX5_TXCMP_CODE_ERROR) {
3539 * The routine returns this code when some error
3540 * in the incoming packets format occurred.
3542 txq->stats.oerrors++;
3545 if (ret == MLX5_TXCMP_CODE_SINGLE) {
3547 * The single-segment packet was encountered
3548 * in the array, try to send it with the
3549 * best optimized way, possible engaging eMPW.
3551 goto enter_send_single;
3553 if (MLX5_TXOFF_CONFIG(TSO) &&
3554 ret == MLX5_TXCMP_CODE_TSO) {
3556 * The single-segment TSO packet was
3557 * encountered in the array.
3559 goto enter_send_tso;
3561 /* We must not get here. Something is going wrong. */
3563 txq->stats.oerrors++;
3566 /* Dedicated branch for single-segment TSO packets. */
3567 if (MLX5_TXOFF_CONFIG(TSO) &&
3568 unlikely(loc.mbuf->ol_flags & RTE_MBUF_F_TX_TCP_SEG)) {
3570 * TSO might require special way for inlining
3571 * (dedicated parameters) and is sent with
3572 * MLX5_OPCODE_TSO opcode only, provide this
3573 * in dedicated branch.
3576 MLX5_ASSERT(NB_SEGS(loc.mbuf) == 1);
3577 MLX5_ASSERT(pkts_n > loc.pkts_sent);
3578 ret = mlx5_tx_burst_tso(txq, pkts, pkts_n, &loc, olx);
3580 * These returned code checks are supposed
3581 * to be optimized out due to routine inlining.
3583 if (ret == MLX5_TXCMP_CODE_EXIT)
3585 if (ret == MLX5_TXCMP_CODE_ERROR) {
3586 txq->stats.oerrors++;
3589 if (ret == MLX5_TXCMP_CODE_SINGLE)
3590 goto enter_send_single;
3591 if (MLX5_TXOFF_CONFIG(MULTI) &&
3592 ret == MLX5_TXCMP_CODE_MULTI) {
3594 * The multi-segment packet was
3595 * encountered in the array.
3597 goto enter_send_multi;
3599 /* We must not get here. Something is going wrong. */
3601 txq->stats.oerrors++;
3605 * The dedicated branch for the single-segment packets
3606 * without TSO. Often these ones can be sent using
3607 * MLX5_OPCODE_EMPW with multiple packets in one WQE.
3608 * The routine builds the WQEs till it encounters
3609 * the TSO or multi-segment packet (in case if these
3610 * offloads are requested at SQ configuration time).
3613 MLX5_ASSERT(pkts_n > loc.pkts_sent);
3614 ret = mlx5_tx_burst_single(txq, pkts, pkts_n, &loc, olx);
3616 * These returned code checks are supposed
3617 * to be optimized out due to routine inlining.
3619 if (ret == MLX5_TXCMP_CODE_EXIT)
3621 if (ret == MLX5_TXCMP_CODE_ERROR) {
3622 txq->stats.oerrors++;
3625 if (MLX5_TXOFF_CONFIG(MULTI) &&
3626 ret == MLX5_TXCMP_CODE_MULTI) {
3628 * The multi-segment packet was
3629 * encountered in the array.
3631 goto enter_send_multi;
3633 if (MLX5_TXOFF_CONFIG(TSO) &&
3634 ret == MLX5_TXCMP_CODE_TSO) {
3636 * The single-segment TSO packet was
3637 * encountered in the array.
3639 goto enter_send_tso;
3641 /* We must not get here. Something is going wrong. */
3643 txq->stats.oerrors++;
3647 * Main Tx loop is completed, do the rest:
3648 * - set completion request if thresholds are reached
3649 * - doorbell the hardware
3650 * - copy the rest of mbufs to elts (if any)
3652 MLX5_ASSERT(MLX5_TXOFF_CONFIG(INLINE) ||
3653 loc.pkts_sent >= loc.pkts_copy);
3654 /* Take a shortcut if nothing is sent. */
3655 if (unlikely(loc.pkts_sent == loc.pkts_loop))
3657 /* Request CQE generation if limits are reached. */
3658 mlx5_tx_request_completion(txq, &loc, olx);
3660 * Ring QP doorbell immediately after WQE building completion
3661 * to improve latencies. The pure software related data treatment
3662 * can be completed after doorbell. Tx CQEs for this SQ are
3663 * processed in this thread only by the polling.
3665 * The rdma core library can map doorbell register in two ways,
3666 * depending on the environment variable "MLX5_SHUT_UP_BF":
3668 * - as regular cached memory, the variable is either missing or
3669 * set to zero. This type of mapping may cause the significant
3670 * doorbell register writing latency and requires explicit memory
3671 * write barrier to mitigate this issue and prevent write combining.
3673 * - as non-cached memory, the variable is present and set to not "0"
3674 * value. This type of mapping may cause performance impact under
3675 * heavy loading conditions but the explicit write memory barrier is
3676 * not required and it may improve core performance.
3678 * - the legacy behaviour (prior 19.08 release) was to use some
3679 * heuristics to decide whether write memory barrier should
3680 * be performed. This behavior is supported with specifying
3681 * tx_db_nc=2, write barrier is skipped if application provides
3682 * the full recommended burst of packets, it supposes the next
3683 * packets are coming and the write barrier will be issued on
3684 * the next burst (after descriptor writing, at least).
3686 mlx5_tx_dbrec_cond_wmb(txq, loc.wqe_last, !txq->db_nc &&
3687 (!txq->db_heu || pkts_n % MLX5_TX_DEFAULT_BURST));
3688 /* Not all of the mbufs may be stored into elts yet. */
3689 part = MLX5_TXOFF_CONFIG(INLINE) ? 0 : loc.pkts_sent - loc.pkts_copy;
3690 if (!MLX5_TXOFF_CONFIG(INLINE) && part) {
3692 * There are some single-segment mbufs not stored in elts.
3693 * It can be only if the last packet was single-segment.
3694 * The copying is gathered into one place due to it is
3695 * a good opportunity to optimize that with SIMD.
3696 * Unfortunately if inlining is enabled the gaps in pointer
3697 * array may happen due to early freeing of the inlined mbufs.
3699 mlx5_tx_copy_elts(txq, pkts + loc.pkts_copy, part, olx);
3700 loc.pkts_copy = loc.pkts_sent;
3702 MLX5_ASSERT(txq->elts_s >= (uint16_t)(txq->elts_head - txq->elts_tail));
3703 MLX5_ASSERT(txq->wqe_s >= (uint16_t)(txq->wqe_ci - txq->wqe_pi));
3704 if (pkts_n > loc.pkts_sent) {
3706 * If burst size is large there might be no enough CQE
3707 * fetched from completion queue and no enough resources
3708 * freed to send all the packets.
3713 #ifdef MLX5_PMD_SOFT_COUNTERS
3714 /* Increment sent packets counter. */
3715 txq->stats.opackets += loc.pkts_sent;
3717 if (MLX5_TXOFF_CONFIG(INLINE) && loc.mbuf_free)
3718 __mlx5_tx_free_mbuf(txq, pkts, loc.mbuf_free, olx);
3719 return loc.pkts_sent;
3722 #endif /* RTE_PMD_MLX5_TX_H_ */