1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2019 Marvell International Ltd.
7 #include "otx2_ethdev.h"
9 #define NIX_XMIT_FC_OR_RETURN(txq, pkts) do { \
10 /* Cached value is low, Update the fc_cache_pkts */ \
11 if (unlikely((txq)->fc_cache_pkts < (pkts))) { \
12 /* Multiply with sqe_per_sqb to express in pkts */ \
13 (txq)->fc_cache_pkts = \
14 ((txq)->nb_sqb_bufs_adj - *(txq)->fc_mem) << \
15 (txq)->sqes_per_sqb_log2; \
16 /* Check it again for the room */ \
17 if (unlikely((txq)->fc_cache_pkts < (pkts))) \
23 static __rte_always_inline uint16_t
24 nix_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
25 uint16_t pkts, uint64_t *cmd, const uint16_t flags)
27 struct otx2_eth_txq *txq = tx_queue; uint16_t i;
28 const rte_iova_t io_addr = txq->io_addr;
29 void *lmt_addr = txq->lmt_addr;
32 NIX_XMIT_FC_OR_RETURN(txq, pkts);
34 otx2_lmt_mov(cmd, &txq->cmd[0], otx2_nix_tx_ext_subs(flags));
36 /* Perform header writes before barrier for TSO */
37 if (flags & NIX_TX_OFFLOAD_TSO_F) {
38 lso_tun_fmt = txq->lso_tun_fmt;
39 for (i = 0; i < pkts; i++)
40 otx2_nix_xmit_prepare_tso(tx_pkts[i], flags);
43 /* Lets commit any changes in the packet here as no further changes
44 * to the packet will be done unless no fast free is enabled.
46 if (!(flags & NIX_TX_OFFLOAD_MBUF_NOFF_F))
49 for (i = 0; i < pkts; i++) {
50 otx2_nix_xmit_prepare(tx_pkts[i], cmd, flags, lso_tun_fmt);
51 /* Passing no of segdw as 4: HDR + EXT + SG + SMEM */
52 otx2_nix_xmit_prepare_tstamp(cmd, &txq->cmd[0],
53 tx_pkts[i]->ol_flags, 4, flags);
54 otx2_nix_xmit_one(cmd, lmt_addr, io_addr, flags);
57 /* Reduce the cached count */
58 txq->fc_cache_pkts -= pkts;
63 static __rte_always_inline uint16_t
64 nix_xmit_pkts_mseg(void *tx_queue, struct rte_mbuf **tx_pkts,
65 uint16_t pkts, uint64_t *cmd, const uint16_t flags)
67 struct otx2_eth_txq *txq = tx_queue; uint64_t i;
68 const rte_iova_t io_addr = txq->io_addr;
69 void *lmt_addr = txq->lmt_addr;
73 NIX_XMIT_FC_OR_RETURN(txq, pkts);
75 otx2_lmt_mov(cmd, &txq->cmd[0], otx2_nix_tx_ext_subs(flags));
77 /* Perform header writes before barrier for TSO */
78 if (flags & NIX_TX_OFFLOAD_TSO_F) {
79 lso_tun_fmt = txq->lso_tun_fmt;
80 for (i = 0; i < pkts; i++)
81 otx2_nix_xmit_prepare_tso(tx_pkts[i], flags);
84 /* Lets commit any changes in the packet here as no further changes
85 * to the packet will be done unless no fast free is enabled.
87 if (!(flags & NIX_TX_OFFLOAD_MBUF_NOFF_F))
90 for (i = 0; i < pkts; i++) {
91 otx2_nix_xmit_prepare(tx_pkts[i], cmd, flags, lso_tun_fmt);
92 segdw = otx2_nix_prepare_mseg(tx_pkts[i], cmd, flags);
93 otx2_nix_xmit_prepare_tstamp(cmd, &txq->cmd[0],
94 tx_pkts[i]->ol_flags, segdw,
96 otx2_nix_xmit_mseg_one(cmd, lmt_addr, io_addr, segdw);
99 /* Reduce the cached count */
100 txq->fc_cache_pkts -= pkts;
105 #if defined(RTE_ARCH_ARM64)
107 #define NIX_DESCS_PER_LOOP 4
108 static __rte_always_inline uint16_t
109 nix_xmit_pkts_vector(void *tx_queue, struct rte_mbuf **tx_pkts,
110 uint16_t pkts, uint64_t *cmd, const uint16_t flags)
112 uint64x2_t dataoff_iova0, dataoff_iova1, dataoff_iova2, dataoff_iova3;
113 uint64x2_t len_olflags0, len_olflags1, len_olflags2, len_olflags3;
114 uint64_t *mbuf0, *mbuf1, *mbuf2, *mbuf3;
115 uint64x2_t senddesc01_w0, senddesc23_w0;
116 uint64x2_t senddesc01_w1, senddesc23_w1;
117 uint64x2_t sgdesc01_w0, sgdesc23_w0;
118 uint64x2_t sgdesc01_w1, sgdesc23_w1;
119 struct otx2_eth_txq *txq = tx_queue;
120 uint64_t *lmt_addr = txq->lmt_addr;
121 rte_iova_t io_addr = txq->io_addr;
122 uint64x2_t ltypes01, ltypes23;
123 uint64x2_t xtmp128, ytmp128;
124 uint64x2_t xmask01, xmask23;
125 uint64x2_t cmd00, cmd01;
126 uint64x2_t cmd10, cmd11;
127 uint64x2_t cmd20, cmd21;
128 uint64x2_t cmd30, cmd31;
129 uint64_t lmt_status, i;
132 NIX_XMIT_FC_OR_RETURN(txq, pkts);
134 pkts_left = pkts & (NIX_DESCS_PER_LOOP - 1);
135 pkts = RTE_ALIGN_FLOOR(pkts, NIX_DESCS_PER_LOOP);
137 /* Reduce the cached count */
138 txq->fc_cache_pkts -= pkts;
140 /* Lets commit any changes in the packet here as no further changes
141 * to the packet will be done unless no fast free is enabled.
143 if (!(flags & NIX_TX_OFFLOAD_MBUF_NOFF_F))
146 senddesc01_w0 = vld1q_dup_u64(&txq->cmd[0]);
147 senddesc23_w0 = senddesc01_w0;
148 senddesc01_w1 = vdupq_n_u64(0);
149 senddesc23_w1 = senddesc01_w1;
150 sgdesc01_w0 = vld1q_dup_u64(&txq->cmd[2]);
151 sgdesc23_w0 = sgdesc01_w0;
153 for (i = 0; i < pkts; i += NIX_DESCS_PER_LOOP) {
154 /* Clear lower 32bit of SEND_HDR_W0 and SEND_SG_W0 */
155 senddesc01_w0 = vbicq_u64(senddesc01_w0,
156 vdupq_n_u64(0xFFFFFFFF));
157 sgdesc01_w0 = vbicq_u64(sgdesc01_w0,
158 vdupq_n_u64(0xFFFFFFFF));
160 senddesc23_w0 = senddesc01_w0;
161 sgdesc23_w0 = sgdesc01_w0;
163 /* Move mbufs to iova */
164 mbuf0 = (uint64_t *)tx_pkts[0];
165 mbuf1 = (uint64_t *)tx_pkts[1];
166 mbuf2 = (uint64_t *)tx_pkts[2];
167 mbuf3 = (uint64_t *)tx_pkts[3];
169 mbuf0 = (uint64_t *)((uintptr_t)mbuf0 +
170 offsetof(struct rte_mbuf, buf_iova));
171 mbuf1 = (uint64_t *)((uintptr_t)mbuf1 +
172 offsetof(struct rte_mbuf, buf_iova));
173 mbuf2 = (uint64_t *)((uintptr_t)mbuf2 +
174 offsetof(struct rte_mbuf, buf_iova));
175 mbuf3 = (uint64_t *)((uintptr_t)mbuf3 +
176 offsetof(struct rte_mbuf, buf_iova));
178 * Get mbuf's, olflags, iova, pktlen, dataoff
179 * dataoff_iovaX.D[0] = iova,
180 * dataoff_iovaX.D[1](15:0) = mbuf->dataoff
181 * len_olflagsX.D[0] = ol_flags,
182 * len_olflagsX.D[1](63:32) = mbuf->pkt_len
184 dataoff_iova0 = vld1q_u64(mbuf0);
185 len_olflags0 = vld1q_u64(mbuf0 + 2);
186 dataoff_iova1 = vld1q_u64(mbuf1);
187 len_olflags1 = vld1q_u64(mbuf1 + 2);
188 dataoff_iova2 = vld1q_u64(mbuf2);
189 len_olflags2 = vld1q_u64(mbuf2 + 2);
190 dataoff_iova3 = vld1q_u64(mbuf3);
191 len_olflags3 = vld1q_u64(mbuf3 + 2);
193 if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) {
194 struct rte_mbuf *mbuf;
195 /* Set don't free bit if reference count > 1 */
196 xmask01 = vdupq_n_u64(0);
199 mbuf = (struct rte_mbuf *)((uintptr_t)mbuf0 -
200 offsetof(struct rte_mbuf, buf_iova));
202 if (otx2_nix_prefree_seg(mbuf))
203 vsetq_lane_u64(0x80000, xmask01, 0);
205 RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool,
209 mbuf = (struct rte_mbuf *)((uintptr_t)mbuf1 -
210 offsetof(struct rte_mbuf, buf_iova));
211 if (otx2_nix_prefree_seg(mbuf))
212 vsetq_lane_u64(0x80000, xmask01, 1);
214 RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool,
218 mbuf = (struct rte_mbuf *)((uintptr_t)mbuf2 -
219 offsetof(struct rte_mbuf, buf_iova));
220 if (otx2_nix_prefree_seg(mbuf))
221 vsetq_lane_u64(0x80000, xmask23, 0);
223 RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool,
227 mbuf = (struct rte_mbuf *)((uintptr_t)mbuf3 -
228 offsetof(struct rte_mbuf, buf_iova));
229 if (otx2_nix_prefree_seg(mbuf))
230 vsetq_lane_u64(0x80000, xmask23, 1);
232 RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool,
235 senddesc01_w0 = vorrq_u64(senddesc01_w0, xmask01);
236 senddesc23_w0 = vorrq_u64(senddesc23_w0, xmask23);
237 /* Ensuring mbuf fields which got updated in
238 * otx2_nix_prefree_seg are written before LMTST.
242 struct rte_mbuf *mbuf;
243 /* Mark mempool object as "put" since
246 mbuf = (struct rte_mbuf *)((uintptr_t)mbuf0 -
247 offsetof(struct rte_mbuf, buf_iova));
248 RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf,
251 mbuf = (struct rte_mbuf *)((uintptr_t)mbuf1 -
252 offsetof(struct rte_mbuf, buf_iova));
253 RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf,
256 mbuf = (struct rte_mbuf *)((uintptr_t)mbuf2 -
257 offsetof(struct rte_mbuf, buf_iova));
258 RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf,
261 mbuf = (struct rte_mbuf *)((uintptr_t)mbuf3 -
262 offsetof(struct rte_mbuf, buf_iova));
263 RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf,
268 /* Move mbufs to point pool */
269 mbuf0 = (uint64_t *)((uintptr_t)mbuf0 +
270 offsetof(struct rte_mbuf, pool) -
271 offsetof(struct rte_mbuf, buf_iova));
272 mbuf1 = (uint64_t *)((uintptr_t)mbuf1 +
273 offsetof(struct rte_mbuf, pool) -
274 offsetof(struct rte_mbuf, buf_iova));
275 mbuf2 = (uint64_t *)((uintptr_t)mbuf2 +
276 offsetof(struct rte_mbuf, pool) -
277 offsetof(struct rte_mbuf, buf_iova));
278 mbuf3 = (uint64_t *)((uintptr_t)mbuf3 +
279 offsetof(struct rte_mbuf, pool) -
280 offsetof(struct rte_mbuf, buf_iova));
283 (NIX_TX_OFFLOAD_OL3_OL4_CSUM_F |
284 NIX_TX_OFFLOAD_L3_L4_CSUM_F)) {
285 /* Get tx_offload for ol2, ol3, l2, l3 lengths */
287 * E(8):OL2_LEN(7):OL3_LEN(9):E(24):L3_LEN(9):L2_LEN(7)
288 * E(8):OL2_LEN(7):OL3_LEN(9):E(24):L3_LEN(9):L2_LEN(7)
291 asm volatile ("LD1 {%[a].D}[0],[%[in]]\n\t" :
292 [a]"+w"(senddesc01_w1) :
293 [in]"r"(mbuf0 + 2) : "memory");
295 asm volatile ("LD1 {%[a].D}[1],[%[in]]\n\t" :
296 [a]"+w"(senddesc01_w1) :
297 [in]"r"(mbuf1 + 2) : "memory");
299 asm volatile ("LD1 {%[b].D}[0],[%[in]]\n\t" :
300 [b]"+w"(senddesc23_w1) :
301 [in]"r"(mbuf2 + 2) : "memory");
303 asm volatile ("LD1 {%[b].D}[1],[%[in]]\n\t" :
304 [b]"+w"(senddesc23_w1) :
305 [in]"r"(mbuf3 + 2) : "memory");
307 /* Get pool pointer alone */
308 mbuf0 = (uint64_t *)*mbuf0;
309 mbuf1 = (uint64_t *)*mbuf1;
310 mbuf2 = (uint64_t *)*mbuf2;
311 mbuf3 = (uint64_t *)*mbuf3;
313 /* Get pool pointer alone */
314 mbuf0 = (uint64_t *)*mbuf0;
315 mbuf1 = (uint64_t *)*mbuf1;
316 mbuf2 = (uint64_t *)*mbuf2;
317 mbuf3 = (uint64_t *)*mbuf3;
320 const uint8x16_t shuf_mask2 = {
321 0x4, 0x5, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
322 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
324 xtmp128 = vzip2q_u64(len_olflags0, len_olflags1);
325 ytmp128 = vzip2q_u64(len_olflags2, len_olflags3);
327 /* Clear dataoff_iovaX.D[1] bits other than dataoff(15:0) */
328 const uint64x2_t and_mask0 = {
333 dataoff_iova0 = vandq_u64(dataoff_iova0, and_mask0);
334 dataoff_iova1 = vandq_u64(dataoff_iova1, and_mask0);
335 dataoff_iova2 = vandq_u64(dataoff_iova2, and_mask0);
336 dataoff_iova3 = vandq_u64(dataoff_iova3, and_mask0);
339 * Pick only 16 bits of pktlen preset at bits 63:32
340 * and place them at bits 15:0.
342 xtmp128 = vqtbl1q_u8(xtmp128, shuf_mask2);
343 ytmp128 = vqtbl1q_u8(ytmp128, shuf_mask2);
345 /* Add pairwise to get dataoff + iova in sgdesc_w1 */
346 sgdesc01_w1 = vpaddq_u64(dataoff_iova0, dataoff_iova1);
347 sgdesc23_w1 = vpaddq_u64(dataoff_iova2, dataoff_iova3);
349 /* Orr both sgdesc_w0 and senddesc_w0 with 16 bits of
350 * pktlen at 15:0 position.
352 sgdesc01_w0 = vorrq_u64(sgdesc01_w0, xtmp128);
353 sgdesc23_w0 = vorrq_u64(sgdesc23_w0, ytmp128);
354 senddesc01_w0 = vorrq_u64(senddesc01_w0, xtmp128);
355 senddesc23_w0 = vorrq_u64(senddesc23_w0, ytmp128);
357 if ((flags & NIX_TX_OFFLOAD_L3_L4_CSUM_F) &&
358 !(flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F)) {
360 * Lookup table to translate ol_flags to
361 * il3/il4 types. But we still use ol3/ol4 types in
362 * senddesc_w1 as only one header processing is enabled.
364 const uint8x16_t tbl = {
365 /* [0-15] = il4type:il3type */
366 0x04, /* none (IPv6 assumed) */
367 0x14, /* PKT_TX_TCP_CKSUM (IPv6 assumed) */
368 0x24, /* PKT_TX_SCTP_CKSUM (IPv6 assumed) */
369 0x34, /* PKT_TX_UDP_CKSUM (IPv6 assumed) */
370 0x03, /* PKT_TX_IP_CKSUM */
371 0x13, /* PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM */
372 0x23, /* PKT_TX_IP_CKSUM | PKT_TX_SCTP_CKSUM */
373 0x33, /* PKT_TX_IP_CKSUM | PKT_TX_UDP_CKSUM */
374 0x02, /* PKT_TX_IPV4 */
375 0x12, /* PKT_TX_IPV4 | PKT_TX_TCP_CKSUM */
376 0x22, /* PKT_TX_IPV4 | PKT_TX_SCTP_CKSUM */
377 0x32, /* PKT_TX_IPV4 | PKT_TX_UDP_CKSUM */
378 0x03, /* PKT_TX_IPV4 | PKT_TX_IP_CKSUM */
379 0x13, /* PKT_TX_IPV4 | PKT_TX_IP_CKSUM |
382 0x23, /* PKT_TX_IPV4 | PKT_TX_IP_CKSUM |
385 0x33, /* PKT_TX_IPV4 | PKT_TX_IP_CKSUM |
390 /* Extract olflags to translate to iltypes */
391 xtmp128 = vzip1q_u64(len_olflags0, len_olflags1);
392 ytmp128 = vzip1q_u64(len_olflags2, len_olflags3);
395 * E(47):L3_LEN(9):L2_LEN(7+z)
396 * E(47):L3_LEN(9):L2_LEN(7+z)
398 senddesc01_w1 = vshlq_n_u64(senddesc01_w1, 1);
399 senddesc23_w1 = vshlq_n_u64(senddesc23_w1, 1);
401 /* Move OLFLAGS bits 55:52 to 51:48
402 * with zeros preprended on the byte and rest
405 xtmp128 = vshrq_n_u8(xtmp128, 4);
406 ytmp128 = vshrq_n_u8(ytmp128, 4);
408 * E(48):L3_LEN(8):L2_LEN(z+7)
409 * E(48):L3_LEN(8):L2_LEN(z+7)
411 const int8x16_t tshft3 = {
412 -1, 0, 8, 8, 8, 8, 8, 8,
413 -1, 0, 8, 8, 8, 8, 8, 8,
416 senddesc01_w1 = vshlq_u8(senddesc01_w1, tshft3);
417 senddesc23_w1 = vshlq_u8(senddesc23_w1, tshft3);
420 ltypes01 = vqtbl1q_u8(tbl, xtmp128);
421 ltypes23 = vqtbl1q_u8(tbl, ytmp128);
423 /* Just use ld1q to retrieve aura
424 * when we don't need tx_offload
426 mbuf0 = (uint64_t *)((uintptr_t)mbuf0 +
427 offsetof(struct rte_mempool, pool_id));
428 mbuf1 = (uint64_t *)((uintptr_t)mbuf1 +
429 offsetof(struct rte_mempool, pool_id));
430 mbuf2 = (uint64_t *)((uintptr_t)mbuf2 +
431 offsetof(struct rte_mempool, pool_id));
432 mbuf3 = (uint64_t *)((uintptr_t)mbuf3 +
433 offsetof(struct rte_mempool, pool_id));
435 /* Pick only relevant fields i.e Bit 48:55 of iltype
436 * and place it in ol3/ol4type of senddesc_w1
438 const uint8x16_t shuf_mask0 = {
439 0xFF, 0xFF, 0xFF, 0xFF, 0x6, 0xFF, 0xFF, 0xFF,
440 0xFF, 0xFF, 0xFF, 0xFF, 0xE, 0xFF, 0xFF, 0xFF,
443 ltypes01 = vqtbl1q_u8(ltypes01, shuf_mask0);
444 ltypes23 = vqtbl1q_u8(ltypes23, shuf_mask0);
446 /* Prepare ol4ptr, ol3ptr from ol3len, ol2len.
447 * a [E(32):E(16):OL3(8):OL2(8)]
449 * a [E(32):E(16):(OL3+OL2):OL2]
450 * => E(32):E(16)::OL4PTR(8):OL3PTR(8)
452 senddesc01_w1 = vaddq_u8(senddesc01_w1,
453 vshlq_n_u16(senddesc01_w1, 8));
454 senddesc23_w1 = vaddq_u8(senddesc23_w1,
455 vshlq_n_u16(senddesc23_w1, 8));
457 /* Create first half of 4W cmd for 4 mbufs (sgdesc) */
458 cmd01 = vzip1q_u64(sgdesc01_w0, sgdesc01_w1);
459 cmd11 = vzip2q_u64(sgdesc01_w0, sgdesc01_w1);
460 cmd21 = vzip1q_u64(sgdesc23_w0, sgdesc23_w1);
461 cmd31 = vzip2q_u64(sgdesc23_w0, sgdesc23_w1);
463 xmask01 = vdupq_n_u64(0);
465 asm volatile ("LD1 {%[a].H}[0],[%[in]]\n\t" :
466 [a]"+w"(xmask01) : [in]"r"(mbuf0) : "memory");
468 asm volatile ("LD1 {%[a].H}[4],[%[in]]\n\t" :
469 [a]"+w"(xmask01) : [in]"r"(mbuf1) : "memory");
471 asm volatile ("LD1 {%[b].H}[0],[%[in]]\n\t" :
472 [b]"+w"(xmask23) : [in]"r"(mbuf2) : "memory");
474 asm volatile ("LD1 {%[b].H}[4],[%[in]]\n\t" :
475 [b]"+w"(xmask23) : [in]"r"(mbuf3) : "memory");
476 xmask01 = vshlq_n_u64(xmask01, 20);
477 xmask23 = vshlq_n_u64(xmask23, 20);
479 senddesc01_w0 = vorrq_u64(senddesc01_w0, xmask01);
480 senddesc23_w0 = vorrq_u64(senddesc23_w0, xmask23);
481 /* Move ltypes to senddesc*_w1 */
482 senddesc01_w1 = vorrq_u64(senddesc01_w1, ltypes01);
483 senddesc23_w1 = vorrq_u64(senddesc23_w1, ltypes23);
485 /* Create first half of 4W cmd for 4 mbufs (sendhdr) */
486 cmd00 = vzip1q_u64(senddesc01_w0, senddesc01_w1);
487 cmd10 = vzip2q_u64(senddesc01_w0, senddesc01_w1);
488 cmd20 = vzip1q_u64(senddesc23_w0, senddesc23_w1);
489 cmd30 = vzip2q_u64(senddesc23_w0, senddesc23_w1);
491 } else if (!(flags & NIX_TX_OFFLOAD_L3_L4_CSUM_F) &&
492 (flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F)) {
494 * Lookup table to translate ol_flags to
498 const uint8x16_t tbl = {
499 /* [0-15] = ol4type:ol3type */
501 0x03, /* OUTER_IP_CKSUM */
502 0x02, /* OUTER_IPV4 */
503 0x03, /* OUTER_IPV4 | OUTER_IP_CKSUM */
504 0x04, /* OUTER_IPV6 */
505 0x00, /* OUTER_IPV6 | OUTER_IP_CKSUM */
506 0x00, /* OUTER_IPV6 | OUTER_IPV4 */
507 0x00, /* OUTER_IPV6 | OUTER_IPV4 |
510 0x00, /* OUTER_UDP_CKSUM */
511 0x33, /* OUTER_UDP_CKSUM | OUTER_IP_CKSUM */
512 0x32, /* OUTER_UDP_CKSUM | OUTER_IPV4 */
513 0x33, /* OUTER_UDP_CKSUM | OUTER_IPV4 |
516 0x34, /* OUTER_UDP_CKSUM | OUTER_IPV6 */
517 0x00, /* OUTER_UDP_CKSUM | OUTER_IPV6 |
520 0x00, /* OUTER_UDP_CKSUM | OUTER_IPV6 |
523 0x00, /* OUTER_UDP_CKSUM | OUTER_IPV6 |
524 * OUTER_IPV4 | OUTER_IP_CKSUM
528 /* Extract olflags to translate to iltypes */
529 xtmp128 = vzip1q_u64(len_olflags0, len_olflags1);
530 ytmp128 = vzip1q_u64(len_olflags2, len_olflags3);
533 * E(47):OL3_LEN(9):OL2_LEN(7+z)
534 * E(47):OL3_LEN(9):OL2_LEN(7+z)
536 const uint8x16_t shuf_mask5 = {
537 0x6, 0x5, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
538 0xE, 0xD, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
540 senddesc01_w1 = vqtbl1q_u8(senddesc01_w1, shuf_mask5);
541 senddesc23_w1 = vqtbl1q_u8(senddesc23_w1, shuf_mask5);
543 /* Extract outer ol flags only */
544 const uint64x2_t o_cksum_mask = {
549 xtmp128 = vandq_u64(xtmp128, o_cksum_mask);
550 ytmp128 = vandq_u64(ytmp128, o_cksum_mask);
552 /* Extract OUTER_UDP_CKSUM bit 41 and
556 xtmp128 = xtmp128 | vshlq_n_u64(xtmp128, 20);
557 ytmp128 = ytmp128 | vshlq_n_u64(ytmp128, 20);
559 /* Shift oltype by 2 to start nibble from BIT(56)
562 xtmp128 = vshrq_n_u8(xtmp128, 2);
563 ytmp128 = vshrq_n_u8(ytmp128, 2);
565 * E(48):L3_LEN(8):L2_LEN(z+7)
566 * E(48):L3_LEN(8):L2_LEN(z+7)
568 const int8x16_t tshft3 = {
569 -1, 0, 8, 8, 8, 8, 8, 8,
570 -1, 0, 8, 8, 8, 8, 8, 8,
573 senddesc01_w1 = vshlq_u8(senddesc01_w1, tshft3);
574 senddesc23_w1 = vshlq_u8(senddesc23_w1, tshft3);
577 ltypes01 = vqtbl1q_u8(tbl, xtmp128);
578 ltypes23 = vqtbl1q_u8(tbl, ytmp128);
580 /* Just use ld1q to retrieve aura
581 * when we don't need tx_offload
583 mbuf0 = (uint64_t *)((uintptr_t)mbuf0 +
584 offsetof(struct rte_mempool, pool_id));
585 mbuf1 = (uint64_t *)((uintptr_t)mbuf1 +
586 offsetof(struct rte_mempool, pool_id));
587 mbuf2 = (uint64_t *)((uintptr_t)mbuf2 +
588 offsetof(struct rte_mempool, pool_id));
589 mbuf3 = (uint64_t *)((uintptr_t)mbuf3 +
590 offsetof(struct rte_mempool, pool_id));
592 /* Pick only relevant fields i.e Bit 56:63 of oltype
593 * and place it in ol3/ol4type of senddesc_w1
595 const uint8x16_t shuf_mask0 = {
596 0xFF, 0xFF, 0xFF, 0xFF, 0x7, 0xFF, 0xFF, 0xFF,
597 0xFF, 0xFF, 0xFF, 0xFF, 0xF, 0xFF, 0xFF, 0xFF,
600 ltypes01 = vqtbl1q_u8(ltypes01, shuf_mask0);
601 ltypes23 = vqtbl1q_u8(ltypes23, shuf_mask0);
603 /* Prepare ol4ptr, ol3ptr from ol3len, ol2len.
604 * a [E(32):E(16):OL3(8):OL2(8)]
606 * a [E(32):E(16):(OL3+OL2):OL2]
607 * => E(32):E(16)::OL4PTR(8):OL3PTR(8)
609 senddesc01_w1 = vaddq_u8(senddesc01_w1,
610 vshlq_n_u16(senddesc01_w1, 8));
611 senddesc23_w1 = vaddq_u8(senddesc23_w1,
612 vshlq_n_u16(senddesc23_w1, 8));
614 /* Create second half of 4W cmd for 4 mbufs (sgdesc) */
615 cmd01 = vzip1q_u64(sgdesc01_w0, sgdesc01_w1);
616 cmd11 = vzip2q_u64(sgdesc01_w0, sgdesc01_w1);
617 cmd21 = vzip1q_u64(sgdesc23_w0, sgdesc23_w1);
618 cmd31 = vzip2q_u64(sgdesc23_w0, sgdesc23_w1);
620 xmask01 = vdupq_n_u64(0);
622 asm volatile ("LD1 {%[a].H}[0],[%[in]]\n\t" :
623 [a]"+w"(xmask01) : [in]"r"(mbuf0) : "memory");
625 asm volatile ("LD1 {%[a].H}[4],[%[in]]\n\t" :
626 [a]"+w"(xmask01) : [in]"r"(mbuf1) : "memory");
628 asm volatile ("LD1 {%[b].H}[0],[%[in]]\n\t" :
629 [b]"+w"(xmask23) : [in]"r"(mbuf2) : "memory");
631 asm volatile ("LD1 {%[b].H}[4],[%[in]]\n\t" :
632 [b]"+w"(xmask23) : [in]"r"(mbuf3) : "memory");
633 xmask01 = vshlq_n_u64(xmask01, 20);
634 xmask23 = vshlq_n_u64(xmask23, 20);
636 senddesc01_w0 = vorrq_u64(senddesc01_w0, xmask01);
637 senddesc23_w0 = vorrq_u64(senddesc23_w0, xmask23);
638 /* Move ltypes to senddesc*_w1 */
639 senddesc01_w1 = vorrq_u64(senddesc01_w1, ltypes01);
640 senddesc23_w1 = vorrq_u64(senddesc23_w1, ltypes23);
642 /* Create first half of 4W cmd for 4 mbufs (sendhdr) */
643 cmd00 = vzip1q_u64(senddesc01_w0, senddesc01_w1);
644 cmd10 = vzip2q_u64(senddesc01_w0, senddesc01_w1);
645 cmd20 = vzip1q_u64(senddesc23_w0, senddesc23_w1);
646 cmd30 = vzip2q_u64(senddesc23_w0, senddesc23_w1);
648 } else if ((flags & NIX_TX_OFFLOAD_L3_L4_CSUM_F) &&
649 (flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F)) {
650 /* Lookup table to translate ol_flags to
651 * ol4type, ol3type, il4type, il3type of senddesc_w1
653 const uint8x16x2_t tbl = {
656 /* [0-15] = il4type:il3type */
657 0x04, /* none (IPv6) */
658 0x14, /* PKT_TX_TCP_CKSUM (IPv6) */
659 0x24, /* PKT_TX_SCTP_CKSUM (IPv6) */
660 0x34, /* PKT_TX_UDP_CKSUM (IPv6) */
661 0x03, /* PKT_TX_IP_CKSUM */
662 0x13, /* PKT_TX_IP_CKSUM |
665 0x23, /* PKT_TX_IP_CKSUM |
668 0x33, /* PKT_TX_IP_CKSUM |
671 0x02, /* PKT_TX_IPV4 */
672 0x12, /* PKT_TX_IPV4 |
675 0x22, /* PKT_TX_IPV4 |
678 0x32, /* PKT_TX_IPV4 |
681 0x03, /* PKT_TX_IPV4 |
684 0x13, /* PKT_TX_IPV4 | PKT_TX_IP_CKSUM |
687 0x23, /* PKT_TX_IPV4 | PKT_TX_IP_CKSUM |
690 0x33, /* PKT_TX_IPV4 | PKT_TX_IP_CKSUM |
696 /* [16-31] = ol4type:ol3type */
698 0x03, /* OUTER_IP_CKSUM */
699 0x02, /* OUTER_IPV4 */
700 0x03, /* OUTER_IPV4 | OUTER_IP_CKSUM */
701 0x04, /* OUTER_IPV6 */
702 0x00, /* OUTER_IPV6 | OUTER_IP_CKSUM */
703 0x00, /* OUTER_IPV6 | OUTER_IPV4 */
704 0x00, /* OUTER_IPV6 | OUTER_IPV4 |
707 0x00, /* OUTER_UDP_CKSUM */
708 0x33, /* OUTER_UDP_CKSUM |
711 0x32, /* OUTER_UDP_CKSUM |
714 0x33, /* OUTER_UDP_CKSUM |
715 * OUTER_IPV4 | OUTER_IP_CKSUM
717 0x34, /* OUTER_UDP_CKSUM |
720 0x00, /* OUTER_UDP_CKSUM | OUTER_IPV6 |
723 0x00, /* OUTER_UDP_CKSUM | OUTER_IPV6 |
726 0x00, /* OUTER_UDP_CKSUM | OUTER_IPV6 |
727 * OUTER_IPV4 | OUTER_IP_CKSUM
733 /* Extract olflags to translate to oltype & iltype */
734 xtmp128 = vzip1q_u64(len_olflags0, len_olflags1);
735 ytmp128 = vzip1q_u64(len_olflags2, len_olflags3);
738 * E(8):OL2_LN(7):OL3_LN(9):E(23):L3_LN(9):L2_LN(7+z)
739 * E(8):OL2_LN(7):OL3_LN(9):E(23):L3_LN(9):L2_LN(7+z)
741 const uint32x4_t tshft_4 = {
745 senddesc01_w1 = vshlq_u32(senddesc01_w1, tshft_4);
746 senddesc23_w1 = vshlq_u32(senddesc23_w1, tshft_4);
749 * E(32):L3_LEN(8):L2_LEN(7+Z):OL3_LEN(8):OL2_LEN(7+Z)
750 * E(32):L3_LEN(8):L2_LEN(7+Z):OL3_LEN(8):OL2_LEN(7+Z)
752 const uint8x16_t shuf_mask5 = {
753 0x6, 0x5, 0x0, 0x1, 0xFF, 0xFF, 0xFF, 0xFF,
754 0xE, 0xD, 0x8, 0x9, 0xFF, 0xFF, 0xFF, 0xFF,
756 senddesc01_w1 = vqtbl1q_u8(senddesc01_w1, shuf_mask5);
757 senddesc23_w1 = vqtbl1q_u8(senddesc23_w1, shuf_mask5);
759 /* Extract outer and inner header ol_flags */
760 const uint64x2_t oi_cksum_mask = {
765 xtmp128 = vandq_u64(xtmp128, oi_cksum_mask);
766 ytmp128 = vandq_u64(ytmp128, oi_cksum_mask);
768 /* Extract OUTER_UDP_CKSUM bit 41 and
772 xtmp128 = xtmp128 | vshlq_n_u64(xtmp128, 20);
773 ytmp128 = ytmp128 | vshlq_n_u64(ytmp128, 20);
775 /* Shift right oltype by 2 and iltype by 4
776 * to start oltype nibble from BIT(58)
777 * instead of BIT(56) and iltype nibble from BIT(48)
778 * instead of BIT(52).
780 const int8x16_t tshft5 = {
781 8, 8, 8, 8, 8, 8, -4, -2,
782 8, 8, 8, 8, 8, 8, -4, -2,
785 xtmp128 = vshlq_u8(xtmp128, tshft5);
786 ytmp128 = vshlq_u8(ytmp128, tshft5);
788 * E(32):L3_LEN(8):L2_LEN(8):OL3_LEN(8):OL2_LEN(8)
789 * E(32):L3_LEN(8):L2_LEN(8):OL3_LEN(8):OL2_LEN(8)
791 const int8x16_t tshft3 = {
792 -1, 0, -1, 0, 0, 0, 0, 0,
793 -1, 0, -1, 0, 0, 0, 0, 0,
796 senddesc01_w1 = vshlq_u8(senddesc01_w1, tshft3);
797 senddesc23_w1 = vshlq_u8(senddesc23_w1, tshft3);
799 /* Mark Bit(4) of oltype */
800 const uint64x2_t oi_cksum_mask2 = {
805 xtmp128 = vorrq_u64(xtmp128, oi_cksum_mask2);
806 ytmp128 = vorrq_u64(ytmp128, oi_cksum_mask2);
809 ltypes01 = vqtbl2q_u8(tbl, xtmp128);
810 ltypes23 = vqtbl2q_u8(tbl, ytmp128);
812 /* Just use ld1q to retrieve aura
813 * when we don't need tx_offload
815 mbuf0 = (uint64_t *)((uintptr_t)mbuf0 +
816 offsetof(struct rte_mempool, pool_id));
817 mbuf1 = (uint64_t *)((uintptr_t)mbuf1 +
818 offsetof(struct rte_mempool, pool_id));
819 mbuf2 = (uint64_t *)((uintptr_t)mbuf2 +
820 offsetof(struct rte_mempool, pool_id));
821 mbuf3 = (uint64_t *)((uintptr_t)mbuf3 +
822 offsetof(struct rte_mempool, pool_id));
824 /* Pick only relevant fields i.e Bit 48:55 of iltype and
825 * Bit 56:63 of oltype and place it in corresponding
826 * place in senddesc_w1.
828 const uint8x16_t shuf_mask0 = {
829 0xFF, 0xFF, 0xFF, 0xFF, 0x7, 0x6, 0xFF, 0xFF,
830 0xFF, 0xFF, 0xFF, 0xFF, 0xF, 0xE, 0xFF, 0xFF,
833 ltypes01 = vqtbl1q_u8(ltypes01, shuf_mask0);
834 ltypes23 = vqtbl1q_u8(ltypes23, shuf_mask0);
836 /* Prepare l4ptr, l3ptr, ol4ptr, ol3ptr from
837 * l3len, l2len, ol3len, ol2len.
838 * a [E(32):L3(8):L2(8):OL3(8):OL2(8)]
840 * a [E:(L3+L2):(L2+OL3):(OL3+OL2):OL2]
842 * a [E:(L3+L2+OL3+OL2):(L2+OL3+OL2):(OL3+OL2):OL2]
843 * => E(32):IL4PTR(8):IL3PTR(8):OL4PTR(8):OL3PTR(8)
845 senddesc01_w1 = vaddq_u8(senddesc01_w1,
846 vshlq_n_u32(senddesc01_w1, 8));
847 senddesc23_w1 = vaddq_u8(senddesc23_w1,
848 vshlq_n_u32(senddesc23_w1, 8));
850 /* Create second half of 4W cmd for 4 mbufs (sgdesc) */
851 cmd01 = vzip1q_u64(sgdesc01_w0, sgdesc01_w1);
852 cmd11 = vzip2q_u64(sgdesc01_w0, sgdesc01_w1);
853 cmd21 = vzip1q_u64(sgdesc23_w0, sgdesc23_w1);
854 cmd31 = vzip2q_u64(sgdesc23_w0, sgdesc23_w1);
856 /* Continue preparing l4ptr, l3ptr, ol4ptr, ol3ptr */
857 senddesc01_w1 = vaddq_u8(senddesc01_w1,
858 vshlq_n_u32(senddesc01_w1, 16));
859 senddesc23_w1 = vaddq_u8(senddesc23_w1,
860 vshlq_n_u32(senddesc23_w1, 16));
862 xmask01 = vdupq_n_u64(0);
864 asm volatile ("LD1 {%[a].H}[0],[%[in]]\n\t" :
865 [a]"+w"(xmask01) : [in]"r"(mbuf0) : "memory");
867 asm volatile ("LD1 {%[a].H}[4],[%[in]]\n\t" :
868 [a]"+w"(xmask01) : [in]"r"(mbuf1) : "memory");
870 asm volatile ("LD1 {%[b].H}[0],[%[in]]\n\t" :
871 [b]"+w"(xmask23) : [in]"r"(mbuf2) : "memory");
873 asm volatile ("LD1 {%[b].H}[4],[%[in]]\n\t" :
874 [b]"+w"(xmask23) : [in]"r"(mbuf3) : "memory");
875 xmask01 = vshlq_n_u64(xmask01, 20);
876 xmask23 = vshlq_n_u64(xmask23, 20);
878 senddesc01_w0 = vorrq_u64(senddesc01_w0, xmask01);
879 senddesc23_w0 = vorrq_u64(senddesc23_w0, xmask23);
880 /* Move ltypes to senddesc*_w1 */
881 senddesc01_w1 = vorrq_u64(senddesc01_w1, ltypes01);
882 senddesc23_w1 = vorrq_u64(senddesc23_w1, ltypes23);
884 /* Create first half of 4W cmd for 4 mbufs (sendhdr) */
885 cmd00 = vzip1q_u64(senddesc01_w0, senddesc01_w1);
886 cmd10 = vzip2q_u64(senddesc01_w0, senddesc01_w1);
887 cmd20 = vzip1q_u64(senddesc23_w0, senddesc23_w1);
888 cmd30 = vzip2q_u64(senddesc23_w0, senddesc23_w1);
890 /* Just use ld1q to retrieve aura
891 * when we don't need tx_offload
893 mbuf0 = (uint64_t *)((uintptr_t)mbuf0 +
894 offsetof(struct rte_mempool, pool_id));
895 mbuf1 = (uint64_t *)((uintptr_t)mbuf1 +
896 offsetof(struct rte_mempool, pool_id));
897 mbuf2 = (uint64_t *)((uintptr_t)mbuf2 +
898 offsetof(struct rte_mempool, pool_id));
899 mbuf3 = (uint64_t *)((uintptr_t)mbuf3 +
900 offsetof(struct rte_mempool, pool_id));
901 xmask01 = vdupq_n_u64(0);
903 asm volatile ("LD1 {%[a].H}[0],[%[in]]\n\t" :
904 [a]"+w"(xmask01) : [in]"r"(mbuf0) : "memory");
906 asm volatile ("LD1 {%[a].H}[4],[%[in]]\n\t" :
907 [a]"+w"(xmask01) : [in]"r"(mbuf1) : "memory");
909 asm volatile ("LD1 {%[b].H}[0],[%[in]]\n\t" :
910 [b]"+w"(xmask23) : [in]"r"(mbuf2) : "memory");
912 asm volatile ("LD1 {%[b].H}[4],[%[in]]\n\t" :
913 [b]"+w"(xmask23) : [in]"r"(mbuf3) : "memory");
914 xmask01 = vshlq_n_u64(xmask01, 20);
915 xmask23 = vshlq_n_u64(xmask23, 20);
917 senddesc01_w0 = vorrq_u64(senddesc01_w0, xmask01);
918 senddesc23_w0 = vorrq_u64(senddesc23_w0, xmask23);
920 /* Create 4W cmd for 4 mbufs (sendhdr, sgdesc) */
921 cmd00 = vzip1q_u64(senddesc01_w0, senddesc01_w1);
922 cmd01 = vzip1q_u64(sgdesc01_w0, sgdesc01_w1);
923 cmd10 = vzip2q_u64(senddesc01_w0, senddesc01_w1);
924 cmd11 = vzip2q_u64(sgdesc01_w0, sgdesc01_w1);
925 cmd20 = vzip1q_u64(senddesc23_w0, senddesc23_w1);
926 cmd21 = vzip1q_u64(sgdesc23_w0, sgdesc23_w1);
927 cmd30 = vzip2q_u64(senddesc23_w0, senddesc23_w1);
928 cmd31 = vzip2q_u64(sgdesc23_w0, sgdesc23_w1);
932 vst1q_u64(lmt_addr, cmd00);
933 vst1q_u64(lmt_addr + 2, cmd01);
934 vst1q_u64(lmt_addr + 4, cmd10);
935 vst1q_u64(lmt_addr + 6, cmd11);
936 vst1q_u64(lmt_addr + 8, cmd20);
937 vst1q_u64(lmt_addr + 10, cmd21);
938 vst1q_u64(lmt_addr + 12, cmd30);
939 vst1q_u64(lmt_addr + 14, cmd31);
940 lmt_status = otx2_lmt_submit(io_addr);
942 } while (lmt_status == 0);
943 tx_pkts = tx_pkts + NIX_DESCS_PER_LOOP;
946 if (unlikely(pkts_left))
947 pkts += nix_xmit_pkts(tx_queue, tx_pkts, pkts_left, cmd, flags);
953 static __rte_always_inline uint16_t
954 nix_xmit_pkts_vector(void *tx_queue, struct rte_mbuf **tx_pkts,
955 uint16_t pkts, uint64_t *cmd, const uint16_t flags)
957 RTE_SET_USED(tx_queue);
958 RTE_SET_USED(tx_pkts);
966 #define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags) \
967 static uint16_t __rte_noinline __rte_hot \
968 otx2_nix_xmit_pkts_ ## name(void *tx_queue, \
969 struct rte_mbuf **tx_pkts, uint16_t pkts) \
973 /* For TSO inner checksum is a must */ \
974 if (((flags) & NIX_TX_OFFLOAD_TSO_F) && \
975 !((flags) & NIX_TX_OFFLOAD_L3_L4_CSUM_F)) \
977 return nix_xmit_pkts(tx_queue, tx_pkts, pkts, cmd, flags); \
980 NIX_TX_FASTPATH_MODES
983 #define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags) \
984 static uint16_t __rte_noinline __rte_hot \
985 otx2_nix_xmit_pkts_mseg_ ## name(void *tx_queue, \
986 struct rte_mbuf **tx_pkts, uint16_t pkts) \
988 uint64_t cmd[(sz) + NIX_TX_MSEG_SG_DWORDS - 2]; \
990 /* For TSO inner checksum is a must */ \
991 if (((flags) & NIX_TX_OFFLOAD_TSO_F) && \
992 !((flags) & NIX_TX_OFFLOAD_L3_L4_CSUM_F)) \
994 return nix_xmit_pkts_mseg(tx_queue, tx_pkts, pkts, cmd, \
995 (flags) | NIX_TX_MULTI_SEG_F); \
998 NIX_TX_FASTPATH_MODES
1001 #define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags) \
1002 static uint16_t __rte_noinline __rte_hot \
1003 otx2_nix_xmit_pkts_vec_ ## name(void *tx_queue, \
1004 struct rte_mbuf **tx_pkts, uint16_t pkts) \
1008 /* VLAN, TSTMP, TSO is not supported by vec */ \
1009 if ((flags) & NIX_TX_OFFLOAD_VLAN_QINQ_F || \
1010 (flags) & NIX_TX_OFFLOAD_TSTAMP_F || \
1011 (flags) & NIX_TX_OFFLOAD_TSO_F) \
1013 return nix_xmit_pkts_vector(tx_queue, tx_pkts, pkts, cmd, (flags)); \
1016 NIX_TX_FASTPATH_MODES
1020 pick_tx_func(struct rte_eth_dev *eth_dev,
1021 const eth_tx_burst_t tx_burst[2][2][2][2][2][2][2])
1023 struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
1025 /* [SEC] [TSTMP] [NOFF] [VLAN] [OL3_OL4_CSUM] [IL3_IL4_CSUM] */
1026 eth_dev->tx_pkt_burst = tx_burst
1027 [!!(dev->tx_offload_flags & NIX_TX_OFFLOAD_SECURITY_F)]
1028 [!!(dev->tx_offload_flags & NIX_TX_OFFLOAD_TSO_F)]
1029 [!!(dev->tx_offload_flags & NIX_TX_OFFLOAD_TSTAMP_F)]
1030 [!!(dev->tx_offload_flags & NIX_TX_OFFLOAD_MBUF_NOFF_F)]
1031 [!!(dev->tx_offload_flags & NIX_TX_OFFLOAD_VLAN_QINQ_F)]
1032 [!!(dev->tx_offload_flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F)]
1033 [!!(dev->tx_offload_flags & NIX_TX_OFFLOAD_L3_L4_CSUM_F)];
1037 otx2_eth_set_tx_function(struct rte_eth_dev *eth_dev)
1039 struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
1041 const eth_tx_burst_t nix_eth_tx_burst[2][2][2][2][2][2][2] = {
1042 #define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags) \
1043 [f6][f5][f4][f3][f2][f1][f0] = otx2_nix_xmit_pkts_ ## name,
1045 NIX_TX_FASTPATH_MODES
1049 const eth_tx_burst_t nix_eth_tx_burst_mseg[2][2][2][2][2][2][2] = {
1050 #define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags) \
1051 [f6][f5][f4][f3][f2][f1][f0] = otx2_nix_xmit_pkts_mseg_ ## name,
1053 NIX_TX_FASTPATH_MODES
1057 const eth_tx_burst_t nix_eth_tx_vec_burst[2][2][2][2][2][2][2] = {
1058 #define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags) \
1059 [f6][f5][f4][f3][f2][f1][f0] = otx2_nix_xmit_pkts_vec_ ## name,
1061 NIX_TX_FASTPATH_MODES
1065 if (dev->scalar_ena ||
1066 (dev->tx_offload_flags &
1067 (NIX_TX_OFFLOAD_VLAN_QINQ_F | NIX_TX_OFFLOAD_TSTAMP_F |
1068 NIX_TX_OFFLOAD_TSO_F)))
1069 pick_tx_func(eth_dev, nix_eth_tx_burst);
1071 pick_tx_func(eth_dev, nix_eth_tx_vec_burst);
1073 if (dev->tx_offloads & DEV_TX_OFFLOAD_MULTI_SEGS)
1074 pick_tx_func(eth_dev, nix_eth_tx_burst_mseg);